From ddf57828c783c5259921b3e632aac4c81fffefa4 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:05:57 +0000 Subject: [PATCH 001/251] add draft templating, tests missing --- CHANGELOG.md | 9 + Project.toml | 2 + examples/working_with_aitemplates.jl | 76 ++++++ src/PromptingTools.jl | 17 +- src/llm_interface.jl | 34 ++- src/llm_openai.jl | 82 +++---- src/messages.jl | 48 ++-- src/precompilation.jl | 0 src/templates.jl | 222 ++++++++++++++++++ templates/classification/JudgeIsItTrue.json | 1 + templates/persona-task/AssistantAsk.json | 1 + .../persona-task/DetailOrientedTask.json | 1 + templates/persona-task/JuliaExpertAsk.json | 1 + .../persona-task/JuliaExpertCoTTask.json | 1 + 14 files changed, 432 insertions(+), 63 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 examples/working_with_aitemplates.jl create mode 100644 src/precompilation.jl create mode 100644 src/templates.jl create mode 100644 templates/classification/JudgeIsItTrue.json create mode 100644 templates/persona-task/AssistantAsk.json create mode 100644 templates/persona-task/DetailOrientedTask.json create mode 100644 templates/persona-task/JuliaExpertAsk.json create mode 100644 templates/persona-task/JuliaExpertCoTTask.json diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..9f2cfffe9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Added +- Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). \ No newline at end of file diff --git a/Project.toml b/Project.toml index 0454efa99..4a142b05f 100644 --- a/Project.toml +++ b/Project.toml @@ -7,11 +7,13 @@ version = "0.2.0-DEV" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" +PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" [compat] HTTP = "1" JSON3 = "1" OpenAI = "0.8.7" +PrecompileTools = "1" julia = "1.9,1.10" [extras] diff --git a/examples/working_with_aitemplates.jl b/examples/working_with_aitemplates.jl new file mode 100644 index 000000000..bf3f2ae57 --- /dev/null +++ b/examples/working_with_aitemplates.jl @@ -0,0 +1,76 @@ +using PromptingTools +const PT = PromptingTools +using PromptingTools: AIMessage, + SystemMessage, UserMessage, DataMessage, MetadataMessage, render +using JSON3 + +# Create a few templates directly +msg = [ + MetadataMessage(; content = "", + description = "Basic template for LLM-based classification whether provided statement is true/false/unknown.", + version = "1"), + SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + UserMessage("# Statement\n\n{{it}}"), +] + +JSON3.write("templates/test.json", msg) +JSON3.read("templates/test.json", Vector{PT.AbstractChatMessage}) + +# Standard definition +msg = [ + SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + UserMessage("# Statement\n\n{{it}}"), +] +PT.save_template(joinpath("templates", "classification", "JudgeIsItTrue.json"), + msg; + description = "LLM-based classification whether provided statement is true/false/unknown. Statement is provided via `it` placeholder.") + +msg = [ + SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer."), + UserMessage("# Question\n\n{{ask}}"), +] +PT.save_template(joinpath("templates", "persona-task", "AssistantAsk.json"), + msg; + description = "Helpful assistant for asking generic questions. Placeholders: `ask`") + +msg = [ + SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer."), + UserMessage("# Question\n\n{{ask}}"), +] +PT.save_template(joinpath("templates", "persona-task", "JuliaExpertAsk.json"), + msg; + description = "For asking questions about Julia language. Placeholders: `ask`") + +msg = [ + SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You precisely follow the given task and use the data when provided. First, think through your approach step by step. Only then start with the answer."), + UserMessage("# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}"), +] +PT.save_template(joinpath("templates", "persona-task", "JuliaExpertCoTTask.json"), msg; + description = "For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`") + +msg = [ + PT.SystemMessage("You are a world-class AI assistant. You are detail oriented, diligent, and have a great memory. Your communication is brief and concise."), + PT.UserMessage("# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}"), +] +PT.save_template(joinpath("templates", "persona-task", "DetailOrientedTask.json"), + msg; + description = "Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholders: `task`, `data`.") + +# Load one test +t, m = PT.load_template("templates/test.json") + +# Load templates +PT.load_templates!() + +# Render templates +template = AITemplate(:JudgeIsItTrue) + +render(PT.PROMPT_SCHEMA, template) +render(template) + +# Search for templates +tmp = aitemplates("template") + +# Hack for a nicer display in vscode +using DataFrames +DataFrame(tmp) |> vscodedisplay \ No newline at end of file diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 42feb29a8..5fedebf48 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -2,7 +2,9 @@ module PromptingTools using OpenAI using JSON3 +using JSON3: StructTypes using HTTP +using PrecompileTools # GLOBALS const MODEL_CHAT = "gpt-3.5-turbo" @@ -20,7 +22,7 @@ const MODEL_ALIASES = Dict("gpt3" => "gpt-3.5-turbo", "gpt4t" => "gpt-4-1106-preview", # 4t is for "4 turbo" "gpt3t" => "gpt-3.5-turbo-1106", # 3t is for "3 turbo" "ada" => "text-embedding-ada-002") -# below is defined in llm_interace.jl ! +# the below default is defined in llm_interace.jl ! # const PROMPT_SCHEMA = OpenAISchema() include("utils.jl") @@ -34,6 +36,11 @@ export AIMessage # export UserMessage, SystemMessage, DataMessage # for debugging only include("messages.jl") +export aitemplates, AITemplate +include("templates.jl") +const TEMPLATE_STORE = Dict{Symbol, Any}() +const TEMPLATE_METADATA = Vector{AITemplateMetadata}() + ## Individual interfaces include("llm_openai.jl") @@ -41,4 +48,12 @@ include("llm_openai.jl") export @ai_str, @aai_str include("macros.jl") +function __init__() + # Load templates + load_templates!() end + +# Enable precompilation to reduce start time +# @setup_workload include("precompilation.jl"); + +end # module PromptingTools diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 526abec38..36f6f5b86 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -40,7 +40,9 @@ end abstract type AbstractChatMLSchema <: AbstractPromptSchema end """ -ChatMLSchema is used by many open-source chatbots, by OpenAI models under the hood and by several models and inferfaces (eg, Ollama, vLLM) +ChatMLSchema is used by many open-source chatbots, by OpenAI models (under the hood) and by several models and inferfaces (eg, Ollama, vLLM) + +You can explore it on [tiktokenizer](https://tiktokenizer.vercel.app/) It uses the following conversation structure: ``` @@ -54,7 +56,19 @@ It uses the following conversation structure: """ struct ChatMLSchema <: AbstractChatMLSchema end -## Dispatch into defaults +abstract type AbstractManagedSchema <: AbstractPromptSchema end + +""" +Ollama by default manages different models and their associated prompt schemas when you pass `system_prompt` and `prompt` fields to the API. + +Warning: It works only for 1 system message and 1 user message, so anything more than that has to be rejected. + +If you need to pass more messagese / longer conversational history, you can use define the model-specific schema directly and pass your Ollama requests with `raw=true`, + which disables and templating and schema management by Ollama. +""" +struct OllamaManagedSchema <: AbstractManagedSchema end + +## Dispatch into default schema const PROMPT_SCHEMA = OpenAISchema() aigenerate(prompt; kwargs...) = aigenerate(PROMPT_SCHEMA, prompt; kwargs...) @@ -62,3 +76,19 @@ function aiembed(doc_or_docs, args...; kwargs...) aiembed(PROMPT_SCHEMA, doc_or_docs, args...; kwargs...) end aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) + +## Dispatch for AI templates (unpacks the messages) +function aigenerate(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aigenerate(schema, render(schema, template); kwargs...) +end +function aiclassify(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aiclassify(schema, render(schema, template); kwargs...) +end + +# Shortcut for symbols +function aigenerate(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aigenerate(schema, AITemplate(template); kwargs...) +end +function aiclassify(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aiclassify(schema, AITemplate(template); kwargs...) +end \ No newline at end of file diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 83d42258a..a0d925ef2 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -1,8 +1,8 @@ ## Rendering of converation history for the OpenAI API "Builds a history of the conversation to provide the prompt to the API. All kwargs are passed as replacements such that `{{key}}=>value` in the template.}}" function render(schema::AbstractOpenAISchema, - messages::Vector{<:AbstractMessage}; - kwargs...) + messages::Vector{<:AbstractMessage}; + kwargs...) ## conversation = Dict{String, String}[] # TODO: concat system messages together @@ -98,12 +98,12 @@ msg=aigenerate(conversation) ``` """ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt; verbose::Bool = true, - api_key::String = API_KEY, - model::String = MODEL_CHAT, - http_kwargs::NamedTuple = (retry_non_idempotent = true, - retries = 5, - readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), - kwargs...) + api_key::String = API_KEY, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) ## global MODEL_ALIASES, MODEL_COSTS ## Find the unique ID for the model alias provided @@ -126,15 +126,15 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt; verbose::Bool = end # Extend OpenAI create_chat to allow for testing/debugging function OpenAI.create_chat(schema::AbstractOpenAISchema, - api_key::AbstractString, - model::AbstractString, - conversation; - kwargs...) + api_key::AbstractString, + model::AbstractString, + conversation; + kwargs...) OpenAI.create_chat(api_key, model, conversation; kwargs...) end function OpenAI.create_chat(schema::TestEchoOpenAISchema, api_key::AbstractString, - model::AbstractString, - conversation; kwargs...) + model::AbstractString, + conversation; kwargs...) schema.model_id = model schema.inputs = conversation return schema @@ -194,14 +194,14 @@ msg.content' * msg.content[:, 1] # [1.0, 0.787] """ function aiembed(prompt_schema::AbstractOpenAISchema, - doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, - postprocess::F = identity; verbose::Bool = true, - api_key::String = API_KEY, - model::String = MODEL_EMBEDDING, - http_kwargs::NamedTuple = (retry_non_idempotent = true, - retries = 5, - readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), - kwargs...) where {F <: Function} + doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, + postprocess::F = identity; verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_EMBEDDING, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) where {F <: Function} ## global MODEL_ALIASES, MODEL_COSTS ## Find the unique ID for the model alias provided @@ -224,15 +224,15 @@ function aiembed(prompt_schema::AbstractOpenAISchema, end # Extend OpenAI create_embeddings to allow for testing function OpenAI.create_embeddings(schema::AbstractOpenAISchema, - api_key::AbstractString, - docs, - model::AbstractString; - kwargs...) + api_key::AbstractString, + docs, + model::AbstractString; + kwargs...) OpenAI.create_embeddings(api_key, docs, model; kwargs...) end function OpenAI.create_embeddings(schema::TestEchoOpenAISchema, api_key::AbstractString, - docs, - model::AbstractString; kwargs...) + docs, + model::AbstractString; kwargs...) schema.model_id = model schema.inputs = docs return schema @@ -246,9 +246,9 @@ end Classifies the given prompt/statement as true/false/unknown. -Note: this is a very simple classifier, it is not meant to be used in production. Credit goes to: https://twitter.com/AAAzzam/status/1669753721574633473 +Note: this is a very simple classifier, it is not meant to be used in production. Credit goes to [AAAzzam](https://twitter.com/AAAzzam/status/1669753721574633473). -It uses Logit bias trick to force the model to output only true/false/unknown. +It uses Logit bias trick and limits the output to 1 token to force the model to output only true/false/unknown. Output tokens used (via `api_kwargs`): - 837: ' true' @@ -272,24 +272,24 @@ tryparse(Bool, aiclassify("Is two plus two four?")) isa Bool # true Output of type `Nothing` marks that the model couldn't classify the statement as true/false. Ideally, we would like to re-use some helpful system prompt to get more accurate responses. -For this reason we have templates, eg, `:IsStatementTrue`. By specifying the template, we can provide our statement as the expected variable (`statement` in this case) +For this reason we have templates, eg, `:JudgeIsItTrue`. By specifying the template, we can provide our statement as the expected variable (`it` in this case) See that the model now correctly classifies the statement as "unknown". ```julia -aiclassify(:IsStatementTrue; statement = "Is two plus three a vegetable on Mars?") # unknown +aiclassify(:JudgeIsItTrue; it = "Is two plus three a vegetable on Mars?") # unknown ``` For better results, use higher quality models like gpt4, eg, ```julia -aiclassify(:IsStatementTrue; - statement = "If I had two apples and I got three more, I have five apples now.", +aiclassify(:JudgeIsItTrue; + it = "If I had two apples and I got three more, I have five apples now.", model = "gpt4") # true ``` """ function aiclassify(prompt_schema::AbstractOpenAISchema, prompt; - api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), - max_tokens = 1, temperature = 0), - kwargs...) + api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), + max_tokens = 1, temperature = 0), + kwargs...) ## msg = aigenerate(prompt_schema, prompt; @@ -297,11 +297,3 @@ function aiclassify(prompt_schema::AbstractOpenAISchema, prompt; kwargs...) return msg end -# Dispatch for templates -function aiclassify(prompt_schema::AbstractOpenAISchema, - template_sym::Symbol; - kwargs...) - # render template into prompt - prompt = render(prompt_schema, Val(template_sym)) - return aiclassify(prompt_schema, prompt; kwargs...) -end diff --git a/src/messages.jl b/src/messages.jl index b8f254d5c..d86e1c432 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -5,25 +5,38 @@ abstract type AbstractMessage end abstract type AbstractChatMessage <: AbstractMessage end # with text-based content abstract type AbstractDataMessage <: AbstractMessage end # with data-based content, eg, embeddings -Base.@kwdef mutable struct SystemMessage{T <: AbstractString} <: AbstractChatMessage +# Workaround to be able to add metadata to serialized conversations, templates, etc. +# Ignored by `render` directives +Base.@kwdef struct MetadataMessage{T <: AbstractString} <: AbstractChatMessage + content::T + description::String = "" + version::String = "1" + source::String = "" + _type::Symbol = :metadatamessage +end +Base.@kwdef struct SystemMessage{T <: AbstractString} <: AbstractChatMessage content::T variables::Vector{Symbol} = _extract_handlebar_variables(content) + _type::Symbol = :systemmessage end -Base.@kwdef mutable struct UserMessage{T <: AbstractString} <: AbstractChatMessage +Base.@kwdef struct UserMessage{T <: AbstractString} <: AbstractChatMessage content::T variables::Vector{Symbol} = _extract_handlebar_variables(content) + _type::Symbol = :usermessage end Base.@kwdef struct AIMessage{T <: Union{AbstractString, Nothing}} <: AbstractChatMessage content::T = nothing status::Union{Int, Nothing} = nothing tokens::Tuple{Int, Int} = (-1, -1) elapsed::Float64 = -1.0 + _type::Symbol = :aimessage end -Base.@kwdef mutable struct DataMessage{T <: Any} <: AbstractDataMessage +Base.@kwdef struct DataMessage{T <: Any} <: AbstractDataMessage content::T status::Union{Int, Nothing} = nothing tokens::Tuple{Int, Int} = (-1, -1) elapsed::Float64 = -1.0 + _type::Symbol = :datamessage end # content-only constructor @@ -45,6 +58,8 @@ function Base.show(io::IO, ::MIME"text/plain", m::AbstractChatMessage) printstyled(io, type_; color = :light_green) elseif m isa UserMessage printstyled(io, type_; color = :light_red) + elseif m isa MetadataMessage + printstyled(io, type_; color = :light_blue) else print(io, type_) end @@ -59,8 +74,8 @@ end ## Dispatch for render function render(schema::AbstractPromptSchema, - messages::Vector{<:AbstractMessage}; - kwargs...) + messages::Vector{<:AbstractMessage}; + kwargs...) render(schema, messages; kwargs...) end function render(schema::AbstractPromptSchema, msg::AbstractMessage; kwargs...) @@ -70,13 +85,16 @@ function render(schema::AbstractPromptSchema, msg::AbstractString; kwargs...) render(schema, [UserMessage(; content = msg)]; kwargs...) end -## Prompt Templates -# ie, a way to re-use similar prompting patterns (eg, aiclassifier) -# flow: template -> messages |+ kwargs variables -> chat history -# Defined through Val() to allow for dispatch -function render(prompt_schema::AbstractOpenAISchema, template::Val{:IsStatementTrue}) - [ - SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), - UserMessage("##Statement\n\n{{statement}}"), - ] -end +## Serialization via JSON3 +StructTypes.StructType(::Type{AbstractChatMessage}) = StructTypes.AbstractType() +StructTypes.StructType(::Type{MetadataMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{SystemMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{UserMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{AIMessage}) = StructTypes.Struct() +StructTypes.subtypekey(::Type{AbstractChatMessage}) = :_type +function StructTypes.subtypes(::Type{AbstractChatMessage}) + (usermessage = UserMessage, + aimessage = AIMessage, + systemmessage = SystemMessage, + metadatamessage = MetadataMessage) +end \ No newline at end of file diff --git a/src/precompilation.jl b/src/precompilation.jl new file mode 100644 index 000000000..e69de29bb diff --git a/src/templates.jl b/src/templates.jl new file mode 100644 index 000000000..496443b5a --- /dev/null +++ b/src/templates.jl @@ -0,0 +1,222 @@ +# This file contains the templating system which translates a symbol (=template name) into a set of messages under the specified schema. +# Templates are stored as JSON files in the `templates` folder. +# Once loaded, they are stored in global variable `TEMPLATE_STORE` +# +# Flow: template -> messages |+ kwargs variables -> chat history to pass to the model + +## Types +""" + AITemplate + +AITemplate is a template for a conversation prompt. + This type is merely a container for the template name, which is resolved into a set of messages (=prompt) by `render`. + +# Naming Convention +- Template names should be in CamelCase +- Follow the format `......` where possible, eg, `JudgeIsItTrue`, `` + - Starting with the Persona (=System prompt), eg, `Judge` = persona is meant to `judge` some provided information + - Variable to be filled in with context, eg, `It` = placeholder `it` + - Ending with the variable name is helpful, eg, `JuliaExpertTask` for a persona to be an expert in Julia language and `task` is the placeholder name +- Ideally, the template name should be self-explanatory, eg, `JudgeIsItTrue` = persona is meant to `judge` some provided information where it is true or false + +# Examples +```julia +julia> AITemplate(:JudgeIsItTrue) +``` + +""" +struct AITemplate + name::Symbol +end + +"Helper for easy searching and reviewing of templates. Defined on loading of each template." +Base.@kwdef struct AITemplateMetadata + name::Symbol + description::String = "" + version::String = "-" + wordcount::Int + variables::Vector{Symbol} = Symbol[] + system_preview::String = "" + user_preview::String = "" + source::String = "" +end +function Base.show(io::IO, ::MIME"text/plain", t::AITemplateMetadata) + # just dumping seems to give ok output + dump(IOContext(io, :limit => true), t, maxdepth = 1) +end +# overload also the vector printing for nicer search results with `aitemplates` +function Base.show(io::IO, m::MIME"text/plain", v::Vector{<:AITemplateMetadata}) + printstyled(io, "$(length(v))-element Vector{AITemplateMetadata}:"; color = :light_blue) + println(io) + [(show(io, m, v[i]); println(io)) for i in eachindex(v)] + nothing +end + +## Rendering messages from templates +function render(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + global TEMPLATE_STORE + haskey(TEMPLATE_STORE, template.name) || + error("Template $(template.name) not found in TEMPLATE_STORE") + # get template + return TEMPLATE_STORE[template.name] +end +# dispatch on default schema +function render(template::AITemplate; kwargs...) + global PROMPT_SCHEMA + render(PROMPT_SCHEMA, template; kwargs...) +end + +## Loading / Saving +"Saves provided messaging template (`messages`) to `io_or_file`. Automatically adds metadata based on provided keyword arguments." +function save_template(io_or_file::Union{IO, AbstractString}, + messages::AbstractVector{<:AbstractChatMessage}; + content::AbstractString = "Template Metadata", + description::AbstractString = "", + version::AbstractString = "1", + source::AbstractString = "") + + # create metadata + metadata_msg = MetadataMessage(; content, description, version, source) + + # save template to IO or file + JSON3.write(io_or_file, [metadata_msg, messages...]) +end +"Loads messaging template from `io_or_file` and returns tuple of template messages and metadata." +function load_template(io_or_file::Union{IO, AbstractString}) + messages = JSON3.read(io_or_file, Vector{AbstractChatMessage}) + template, metadata = AbstractChatMessage[], MetadataMessage[] + for i in eachindex(messages) + msg = messages[i] + if msg isa MetadataMessage + push!(metadata, msg) + else + push!(template, msg) + end + end + return template, metadata +end + +""" + remove_templates!() + +Removes all templates from `TEMPLATE_STORE` and `TEMPLATE_METADATA`. +""" +remove_templates!(; store = TEMPLATE_STORE, metadata_store = TEMPLATE_METADATA) = (empty!(store); empty!(metadata_store); nothing) + +""" + load_templates!(; remove_templates::Bool=true) + +Loads templates from folder `templates/` in the package root and stores them in `TEMPLATE_STORE` and `TEMPLATE_METADATA`. + +Note: Automatically removes any existing templates and metadata from `TEMPLATE_STORE` and `TEMPLATE_METADATA` if `remove_templates=true`. +""" +function load_templates!(dir_templates::String = joinpath(@__DIR__, "..", "templates"); + remove_templates::Bool = true, + store::Dict{Symbol, <:Any} = TEMPLATE_STORE, + metadata_store::Vector{<:AITemplateMetadata} = TEMPLATE_METADATA,) + # first remove any old templates and their metadata + remove_templates && remove_templates!(; store, metadata_store) + # recursively load all templates from the `templates` folder + for (root, dirs, files) in walkdir(dir_templates) + for file in files + if endswith(file, ".json") + template_name = Symbol(split(basename(file), ".")[begin]) + template, metadata_msgs = load_template(joinpath(root, file)) + # add to store + if haskey(store, template_name) + @warn("Template $(template_name) already exists, overwriting! Metadata will be duplicated.") + end + store[template_name] = template + + # prepare the metadata + wordcount = 0 + system_preview = "" + user_preview = "" + variables = Symbol[] + for i in eachindex(template) + msg = template[i] + wordcount += length(msg.content) + if hasproperty(msg, :variables) + append!(variables, msg.variables) + end + # truncate previews to 100 characters + if msg isa SystemMessage && length(system_preview) < 100 + system_preview *= first(msg.content, 100) + elseif msg isa UserMessage && length(user_preview) < 100 + user_preview *= first(msg.content, 100) + end + end + if !isempty(metadata_msgs) + # use the first metadata message found if available + meta = first(metadata_msgs) + metadata = AITemplateMetadata(; name = template_name, + meta.description, meta.version, meta.source, + wordcount, + system_preview = first(system_preview, 100), + user_preview = first(user_preview, 100), + variables = unique(variables)) + else + metadata = AITemplateMetadata(; name = template_name, + wordcount, + system_preview = first(system_preview, 100), + user_preview = first(user_preview, 100), + variables = unique(variables)) + end + # add metadata to store + push!(metadata_store, metadata) + end + end + end + return nothing +end + +## Searching for templates +""" + aitemplates + +Find easily the most suitable templates for your use case. + +You can search by: +- `query::Symbol` which looks look only for partial matches in the template `name` +- `query::AbstractString` which looks for partial matches in the template `name` or `description` +- `query::Regex` which looks for matches in the template `name`, `description` or any of the message previews + +# Examples +```julia + +``` +""" +function aitemplates end +"Find the top `limit` templates whose `name::Symbol` partially matches the `query_name::Symbol` in `TEMPLATE_METADATA`." +function aitemplates(query_name::Symbol; + limit::Int = 10, + metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) + query_str = lowercase(string(query_name)) + found_templates = filter(x -> occursin(query_str, + lowercase(string(x.name))), metadata_store) + return first(found_templates, limit) +end +"Find the top `limit` templates whose `name` or `description` fields partially match the `query_key::String` in `TEMPLATE_METADATA`." +function aitemplates(query_key::AbstractString; + limit::Int = 10, + metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) + query_str = lowercase(query_key) + found_templates = filter(x -> occursin(query_str, lowercase(string(x.name))) || + occursin(query_str, lowercase(string(x.description))), + metadata_store) + return first(found_templates, limit) +end +"Find the top `limit` templates where provided `query_key::Regex` matches either of `name`, `description` or previews or User or System messages in `TEMPLATE_METADATA`." +function aitemplates(query_key::Regex; + limit::Int = 10, + metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) + found_templates = filter(x -> occursin(query_key, + string(x.name)) || + occursin(query_key, + x.description) || + occursin(query_key, + x.system_preview) || + occursin(query_key, x.user_preview), + metadata_store) + return first(found_templates, limit) +end \ No newline at end of file diff --git a/templates/classification/JudgeIsItTrue.json b/templates/classification/JudgeIsItTrue.json new file mode 100644 index 000000000..48b41c40d --- /dev/null +++ b/templates/classification/JudgeIsItTrue.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"LLM-based classification whether provided statement is true/false/unknown. Statement is provided via `it` placeholder.","version":"1","source":"","_type":"metadatamessage"},{"content":"You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide.","variables":[],"_type":"systemmessage"},{"content":"# Statement\n\n{{it}}","variables":["it"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/persona-task/AssistantAsk.json b/templates/persona-task/AssistantAsk.json new file mode 100644 index 000000000..7733975ba --- /dev/null +++ b/templates/persona-task/AssistantAsk.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"Helpful assistant for asking generic questions. Placeholders: `ask`","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.","variables":[],"_type":"systemmessage"},{"content":"# Question\n\n{{ask}}","variables":["ask"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/persona-task/DetailOrientedTask.json b/templates/persona-task/DetailOrientedTask.json new file mode 100644 index 000000000..1884f5f1a --- /dev/null +++ b/templates/persona-task/DetailOrientedTask.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholders: `task`, `data`.","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class AI assistant. You are detail oriented, diligent, and have a great memory. Your communication is brief and concise.","variables":[],"_type":"systemmessage"},{"content":"# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}","variables":["task","data"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertAsk.json b/templates/persona-task/JuliaExpertAsk.json new file mode 100644 index 000000000..21621ea88 --- /dev/null +++ b/templates/persona-task/JuliaExpertAsk.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For asking questions about Julia language. Placeholders: `ask`","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.","variables":[],"_type":"systemmessage"},{"content":"# Question\n\n{{ask}}","variables":["ask"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertCoTTask.json b/templates/persona-task/JuliaExpertCoTTask.json new file mode 100644 index 000000000..157afa29c --- /dev/null +++ b/templates/persona-task/JuliaExpertCoTTask.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You precisely follow the given task and use the data when provided. First, think through your approach step by step. Only then start with the answer.","variables":[],"_type":"systemmessage"},{"content":"# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}","variables":["task","data"],"_type":"usermessage"}] \ No newline at end of file From aaaafb587fbd847828936332e552587a8af80bc2 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:17:05 +0000 Subject: [PATCH 002/251] update example + tests --- examples/working_with_aitemplates.jl | 125 ++++++++++++--------------- src/llm_interface.jl | 18 +--- src/llm_openai.jl | 9 +- src/messages.jl | 7 ++ src/precompilation.jl | 15 ++++ src/templates.jl | 16 ++++ test/messages.jl | 4 +- test/runtests.jl | 2 + test/templates.jl | 74 ++++++++++++++++ 9 files changed, 175 insertions(+), 95 deletions(-) create mode 100644 test/templates.jl diff --git a/examples/working_with_aitemplates.jl b/examples/working_with_aitemplates.jl index bf3f2ae57..000a1a082 100644 --- a/examples/working_with_aitemplates.jl +++ b/examples/working_with_aitemplates.jl @@ -1,76 +1,57 @@ +# This file contains examples of how to work with AITemplate(s). + using PromptingTools const PT = PromptingTools -using PromptingTools: AIMessage, - SystemMessage, UserMessage, DataMessage, MetadataMessage, render -using JSON3 - -# Create a few templates directly -msg = [ - MetadataMessage(; content = "", - description = "Basic template for LLM-based classification whether provided statement is true/false/unknown.", - version = "1"), - SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), - UserMessage("# Statement\n\n{{it}}"), -] - -JSON3.write("templates/test.json", msg) -JSON3.read("templates/test.json", Vector{PT.AbstractChatMessage}) - -# Standard definition -msg = [ - SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), - UserMessage("# Statement\n\n{{it}}"), -] -PT.save_template(joinpath("templates", "classification", "JudgeIsItTrue.json"), - msg; - description = "LLM-based classification whether provided statement is true/false/unknown. Statement is provided via `it` placeholder.") - -msg = [ - SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer."), - UserMessage("# Question\n\n{{ask}}"), -] -PT.save_template(joinpath("templates", "persona-task", "AssistantAsk.json"), - msg; - description = "Helpful assistant for asking generic questions. Placeholders: `ask`") - -msg = [ - SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer."), - UserMessage("# Question\n\n{{ask}}"), -] -PT.save_template(joinpath("templates", "persona-task", "JuliaExpertAsk.json"), - msg; - description = "For asking questions about Julia language. Placeholders: `ask`") - -msg = [ - SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You precisely follow the given task and use the data when provided. First, think through your approach step by step. Only then start with the answer."), - UserMessage("# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}"), -] -PT.save_template(joinpath("templates", "persona-task", "JuliaExpertCoTTask.json"), msg; - description = "For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`") - -msg = [ - PT.SystemMessage("You are a world-class AI assistant. You are detail oriented, diligent, and have a great memory. Your communication is brief and concise."), - PT.UserMessage("# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}"), -] -PT.save_template(joinpath("templates", "persona-task", "DetailOrientedTask.json"), - msg; - description = "Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholders: `task`, `data`.") - -# Load one test -t, m = PT.load_template("templates/test.json") - -# Load templates -PT.load_templates!() - -# Render templates -template = AITemplate(:JudgeIsItTrue) - -render(PT.PROMPT_SCHEMA, template) -render(template) - -# Search for templates -tmp = aitemplates("template") -# Hack for a nicer display in vscode +# LLM responses are only as good as the prompts you give them. However, great prompts take long time to write -- AITemplate are a way to re-use great prompts! +# +# AITemplates are just a collection of templated prompts (ie, set of "messages" that have placeholders like {{question}}) +# +# They are saved as JSON files in the `templates` directory. +# They are automatically loaded on package import, but you can always force a re-load with `PT.load_templates!()` +PT.load_templates!(); + +# You can (create them) and use them for any ai* function instead of a prompt: +# Let's use a template called :JuliaExpertAsk +# alternatively, you can use `AITemplate(:JuliaExpertAsk)` for cleaner dispatch +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +# ... some response from GPT3.5 +# +# You can see that it had a placeholder for the actual question (`ask`) that we provided as a keyword argument. +# We did not have to write any system prompt for personas, tone, etc. -- it was all provided by the template! +# +# How to know which templates are available? You can search for them with `aitemplates()`: +# You can search by Symbol (only for partial name match), String (partial match on name or description), or Regex (more fields) +tmps = aitemplates("JuliaExpertAsk") +# Outputs a list of available templates that match the search -- there is just one in this case: +# +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +# +# You see not just the description, but also a preview of the actual prompts, placeholders available, and the length (to gauge how much it would cost). +# +# If you use VSCode, you can display them in a nice scrollable table with `vscodedisplay`: using DataFrames -DataFrame(tmp) |> vscodedisplay \ No newline at end of file +DataFrame(tmp) |> vscodedisplay +# +# +# You can also just `render` the template to see the underlying mesages: +msgs = PT.render(AITemplate(:JuliaExpertAsk)) +# +# 2-element Vector{PromptingTools.AbstractChatMessage}: +# PromptingTools.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") +# PromptingTools.UserMessage("# Question\n\n{{ask}}") +# +# Now, you know exactly what's in the template! +# +# If you want to modify it, simply change it and save it as a new file with `save_template` (see the docs `?save_template` for more details): +# +# !!! If you have some good templates, please consider sharing them with the community by opening a PR to the `templates` directory! \ No newline at end of file diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 36f6f5b86..31576bc56 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -75,20 +75,4 @@ aigenerate(prompt; kwargs...) = aigenerate(PROMPT_SCHEMA, prompt; kwargs...) function aiembed(doc_or_docs, args...; kwargs...) aiembed(PROMPT_SCHEMA, doc_or_docs, args...; kwargs...) end -aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) - -## Dispatch for AI templates (unpacks the messages) -function aigenerate(schema::AbstractPromptSchema, template::AITemplate; kwargs...) - aigenerate(schema, render(schema, template); kwargs...) -end -function aiclassify(schema::AbstractPromptSchema, template::AITemplate; kwargs...) - aiclassify(schema, render(schema, template); kwargs...) -end - -# Shortcut for symbols -function aigenerate(schema::AbstractPromptSchema, template::Symbol; kwargs...) - aigenerate(schema, AITemplate(template); kwargs...) -end -function aiclassify(schema::AbstractPromptSchema, template::Symbol; kwargs...) - aiclassify(schema, AITemplate(template); kwargs...) -end \ No newline at end of file +aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) \ No newline at end of file diff --git a/src/llm_openai.jl b/src/llm_openai.jl index a0d925ef2..a5041e25c 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -39,7 +39,7 @@ end ## User-Facing API """ - aigenerate([prompt_schema::AbstractOpenAISchema,] prompt; verbose::Bool = true, + aigenerate([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, model::String = MODEL_CHAT, http_kwargs::NamedTuple = (; retry_non_idempotent = true, @@ -97,7 +97,8 @@ msg=aigenerate(conversation) # AIMessage("Ah, strong feelings you have for your iPhone. A Jedi's path, this is not... ") ``` """ -function aigenerate(prompt_schema::AbstractOpenAISchema, prompt; verbose::Bool = true, +function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; + verbose::Bool = true, api_key::String = API_KEY, model::String = MODEL_CHAT, http_kwargs::NamedTuple = (retry_non_idempotent = true, @@ -239,7 +240,7 @@ function OpenAI.create_embeddings(schema::TestEchoOpenAISchema, api_key::Abstrac end """ - aiclassify(prompt_schema::AbstractOpenAISchema, prompt; + aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), max_tokens = 1, temperature = 0), kwargs...) @@ -286,7 +287,7 @@ aiclassify(:JudgeIsItTrue; ``` """ -function aiclassify(prompt_schema::AbstractOpenAISchema, prompt; +function aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), max_tokens = 1, temperature = 0), kwargs...) diff --git a/src/messages.jl b/src/messages.jl index d86e1c432..bc460f7c7 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -5,6 +5,13 @@ abstract type AbstractMessage end abstract type AbstractChatMessage <: AbstractMessage end # with text-based content abstract type AbstractDataMessage <: AbstractMessage end # with data-based content, eg, embeddings +## Allowed inputs for ai* functions, AITemplate is resolved one level higher +const ALLOWED_PROMPT_TYPE = Union{ + AbstractString, + AbstractMessage, + Vector{<:AbstractMessage}, +} + # Workaround to be able to add metadata to serialized conversations, templates, etc. # Ignored by `render` directives Base.@kwdef struct MetadataMessage{T <: AbstractString} <: AbstractChatMessage diff --git a/src/precompilation.jl b/src/precompilation.jl index e69de29bb..0bc1802a6 100644 --- a/src/precompilation.jl +++ b/src/precompilation.jl @@ -0,0 +1,15 @@ +# Load templates +load_templates!() + +# API Calls +mock_response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) +schema = TestEchoOpenAISchema(; response = mock_response, status = 200) +# API calls +msg = aigenerate(schema, "I want to ask {{it}}"; it = "Is this correct?") +msg = aiclassify(schema, "I want to ask {{it}}"; it = "Is this correct?") + +# Use of Templates +template_name = :JudgeIsItTrue +msg = aigenerate(schema, template_name; it = "Is this correct?") +msg = aiclassify(schema, template_name; it = "Is this correct?") \ No newline at end of file diff --git a/src/templates.jl b/src/templates.jl index 496443b5a..ed1babb92 100644 --- a/src/templates.jl +++ b/src/templates.jl @@ -219,4 +219,20 @@ function aitemplates(query_key::Regex; occursin(query_key, x.user_preview), metadata_store) return first(found_templates, limit) +end + +## Dispatch for AI templates (unpacks the messages) +function aigenerate(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aigenerate(schema, render(schema, template); kwargs...) +end +function aiclassify(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aiclassify(schema, render(schema, template); kwargs...) +end + +# Shortcut for symbols +function aigenerate(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aigenerate(schema, AITemplate(template); kwargs...) +end +function aiclassify(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aiclassify(schema, AITemplate(template); kwargs...) end \ No newline at end of file diff --git a/test/messages.jl b/test/messages.jl index 0068712c3..ec1aa0a31 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -1,9 +1,9 @@ -using PromptingTools: AIMessage, SystemMessage, UserMessage, DataMessage +using PromptingTools: AIMessage, SystemMessage, MetadataMessage, UserMessage, DataMessage @testset "Message constructors" begin # Creates an instance of MSG with the given content string. content = "Hello, world!" - for T in [AIMessage, SystemMessage, UserMessage] + for T in [AIMessage, SystemMessage, UserMessage, MetadataMessage] # args msg = T(content) @test typeof(msg) <: T diff --git a/test/runtests.jl b/test/runtests.jl index a9e3a46ea..b7f138d24 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ using PromptingTools +using JSON3 using Test using Aqua const PT = PromptingTools @@ -10,4 +11,5 @@ end include("utils.jl") include("messages.jl") include("llm_openai.jl") + include("templates.jl") end diff --git a/test/templates.jl b/test/templates.jl new file mode 100644 index 000000000..51f027461 --- /dev/null +++ b/test/templates.jl @@ -0,0 +1,74 @@ +using PromptingTools: AbstractChatMessage, SystemMessage, UserMessage, MetadataMessage +using PromptingTools: render +using PromptingTools: save_template, load_template, load_templates!, aitemplates +using PromptingTools: TestEchoOpenAISchema + +@testset "Templates - save/load" begin + description = "Some description" + version = "1.1" + msgs = [ + SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + UserMessage("# Statement\n\n{{it}}"), + ] + tmp, _ = mktemp() + save_template(tmp, + msgs; + description, version) + template, metadata = load_template(tmp) + @test template == msgs + @test metadata[1].description == description + @test metadata[1].version == version + @test metadata[1].content == "Template Metadata" + @test metadata[1].source == "" +end + +@testset "Template rendering" begin + template = AITemplate(:JudgeIsItTrue) + expected_output = AbstractChatMessage[SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + UserMessage("# Statement\n\n{{it}}")] + @test expected_output == render(PT.PROMPT_SCHEMA, template) + @test expected_output == render(template) +end + +@testset "Templates - search" begin + # search all + tmps = aitemplates("") + @test tmps == PT.TEMPLATE_METADATA + # Exact search for JudgeIsItTrue + tmps = aitemplates(:JudgeIsItTrue) + @test length(tmps) == 1 + @test tmps[1].name == :JudgeIsItTrue + # Search for multiple with :Task in name + tmps1 = aitemplates(:Task) + @test length(tmps1) >= 1 + tmps2 = aitemplates("Task") + @test length(tmps2) == length(tmps1) + # Search via regex + tmps = aitemplates(r"IMPARTIAL AI JUDGE"i) + @test length(tmps) >= 1 +end + +@testset "Templates - Echo aigenerate call" begin + # E2E test for aigenerate with rendering template and filling the placeholders + template_name = :JudgeIsItTrue + expected_template_rendered = render(AITemplate(template_name)) |> + x -> render(PT.PROMPT_SCHEMA, x; it = "Is this correct?") + # corresponds to OpenAI API v1 + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + + # AIGeneration API - use AITemplate(:) + schema1 = TestEchoOpenAISchema(; response, status = 200) + msg = aigenerate(schema1, AITemplate(template_name); it = "Is this correct?") + @test schema1.inputs == expected_template_rendered + + # AIGeneration API - use template name as symbol + schema2 = TestEchoOpenAISchema(; response, status = 200) + msg = aigenerate(schema2, template_name; it = "Is this correct?") + @test schema2.inputs == expected_template_rendered + + # AIClassify API - use symbol dispatch + schema3 = TestEchoOpenAISchema(; response, status = 200) + msg = aiclassify(schema3, template_name; it = "Is this correct?") + @test schema3.inputs == expected_template_rendered +end \ No newline at end of file From f6fff58daa86b2a48a29b62fd8a35eb334893612 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 15 Nov 2023 20:38:43 +0000 Subject: [PATCH 003/251] reformat JSON + cleanup docs --- .gitignore | 4 +- README.md | 94 +++++++++++++++++-- examples/working_with_aitemplates.jl | 20 +++- src/PromptingTools.jl | 3 +- src/precompilation.jl | 4 +- src/templates.jl | 90 +++++++++++++++++- templates/classification/JudgeIsItTrue.json | 22 ++++- templates/persona-task/AssistantAsk.json | 22 ++++- .../persona-task/DetailOrientedTask.json | 23 ++++- templates/persona-task/JuliaExpertAsk.json | 22 ++++- .../persona-task/JuliaExpertCoTTask.json | 23 ++++- 11 files changed, 303 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 3efc9d887..40f731fe8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,5 @@ /docs/Manifest.toml /docs/build/ -/.DS_Store # macOS folder metadata -/.vscode \ No newline at end of file +**/.DS_Store +**/.vscode \ No newline at end of file diff --git a/README.md b/README.md index 9ada7e181..62342a48c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ ai"What is the capital of \$(country)?" # AIMessage("The capital of Spain is Madrid.") ``` -Pro tip: Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4`. Great for those extra hard questions! +Pro tip: Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4` (use `gpt4t` for the new GPT-4 Turbo model). Great for those extra hard questions! For more complex prompt templates, you can use handlebars-style templating and provide variables as keyword arguments: @@ -50,6 +50,8 @@ Pro tip: Use `asyncmap` to run multiple AI-powered tasks concurrently. Pro tip: If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` +For more practical examples, see the `examples/` folder and the [Advanced Examples](#advanced-examples) section below. + ## Table of Contents - [PromptingTools.jl: "Your Daily Dose of AI Efficiency."](#promptingtoolsjl-your-daily-dose-of-ai-efficiency) @@ -57,7 +59,9 @@ Pro tip: If you use slow models (like GPT-4), you can use async version of `@ai_ - [Table of Contents](#table-of-contents) - [Why PromptingTools.jl](#why-promptingtoolsjl) - [Advanced Examples](#advanced-examples) + - [Seamless Integration Into Your Workflow](#seamless-integration-into-your-workflow) - [Advanced Prompts / Conversations](#advanced-prompts--conversations) + - [Templated Prompts](#templated-prompts) - [Asynchronous Execution](#asynchronous-execution) - [Model Aliases](#model-aliases) - [Embeddings](#embeddings) @@ -90,11 +94,32 @@ Some features: ## Advanced Examples -TODO: +TODOs: + +- [ ] Add more practical examples (with DataFrames!) +- [ ] Add mini tasks with structured extraction +- [ ] Add an example of how to build a RAG app in 50 lines + +### Seamless Integration Into Your Workflow +Google search is great, but it's a context switch. You often have to open a few pages and read through the discussion to find the answer you need. Same with the ChatGPT website. + +Imagine you are in VSCode, editing your `.gitignore` file. How do I ignore a file in all subfolders again? -[ ] Add more practical examples (DataFrames!) -[ ] Add mini tasks with structured extraction -[ ] Add an example of how to build a RAG app in 50 lines +All you need to do is to type: +`aai"What to write in .gitignore to ignore file XYZ in any folder or subfolder?"` + +With `aai""` (as opposed to `ai""`), we make a non-blocking call to the LLM to not prevent you from continuing your work. When the answer is ready, we log it from the background: + +> [ Info: Tokens: 102 @ Cost: $0.0002 in 2.7 seconds +> ┌ Info: AIMessage> To ignore a file called "XYZ" in any folder or subfolder, you can add the following line to your .gitignore file: +> │ +> │ ``` +> │ **/XYZ +> │ ``` +> │ +> └ This pattern uses the double asterisk (`**`) to match any folder or subfolder, and then specifies the name of the file you want to ignore. + +You probably saved 3-5 minutes on this task and probably another 5-10 minutes, because of the context switch/distraction you avoided. It's a small win, but it adds up quickly. ### Advanced Prompts / Conversations @@ -126,6 +151,59 @@ aigenerate(new_conversation; object = "old iPhone") ``` > AIMessage("Hmm, possess an old iPhone, I do not. But experience with attachments, I have. Detachment, I learned. True power and freedom, it brings...") +### Templated Prompts + +With LLMs, the quality / robustness of your results depends on the quality of your prompts. But writing prompts is hard! That's why we offer a templating system to save you time and effort. + +To use a specific template (eg, `` to ask a Julia language): +```julia +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +``` + +The above is equivalent to a more verbose version that explicitly uses the dispatch on `AITemplate`: +```julia +msg = aigenerate(AITemplate(:JuliaExpertAsk); ask = "How do I add packages?") +``` + +Find available templates with `aitemplates`: +```julia +tmps = aitemplates("JuliaExpertAsk") +# Will surface one specific template +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +``` +The above gives you a good idea of what the template is about, what placeholders are available, and how much it would cost to use it (=wordcount). + +Search for all Julia-related templates: +```julia +tmps = aitemplates("Julia") +# 2-element Vector{AITemplateMetadata}... -> more to come later! +``` + +If you are on VSCode, you can leverage nice tabular display with `vscodedisplay`: +```julia +using DataFrames +tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay +``` + +I have my selected template, how do I use it? Just use the "name" in `aigenerate` or `aiclassify` + like you see in the first example! + +You can inspect any template by "rendering" it (this is what the LLM will see): +```julia +julia> AITemplate(:JudgeIsItTrue) |> PromptingTools.render +``` + +See more examples in the [examples/](examples/) folder. + ### Asynchronous Execution You can leverage `asyncmap` to run multiple AI-powered tasks concurrently, improving performance for batch operations. @@ -183,14 +261,16 @@ aiclassify("Is two plus two four?") System prompts and higher-quality models can be used for more complex tasks, including knowing when to defer to a human: ```julia -aiclassify(:IsStatementTrue; statement = "Is two plus three a vegetable on Mars?", model = "gpt4") +aiclassify(:JudgeIsItTrue; it = "Is two plus three a vegetable on Mars?", model = "gpt4t") # unknown ``` -In the above example, we used a prompt template `:IsStatementTrue`, which automatically expands into the following system prompt (and a separate user prompt): +In the above example, we used a prompt template `:JudgeIsItTrue`, which automatically expands into the following system prompt (and a separate user prompt): > "You are an impartial AI judge evaluating whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide." +For more information on templates, see the [Templated Prompts](#templated-prompts) section. + ### Data Extraction !!! Experimental diff --git a/examples/working_with_aitemplates.jl b/examples/working_with_aitemplates.jl index 000a1a082..520f3533b 100644 --- a/examples/working_with_aitemplates.jl +++ b/examples/working_with_aitemplates.jl @@ -52,6 +52,22 @@ msgs = PT.render(AITemplate(:JuliaExpertAsk)) # # Now, you know exactly what's in the template! # -# If you want to modify it, simply change it and save it as a new file with `save_template` (see the docs `?save_template` for more details): +# If you want to modify it, simply change it and save it as a new file with `save_template` (see the docs `?save_template` for more details). +# +# Let's adjust the previous template to be more specific to a data analysis question: +tpl = [PT.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. You're also a senior Data Scientist and proficient in data analysis in Julia. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + PT.UserMessage("# Question\n\n{{ask}}")] +# Templates are saved in the `templates` directory of the package. Name of the file will become the template name (eg, call `:JuliaDataExpertAsk`) +filename = joinpath(pkgdir(PromptingTools), + "templates", + "persona-task", + "JuliaDataExpertAsk.json") +PT.save_template(filename, + tpl; + description = "For asking data analysis questions in Julia language. Placeholders: `ask`") +rm(filename) # cleanup if we don't like it +# +# When you create a new template, remember to re-load the templates with `load_templates!()` so that it's available for use. +PT.load_templates!(); # -# !!! If you have some good templates, please consider sharing them with the community by opening a PR to the `templates` directory! \ No newline at end of file +# !!! If you have some good templates (or suggestions for the existing ones), please consider sharing them with the community by opening a PR to the `templates` directory! diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 5fedebf48..8e7697005 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -38,6 +38,7 @@ include("messages.jl") export aitemplates, AITemplate include("templates.jl") + const TEMPLATE_STORE = Dict{Symbol, Any}() const TEMPLATE_METADATA = Vector{AITemplateMetadata}() @@ -54,6 +55,6 @@ function __init__() end # Enable precompilation to reduce start time -# @setup_workload include("precompilation.jl"); +@compile_workload include("precompilation.jl") end # module PromptingTools diff --git a/src/precompilation.jl b/src/precompilation.jl index 0bc1802a6..61b0c0aa8 100644 --- a/src/precompilation.jl +++ b/src/precompilation.jl @@ -1,5 +1,5 @@ # Load templates -load_templates!() +load_templates!(); # API Calls mock_response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], @@ -12,4 +12,4 @@ msg = aiclassify(schema, "I want to ask {{it}}"; it = "Is this correct?") # Use of Templates template_name = :JudgeIsItTrue msg = aigenerate(schema, template_name; it = "Is this correct?") -msg = aiclassify(schema, template_name; it = "Is this correct?") \ No newline at end of file +msg = aiclassify(schema, template_name; it = "Is this correct?"); \ No newline at end of file diff --git a/src/templates.jl b/src/templates.jl index ed1babb92..772aa21ec 100644 --- a/src/templates.jl +++ b/src/templates.jl @@ -2,7 +2,7 @@ # Templates are stored as JSON files in the `templates` folder. # Once loaded, they are stored in global variable `TEMPLATE_STORE` # -# Flow: template -> messages |+ kwargs variables -> chat history to pass to the model +# Flow: template -> messages |+ kwargs variables -> "conversation" to pass to the model ## Types """ @@ -20,10 +20,55 @@ AITemplate is a template for a conversation prompt. - Ideally, the template name should be self-explanatory, eg, `JudgeIsItTrue` = persona is meant to `judge` some provided information where it is true or false # Examples + +Save time by re-using pre-made templates, just fill in the placeholders with the keyword arguments: +```julia +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +``` + +The above is equivalent to a more verbose version that explicitly uses the dispatch on `AITemplate`: +```julia +msg = aigenerate(AITemplate(:JuliaExpertAsk); ask = "How do I add packages?") +``` + +Find available templates with `aitemplates`: ```julia -julia> AITemplate(:JudgeIsItTrue) +tmps = aitemplates("JuliaExpertAsk") +# Will surface one specific template +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" ``` +The above gives you a good idea of what the template is about, what placeholders are available, and how much it would cost to use it (=wordcount). + +Search for all Julia-related templates: +```julia +tmps = aitemplates("Julia") +# 2-element Vector{AITemplateMetadata}... -> more to come later! +``` + +If you are on VSCode, you can leverage nice tabular display with `vscodedisplay`: +```julia +using DataFrames +tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay +``` + +I have my selected template, how do I use it? Just use the "name" in `aigenerate` or `aiclassify` + like you see in the first example! +You can inspect any template by "rendering" it (this is what the LLM will see): +```julia +julia> AITemplate(:JudgeIsItTrue) |> PromptingTools.render +``` + +See also: `save_template`, `load_template`, `load_templates!` for more advanced use cases (and the corresponding script in `examples/` folder) """ struct AITemplate name::Symbol @@ -60,7 +105,9 @@ function render(schema::AbstractPromptSchema, template::AITemplate; kwargs...) # get template return TEMPLATE_STORE[template.name] end + # dispatch on default schema +"Renders provided messaging template (`template`) under the default schema (`PROMPT_SCHEMA`)." function render(template::AITemplate; kwargs...) global PROMPT_SCHEMA render(PROMPT_SCHEMA, template; kwargs...) @@ -181,13 +228,46 @@ You can search by: - `query::AbstractString` which looks for partial matches in the template `name` or `description` - `query::Regex` which looks for matches in the template `name`, `description` or any of the message previews +# Keyword Arguments +- `limit::Int` limits the number of returned templates (Defaults to 10) + # Examples + +Find available templates with `aitemplates`: ```julia +tmps = aitemplates("JuliaExpertAsk") +# Will surface one specific template +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +``` +The above gives you a good idea of what the template is about, what placeholders are available, and how much it would cost to use it (=wordcount). +Search for all Julia-related templates: +```julia +tmps = aitemplates("Julia") +# 2-element Vector{AITemplateMetadata}... -> more to come later! +``` + +If you are on VSCode, you can leverage nice tabular display with `vscodedisplay`: +```julia +using DataFrames +tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay ``` + +I have my selected template, how do I use it? Just use the "name" in `aigenerate` or `aiclassify` + like you see in the first example! """ function aitemplates end -"Find the top `limit` templates whose `name::Symbol` partially matches the `query_name::Symbol` in `TEMPLATE_METADATA`." + +"Find the top-`limit` templates whose `name::Symbol` partially matches the `query_name::Symbol` in `TEMPLATE_METADATA`." function aitemplates(query_name::Symbol; limit::Int = 10, metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) @@ -196,7 +276,7 @@ function aitemplates(query_name::Symbol; lowercase(string(x.name))), metadata_store) return first(found_templates, limit) end -"Find the top `limit` templates whose `name` or `description` fields partially match the `query_key::String` in `TEMPLATE_METADATA`." +"Find the top-`limit` templates whose `name` or `description` fields partially match the `query_key::String` in `TEMPLATE_METADATA`." function aitemplates(query_key::AbstractString; limit::Int = 10, metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) @@ -206,7 +286,7 @@ function aitemplates(query_key::AbstractString; metadata_store) return first(found_templates, limit) end -"Find the top `limit` templates where provided `query_key::Regex` matches either of `name`, `description` or previews or User or System messages in `TEMPLATE_METADATA`." +"Find the top-`limit` templates where provided `query_key::Regex` matches either of `name`, `description` or previews or User or System messages in `TEMPLATE_METADATA`." function aitemplates(query_key::Regex; limit::Int = 10, metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) diff --git a/templates/classification/JudgeIsItTrue.json b/templates/classification/JudgeIsItTrue.json index 48b41c40d..9ca7c0e02 100644 --- a/templates/classification/JudgeIsItTrue.json +++ b/templates/classification/JudgeIsItTrue.json @@ -1 +1,21 @@ -[{"content":"Template Metadata","description":"LLM-based classification whether provided statement is true/false/unknown. Statement is provided via `it` placeholder.","version":"1","source":"","_type":"metadatamessage"},{"content":"You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide.","variables":[],"_type":"systemmessage"},{"content":"# Statement\n\n{{it}}","variables":["it"],"_type":"usermessage"}] \ No newline at end of file +[ + { + "content": "Template Metadata", + "description": "LLM-based classification whether provided statement is true/false/unknown. Statement is provided via `it` placeholder.", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Statement\n\n{{it}}", + "variables": [ + "it" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/AssistantAsk.json b/templates/persona-task/AssistantAsk.json index 7733975ba..d3d97601e 100644 --- a/templates/persona-task/AssistantAsk.json +++ b/templates/persona-task/AssistantAsk.json @@ -1 +1,21 @@ -[{"content":"Template Metadata","description":"Helpful assistant for asking generic questions. Placeholders: `ask`","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.","variables":[],"_type":"systemmessage"},{"content":"# Question\n\n{{ask}}","variables":["ask"],"_type":"usermessage"}] \ No newline at end of file +[ + { + "content": "Template Metadata", + "description": "Helpful assistant for asking generic questions. Placeholders: `ask`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Question\n\n{{ask}}", + "variables": [ + "ask" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/DetailOrientedTask.json b/templates/persona-task/DetailOrientedTask.json index 1884f5f1a..dd62cf896 100644 --- a/templates/persona-task/DetailOrientedTask.json +++ b/templates/persona-task/DetailOrientedTask.json @@ -1 +1,22 @@ -[{"content":"Template Metadata","description":"Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholders: `task`, `data`.","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class AI assistant. You are detail oriented, diligent, and have a great memory. Your communication is brief and concise.","variables":[],"_type":"systemmessage"},{"content":"# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}","variables":["task","data"],"_type":"usermessage"}] \ No newline at end of file +[ + { + "content": "Template Metadata", + "description": "Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholders: `task`, `data`.", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class AI assistant. You are detail oriented, diligent, and have a great memory. Your communication is brief and concise.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}", + "variables": [ + "task", + "data" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertAsk.json b/templates/persona-task/JuliaExpertAsk.json index 21621ea88..428cbe533 100644 --- a/templates/persona-task/JuliaExpertAsk.json +++ b/templates/persona-task/JuliaExpertAsk.json @@ -1 +1,21 @@ -[{"content":"Template Metadata","description":"For asking questions about Julia language. Placeholders: `ask`","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.","variables":[],"_type":"systemmessage"},{"content":"# Question\n\n{{ask}}","variables":["ask"],"_type":"usermessage"}] \ No newline at end of file +[ + { + "content": "Template Metadata", + "description": "For asking questions about Julia language. Placeholders: `ask`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Question\n\n{{ask}}", + "variables": [ + "ask" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertCoTTask.json b/templates/persona-task/JuliaExpertCoTTask.json index 157afa29c..a45885bc4 100644 --- a/templates/persona-task/JuliaExpertCoTTask.json +++ b/templates/persona-task/JuliaExpertCoTTask.json @@ -1 +1,22 @@ -[{"content":"Template Metadata","description":"For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`","version":"1","source":"","_type":"metadatamessage"},{"content":"You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You precisely follow the given task and use the data when provided. First, think through your approach step by step. Only then start with the answer.","variables":[],"_type":"systemmessage"},{"content":"# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}","variables":["task","data"],"_type":"usermessage"}] \ No newline at end of file +[ + { + "content": "Template Metadata", + "description": "For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You precisely follow the given task and use the data when provided. When no data is provided, create some examples. First, think through your approach step by step. Then implement the solution.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}", + "variables": [ + "task", + "data" + ], + "_type": "usermessage" + } +] \ No newline at end of file From efd411b4829b77242d45a8e41a810c04c7294497 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:02:16 +0000 Subject: [PATCH 004/251] add compats --- Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Project.toml b/Project.toml index 4a142b05f..18294e9af 100644 --- a/Project.toml +++ b/Project.toml @@ -10,10 +10,12 @@ OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" [compat] +Aqua = "0.7" HTTP = "1" JSON3 = "1" OpenAI = "0.8.7" PrecompileTools = "1" +Test = "<0.0.1, 1" julia = "1.9,1.10" [extras] From f41c74cdafb309b283d8ed89eade4e428e55e51b Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:44:08 +0000 Subject: [PATCH 005/251] add extraction --- README.md | 52 +++++++++- src/PromptingTools.jl | 5 +- src/extraction.jl | 180 ++++++++++++++++++++++++++++++++++ src/llm_interface.jl | 5 +- src/llm_openai.jl | 146 +++++++++++++++++++++++++++- src/messages.jl | 11 ++- src/precompilation.jl | 13 ++- test/extraction.jl | 221 ++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 4 +- 9 files changed, 621 insertions(+), 16 deletions(-) create mode 100644 src/extraction.jl create mode 100644 test/extraction.jl diff --git a/README.md b/README.md index 62342a48c..4ef2047a4 100644 --- a/README.md +++ b/README.md @@ -90,16 +90,17 @@ Some features: - **Easy to Remember**: All exported functions start with `ai...` for better discoverability - **Light Wraper Types**: Benefit from Julia's multiple dispatch by having AI outputs wrapped in specific types - **Minimal Dependencies**: Enjoy an easy addition to your global environment with very light dependencies -- **No Context Switching**: Access cutting-edge LLMs with no context switching and minimum extra keystrokes +- **No Context Switching**: Access cutting-edge LLMs with no context switching and minimum extra keystrokes directly in your REPL ## Advanced Examples TODOs: - [ ] Add more practical examples (with DataFrames!) -- [ ] Add mini tasks with structured extraction - [ ] Add an example of how to build a RAG app in 50 lines +Noteworthy functions: `aigenerate`, `aiembed`, `aiclassify`, `aiextract`, `aitemplates` + ### Seamless Integration Into Your Workflow Google search is great, but it's a context switch. You often have to open a few pages and read through the discussion to find the answer you need. Same with the ChatGPT website. @@ -273,9 +274,52 @@ For more information on templates, see the [Templated Prompts](#templated-prompt ### Data Extraction -!!! Experimental +Are you tired of extracting data with regex? You can use LLMs to extract structured data from text! + +All you have to do is to define the structure of the data you want to extract and the LLM will do the rest. + +Define a `return_type` with struct. Provide docstrings if needed (improves results and helps with documentation). + +Let's start with a hard task - extracting the current weather in a given location: +```julia +@enum TemperatureUnits celsius fahrenheit +"""Extract the current weather in a given location + +# Arguments +- `location`: The city and state, e.g. "San Francisco, CA" +- `unit`: The unit of temperature to return, either `celsius` or `fahrenheit` +""" +struct CurrentWeather + location::String + unit::Union{Nothing,TemperatureUnits} +end + +# Note that we provide the TYPE itself, not an instance of it! +msg = aiextract("What's the weather in Salt Lake City in C?"; return_type=CurrentWeather) +msg.content +# CurrentWeather("Salt Lake City, UT", celsius) +``` + +But you can use it even for more complex tasks, like extracting many entities from a text: + +```julia +"Person's age, height, and weight." +struct MyMeasurement + age::Int + height::Union{Int,Nothing} + weight::Union{Nothing,Float64} +end +struct ManyMeasurements + measurements::Vector{MyMeasurement} +end +msg = aiextract("James is 30, weighs 80kg. He's 180cm tall. Then Jack is 19 but really tall - over 190!"; return_type=ManyMeasurements) +msg.content.measurements +# 2-element Vector{MyMeasurement}: +# MyMeasurement(30, 180, 80.0) +# MyMeasurement(19, 190, nothing) +``` -TBU... with `aiextract` +There is even a wrapper to help you catch errors together with helpful explanations on why parsing failed. See `?PromptingTools.MaybeExtract` for more information. ### More Examples diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 8e7697005..8c72216ca 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -27,7 +27,7 @@ const MODEL_ALIASES = Dict("gpt3" => "gpt-3.5-turbo", include("utils.jl") -export aigenerate, aiembed, aiclassify +export aigenerate, aiembed, aiclassify, aiextract # export render # for debugging only include("llm_interface.jl") @@ -42,6 +42,9 @@ include("templates.jl") const TEMPLATE_STORE = Dict{Symbol, Any}() const TEMPLATE_METADATA = Vector{AITemplateMetadata}() +## Utilities to support structured extraction +include("extraction.jl") + ## Individual interfaces include("llm_openai.jl") diff --git a/src/extraction.jl b/src/extraction.jl new file mode 100644 index 000000000..74ab48463 --- /dev/null +++ b/src/extraction.jl @@ -0,0 +1,180 @@ +# These are utilities to support structured data extraction tasks through the OpenAI function calling interface (wrapped by `aiextract`) +# +# TODOs: +# - add support for enums +to_json_type(s::Type{<:AbstractString}) = "string" +to_json_type(n::Type{<:Real}) = "number" +to_json_type(n::Type{<:Integer}) = "integer" +to_json_type(b::Type{Bool}) = "boolean" +to_json_type(t::Type{<:Union{Missing, Nothing}}) = "null" +to_json_type(t::Type{<:Any}) = "string" # object? + +has_null_type(T::Type{Missing}) = true +has_null_type(T::Type{Nothing}) = true +has_null_type(T::Type) = T isa Union && any(has_null_type, Base.uniontypes(T)) +## For required fields, only Nothing is considered a null type (and be easily parsed by JSON3) +is_required_field(T::Type{Nothing}) = false +function is_required_field(T::Type) + if T isa Union + all(is_required_field, Base.uniontypes(T)) + else + true + end +end + +# Remove null types from Union etc. +remove_null_types(T::Type{Missing}) = Any +remove_null_types(T::Type{Nothing}) = Any +remove_null_types(T::Type{Union{Nothing, Missing}}) = Any +function remove_null_types(T::Type) + T isa Union ? Union{filter(!has_null_type, Base.uniontypes(T))...} : T +end + +function extract_docstring(type::Type; max_description_length::Int = 100) + ## plain struct has supertype Any + ## we ignore the ones that are subtypes for now (to prevent picking up Dicts, etc.) + if supertype(type) == Any + docs = Docs.doc(type) |> string + if !occursin("No documentation found.\n\n", docs) + return first(docs, max_description_length) + end + end + return "" +end + +function to_json_schema(orig_type; max_description_length::Int = 100) + schema = Dict{String, Any}() + type = remove_null_types(orig_type) + if isstructtype(type) + schema["type"] = "object" + schema["properties"] = Dict{String, Any}() + ## extract the field names and types + required_types = String[] + for (field_name, field_type) in zip(fieldnames(type), fieldtypes(type)) + schema["properties"][string(field_name)] = to_json_schema(remove_null_types(field_type); + max_description_length) + ## Hack: no null type (Nothing, Missing) implies it it is a required field + is_required_field(field_type) && push!(required_types, string(field_name)) + end + !isempty(required_types) && (schema["required"] = required_types) + ## docstrings + docs = extract_docstring(type; max_description_length) + !isempty(docs) && (schema["description"] = docs) + else + schema["type"] = to_json_type(type) + end + return schema +end +function to_json_schema(type::Type{<:AbstractString}; max_description_length::Int = 100) + Dict("type" => to_json_type(type)) +end +function to_json_schema(type::Type{T}; + max_description_length::Int = 100) where {T <: Union{AbstractSet, Tuple, AbstractArray}} + element_type = eltype(type) + return Dict("type" => "array", + "items" => to_json_schema(remove_null_types(element_type))) +end +function to_json_schema(type::Type{<:Enum}; max_description_length::Int = 100) + enum_options = Base.Enums.namemap(type) |> values .|> string + return Dict("type" => "string", + "enum" => enum_options) +end +function to_json_schema(type::Type{<:AbstractDict}; max_description_length::Int = 100) + throw(ArgumentError("Dicts are not supported yet as we cannot analyze their keys/values on a type-level. Use a nested Struct instead!")) +end + +""" + function_call_signature(datastructtype::Struct; max_description_length::Int = 100) + +Extract the argument names, types and docstrings from a struct to create the function call signature in JSON schema. + +You must provide a Struct type (not an instance of it) with some fields. + +Note: Fairly experimental, but works for combination of structs, arrays, strings and singletons. + +# Tips +- You can improve the quality of the extraction by writing a helpful docstring for your struct (or any nested struct). It will be provided as a description. + You can even include comments/descriptions about the individual fields. +- All fields are assumed to be required, unless you allow null values (eg, `::Union{Nothing, Int}`). Fields with `Nothing` will be treated as optional. +- Missing values are ignored (eg, `::Union{Missing, Int}` will be treated as Int). It's for broader compatibility and we cannot deserialize it as easily as `Nothing`. + +# Example + +Do you want to extract some specific measurements from a text like age, weight and height? +You need to define the information you need as a struct (`return_type`): +``` +struct MyMeasurement + age::Int + height::Union{Int,Nothing} + weight::Union{Nothing,Float64} +end +signature = function_call_signature(MyMeasurement) +# +# Dict{String, Any} with 3 entries: +# "name" => "MyMeasurement_extractor" +# "parameters" => Dict{String, Any}("properties"=>Dict{String, Any}("height"=>Dict{String, Any}("type"=>"integer"), "weight"=>Dic… +# "description" => "Represents person's age, height, and weight\n" +``` + +You can see that only the field `age` does not allow null values, hence, it's "required". +While `height` and `weight` are optional. +``` +signature["parameters"]["required"] +# ["age"] +``` + +If there are multiple items you want to extract, define a wrapper struct to get a Vector of `MyMeasurement`: +``` +struct MyMeasurementWrapper + measurements::Vector{MyMeasurement} +end + +Or if you want your extraction to fail gracefully when data isn't found, use `MaybeExtract{T}` wrapper (inspired by Instructor package!): +``` +using PromptingTools: MaybeExtract + +type = MaybeExtract{MyMeasurement} +# Effectively the same as: +# struct MaybeExtract{T} +# result::Union{T, Nothing} +# error::Bool // true if a result is found, false otherwise +# message::Union{Nothing, String} // Only present if no result is found, should be short and concise +# end + +# If LLM extraction fails, it will return a Dict with `error` and `message` fields instead of the result! +msg = aiextract("Extract measurements from the text: I am giraffe", type) + +# +# Dict{Symbol, Any} with 2 entries: +# :message => "Sorry, this feature is only available for humans." +# :error => true +``` +That way, you can handle the error gracefully and get a reason why extraction failed. +""" +function function_call_signature(datastructtype::Type; max_description_length::Int = 100) + !isstructtype(datastructtype) && + error("Only Structs are supported (provided type: $datastructtype") + ## Standardize the name + name = string(datastructtype, "_extractor") |> + x -> replace(x, r"[^0-9A-Za-z_-]" => "") |> x -> first(x, 64) + schema = Dict{String, Any}("name" => name, + "parameters" => to_json_schema(datastructtype; max_description_length)) + ## docstrings + docs = extract_docstring(datastructtype; max_description_length) + !isempty(docs) && (schema["description"] = docs) + return schema +end + +# This is kindly borrowed from the awesome Instructor package](https://github.com/jxnl/instructor/blob/main/instructor/dsl/maybe.py). +""" +Extract a result from the provided data, if any, otherwise set the error and message fields. + +# Arguments +- `error::Bool`: `true` if a result is found, `false` otherwise. +- `message::String`: Only present if no result is found, should be short and concise. +""" +struct MaybeExtract{T <: Any} + result::Union{Nothing, T} + error::Bool + message::Union{Nothing, String} +end \ No newline at end of file diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 31576bc56..a5ce942fe 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -11,7 +11,7 @@ function render end function aigenerate end function aiembed end function aiclassify end -function aiextract end # not implemented yet +function aiextract end ## Prompt Schema "Defines different prompting styles based on the model training and fine-tuning." @@ -75,4 +75,5 @@ aigenerate(prompt; kwargs...) = aigenerate(PROMPT_SCHEMA, prompt; kwargs...) function aiembed(doc_or_docs, args...; kwargs...) aiembed(PROMPT_SCHEMA, doc_or_docs, args...; kwargs...) end -aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) \ No newline at end of file +aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) +aiextract(prompt; kwargs...) = aiextract(PROMPT_SCHEMA, prompt; kwargs...) \ No newline at end of file diff --git a/src/llm_openai.jl b/src/llm_openai.jl index a5041e25c..45d9bb29e 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -6,7 +6,6 @@ function render(schema::AbstractOpenAISchema, ## conversation = Dict{String, String}[] # TODO: concat system messages together - # TODO: move system msg to the front has_system_msg = false # replace any handlebar variables in the messages @@ -53,7 +52,6 @@ Generate an AI response based on a given prompt using the OpenAI API. - `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema`) - `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` - `verbose`: A boolean indicating whether to print additional information. -- `prompt_schema`: An abstract schema for the prompt. - `api_key`: A string representing the API key for accessing the OpenAI API. - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. - `http_kwargs`: A named tuple of HTTP keyword arguments. @@ -63,7 +61,7 @@ Generate an AI response based on a given prompt using the OpenAI API. # Returns - `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. -See also: `ai_str` +See also: `ai_str`, `aai_str`, `aiembed`, `aiclassify`, `aiextract` # Example @@ -298,3 +296,145 @@ function aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ kwargs...) return msg end + +""" + aiextract([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; + return_type::Type, + verbose::Bool = true, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = (; + retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + +Extract required information (defined by a struct **`return_type`**) from the provided prompt by leveraging OpenAI function calling mode. + +This is a perfect solution for extracting structured information from text (eg, extract organization names in news articles, etc.) + +It's effectively a light wrapper around `aigenerate` call, which requires additional keyword argument `return_type` to be provided + and will enforce the model outputs to adhere to it. + +# Arguments +- `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema`) +- `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` +- `return_type`: A **struct** TYPE representing the the information we want to extract. Do not provide a struct instance, only the type. + If the struct has a docstring, it will be provided to the model as well. It's used to enforce structured model outputs or provide more information. +- `verbose`: A boolean indicating whether to print additional information. +- `api_key`: A string representing the API key for accessing the OpenAI API. +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `http_kwargs`: A named tuple of HTTP keyword arguments. +- `api_kwargs`: A named tuple of API keyword arguments. +- `kwargs`: Prompt variables to be used to fill the prompt/template + +# Returns +- `msg`: An `DataMessage` object representing the extracted data, including the content, status, tokens, and elapsed time. + Use `msg.content` to access the extracted data. + +See also: `function_call_signature`, `MaybeExtract`, `aigenerate` + +# Example + +Do you want to extract some specific measurements from a text like age, weight and height? +You need to define the information you need as a struct (`return_type`): +``` +"Person's age, height, and weight." +struct MyMeasurement + age::Int # required + height::Union{Int,Nothing} # optional + weight::Union{Nothing,Float64} # optional +end +msg = aiextract("James is 30, weighs 80kg. He's 180cm tall."; return_type=MyMeasurement) +# [ Info: Tokens: 129 @ Cost: \$0.0002 in 1.0 seconds +# PromptingTools.DataMessage(MyMeasurement) +msg.content +# MyMeasurement(30, 180, 80.0) +``` + +The fields that allow `Nothing` are marked as optional in the schema: +``` +msg = aiextract("James is 30."; return_type=MyMeasurement) +# MyMeasurement(30, nothing, nothing) +``` + +If there are multiple items you want to extract, define a wrapper struct to get a Vector of `MyMeasurement`: +``` +struct MyMeasurementWrapper + measurements::Vector{MyMeasurement} +end + +msg = aiextract("James is 30, weighs 80kg. He's 180cm tall. Then Jack is 19 but really tall - over 190!"; return_type=ManyMeasurements) + +msg.content.measurements +# 2-element Vector{MyMeasurement}: +# MyMeasurement(30, 180, 80.0) +# MyMeasurement(19, 190, nothing) +``` + +Or if you want your extraction to fail gracefully when data isn't found, use `MaybeExtract{T}` wrapper + (this trick is inspired by the Instructor package!): +``` +using PromptingTools: MaybeExtract + +type = MaybeExtract{MyMeasurement} +# Effectively the same as: +# struct MaybeExtract{T} +# result::Union{T, Nothing} // The result of the extraction +# error::Bool // true if a result is found, false otherwise +# message::Union{Nothing, String} // Only present if no result is found, should be short and concise +# end + +# If LLM extraction fails, it will return a Dict with `error` and `message` fields instead of the result! +msg = aiextract("Extract measurements from the text: I am giraffe", type) +msg.content +# MaybeExtract{MyMeasurement}(nothing, true, "I'm sorry, but I can only assist with human measurements.") +``` + +That way, you can handle the error gracefully and get a reason why extraction failed (in `msg.content.message`). + +Note that the error message refers to a giraffe not being a human, + because in our `MyMeasurement` docstring, we said that it's for people! +""" +function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; + return_type::Type, + verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + ## + global MODEL_ALIASES, MODEL_COSTS + ## Function calling specifics + functions = [function_call_signature(return_type)] + function_call = Dict(:name => only(functions)["name"]) + ## Add the function call signature to the api_kwargs + api_kwargs = merge(api_kwargs, (; functions, function_call)) + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + conversation = render(prompt_schema, prompt; kwargs...) + time = @elapsed r = create_chat(prompt_schema, api_key, + model_id, + conversation; + http_kwargs, + api_kwargs...) + # "Safe" parsing of the response - it still fails if JSON is invalid + content = try + r.response[:choices][begin][:message][:function_call][:arguments] |> + x -> JSON3.read(x, return_type) + catch e + @warn "There was an error parsing the response: $e. Using the raw response instead." + r.response[:choices][begin][:message][:function_call][:arguments] |> + JSON3.read |> copy + end + msg = DataMessage(; content, + status = Int(r.status), + tokens = (r.response[:usage][:prompt_tokens], + r.response[:usage][:completion_tokens]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + + return msg +end \ No newline at end of file diff --git a/src/messages.jl b/src/messages.jl index bc460f7c7..354557e82 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -75,8 +75,15 @@ end function Base.show(io::IO, ::MIME"text/plain", m::AbstractDataMessage) type_ = string(typeof(m)) |> x -> split(x, "{")[begin] printstyled(io, type_; color = :light_yellow) - size_str = (m.content) isa AbstractArray ? string(size(m.content)) : "-" - print(io, "(", typeof(m.content), " of size ", size_str, ")") + # for Embedding messages + if m.content isa AbstractArray + print(io, "(", typeof(m.content), " of size ", size(m.content), ")") + # for any non-types extraction messages + elseif m.content isa Dict{Symbol, <:Any} + print(io, "(Dict with keys: ", join(keys(m.content), ", "), ")") + else + print(io, "(", typeof(m.content), ")") + end end ## Dispatch for render diff --git a/src/precompilation.jl b/src/precompilation.jl index 61b0c0aa8..e7a809a06 100644 --- a/src/precompilation.jl +++ b/src/precompilation.jl @@ -2,14 +2,21 @@ load_templates!(); # API Calls -mock_response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], +mock_response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello!", + :function_call => Dict(:arguments => JSON3.write(Dict(:x => 1))))), + ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) schema = TestEchoOpenAISchema(; response = mock_response, status = 200) # API calls msg = aigenerate(schema, "I want to ask {{it}}"; it = "Is this correct?") msg = aiclassify(schema, "I want to ask {{it}}"; it = "Is this correct?") - +"With docstring" +struct X123 + x::Int +end +msg = aiextract(schema, "I want to ask {{it}}"; it = "Is this correct?", return_type = X123) # Use of Templates template_name = :JudgeIsItTrue msg = aigenerate(schema, template_name; it = "Is this correct?") -msg = aiclassify(schema, template_name; it = "Is this correct?"); \ No newline at end of file +msg = aiclassify(schema, template_name; it = "Is this correct?"); diff --git a/test/extraction.jl b/test/extraction.jl new file mode 100644 index 000000000..3a21c1964 --- /dev/null +++ b/test/extraction.jl @@ -0,0 +1,221 @@ +using PromptingTools: MaybeExtract, extract_docstring +using PromptingTools: has_null_type, is_required_field, remove_null_types, to_json_schema +using PromptingTools: function_call_signature + +# TODO: check more edge cases like empty structs + +@testset "has_null_type" begin + @test has_null_type(Number) == false + @test has_null_type(Nothing) == true + @test has_null_type(Union{Number, Nothing}) == true + @test has_null_type(Union{Number, Missing}) == true + @test has_null_type(Union{Number, String}) == false + @test is_required_field(Union{Nothing, Missing}) == false +end +@testset "is_required_field" begin + @test is_required_field(Number) == true + @test is_required_field(Nothing) == false + @test is_required_field(Union{Number, Nothing}) == false + @test is_required_field(Union{Number, Missing}) == true + @test is_required_field(Union{Number, String}) == true + @test is_required_field(Union{Nothing, Missing}) == false +end +@testset "remove_null_type" begin + @test remove_null_types(Number) == Number + @test remove_null_types(Nothing) == Any + @test remove_null_types(Union{Number, Nothing}) == Number + @test remove_null_types(Union{Number, Missing}) == Number + @test remove_null_types(Union{Number, String}) == Union{Number, String} + @test remove_null_types(Union{Nothing, Missing}) == Any +end +@testset "extract_docstring" begin + struct MyStructNoDocs + field1::Int + end + docstring = extract_docstring(MyStructNoDocs) + @test docstring == "" + + "I am a docstring." + struct MyStructHasDocs + field1::Int + end + docstring = extract_docstring(MyStructHasDocs) + @test docstring == "I am a docstring.\n" + + docstring = extract_docstring(MyStructHasDocs; max_description_length = 4) + @test docstring == "I am" + + # Ignore docs for generic types + docstring = extract_docstring(Dict) + @test docstring == "" + + ## intentionally broken -- cannot parse docstrings for Structs that have supertype different from Any + abstract type MyBaseType2 end + "Docstring is here!" + struct MyStructWithSuper2 <: MyBaseType2 + field1::Int + end + docstring = extract_docstring(MyStructWithSuper2) + @test_broken haskey(schema, "description") +end + +@testset "to_json_schema" begin + struct MyStruct + field1::Int + field2::String + field3::Union{Nothing, Float64} + field4::Union{Missing, Bool} + end + schema = to_json_schema(MyStruct) + # detect struct type + @test schema["type"] == "object" + # field extraction + @test haskey(schema, "properties") + @test haskey(schema["properties"], "field1") + @test haskey(schema["properties"], "field2") + @test schema["properties"]["field1"]["type"] == "integer" + @test schema["properties"]["field2"]["type"] == "string" + @test schema["properties"]["field3"]["type"] == "number" + @test schema["properties"]["field4"]["type"] == "boolean" + @test schema["required"] == ["field1", "field2", "field4"] + # no docs + @test !haskey(schema, "description") + + ## Check with docs + "Here is a docstring." + struct MyStructWithDocs + a::Int + end + schema = to_json_schema(MyStructWithDocs) + @test schema["type"] == "object" + @test haskey(schema, "description") + @test schema["description"] == "Here is a docstring.\n" + + ## Singleton types (ie, not collections) + schema = to_json_schema(Int) + @test schema["type"] == "integer" + schema = to_json_schema(Float32) + @test schema["type"] == "number" + schema = to_json_schema(Bool) + @test schema["type"] == "boolean" + + ## Check with nested types + schema = to_json_schema(Vector{Float32}) + @test schema["type"] == "array" + @test schema["items"]["type"] == "number" + + ## Special types + @enum TemperatureUnits celsius fahrenheit + schema = to_json_schema(TemperatureUnits) + @test schema["type"] == "string" + @test schema["enum"] == ["celsius", "fahrenheit"] + + ## Nested struct parsing + schema = to_json_schema(Vector{MyStruct}) + @test schema["type"] == "array" + schema_items = schema["items"] + @test schema_items["type"] == "object" + @test haskey(schema_items, "properties") + @test haskey(schema_items["properties"], "field1") + @test haskey(schema_items["properties"], "field2") + @test schema_items["properties"]["field1"]["type"] == "integer" + @test schema_items["properties"]["field2"]["type"] == "string" + @test schema_items["properties"]["field3"]["type"] == "number" + @test schema_items["properties"]["field4"]["type"] == "boolean" + @test schema_items["required"] == ["field1", "field2", "field4"] + + ## Struct in a Struct + struct MyStructWrapper + field1::MyStruct + field2::Int + end + schema = to_json_schema(MyStructWrapper) + @test schema["type"] == "object" + @test schema["properties"]["field2"]["type"] == "integer" + @test schema["required"] == ["field1", "field2"] + schema_mystruct = schema["properties"]["field1"] + @test schema_mystruct["properties"]["field1"]["type"] == "integer" + @test schema_mystruct["properties"]["field2"]["type"] == "string" + @test schema_mystruct["properties"]["field3"]["type"] == "number" + @test schema_mystruct["properties"]["field4"]["type"] == "boolean" + + ## Fallback to string (for tough unions) + @test to_json_schema(Any) == Dict("type" => "string") + @test to_json_schema(Union{Int, String, Real}) == Dict("type" => "string") + + ## Disallowed types + @test_throws ArgumentError to_json_schema(Dict{String, Int}) + + ## No required fields + struct MyStructNoRequired + field1::Union{Nothing, Int} + field2::Union{String, Nothing} + end + schema = to_json_schema(MyStructNoRequired) + @test !haskey(schema, "required") + + ## intentionally broken -- cannot parse docstrings for Structs that have supertype different from Any + abstract type MyBaseType end + "Docstring is here!" + struct MyStructFancy <: MyBaseType + field1::Int + field2::String + end + schema = to_json_schema(MyStructFancy) + @test schema["type"] == "object" + @test schema["properties"]["field1"]["type"] == "integer" + @test schema["properties"]["field2"]["type"] == "string" + @test schema["required"] == ["field1", "field2"] + @test_broken haskey(schema, "description") +end + +@testset "to_json_schema" begin + "Represents person's age, height, and weight" + struct MyMeasurement1 + age::Int + height::Union{Int, Nothing} + weight::Union{Nothing, Float64} + end + schema = to_json_schema(MaybeExtract{MyMeasurement1}) + @test schema["type"] == "object" + @test schema["properties"]["error"]["type"] == "boolean" + @test schema["properties"]["message"]["type"] == "string" + @test schema["required"] == ["error"] + @test haskey(schema, "description") + ## Check that the nested struct is extracted correctly + schema_measurement = schema["properties"]["result"] + @test schema_measurement["type"] == "object" + @test schema_measurement["properties"]["age"]["type"] == "integer" + @test schema_measurement["properties"]["height"]["type"] == "integer" + @test schema_measurement["properties"]["weight"]["type"] == "number" + @test schema_measurement["required"] == ["age"] + ## Check that the nested docstring is extracted correctly + @test schema_measurement["description"] == + "Represents person's age, height, and weight\n" +end + +@testset "function_call_signature" begin + "Some docstring" + struct MyMeasurement2 + age::Int + height::Union{Int, Nothing} + weight::Union{Nothing, Float64} + end + output = function_call_signature(MyMeasurement2)#|> JSON3.pretty + expected_output = Dict{String, Any}("name" => "MyMeasurement2_extractor", + "parameters" => Dict{String, Any}("properties" => Dict{String, Any}("height" => Dict{ + String, + Any, + }("type" => "integer"), + "weight" => Dict{String, Any}("type" => "number"), + "age" => Dict{String, Any}("type" => "integer")), + "required" => ["age"], + "type" => "object", + "description" => "Some docstring\n"), + "description" => "Some docstring\n") + @test output == expected_output + + ## MaybeWraper name cleanup + schema = function_call_signature(MaybeExtract{MyMeasurement2}) + @test schema["name"] == "MaybeExtractMyMeasurement2_extractor" +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index b7f138d24..cfb4e2482 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,11 +5,13 @@ using Aqua const PT = PromptingTools @testset "Code quality (Aqua.jl)" begin - Aqua.test_all(PromptingTools) + # Skipping unbound_args check because we need our `MaybeExtract` type to be unboard + Aqua.test_all(PromptingTools; unbound_args = false) end @testset "PromptingTools.jl" begin include("utils.jl") include("messages.jl") + include("extraction.jl") include("llm_openai.jl") include("templates.jl") end From 456709b508bf8800a070800bf6c05bd717a0ebbd Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:45:14 +0000 Subject: [PATCH 006/251] add changelog note --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f2cfffe9..8fe1b8a50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,4 +6,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). \ No newline at end of file +- Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). +- Add `aiextract` function to extract structured information from text quickly and easily. See `?aiextract` for more information. \ No newline at end of file From ad975847ca366a84bb236a8d02c9d6d66267d130 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:51:27 +0000 Subject: [PATCH 007/251] push updated prompts --- templates/extraction/ExtractData.json | 21 ++++++++++++++++++++ templates/general/BlankSystemUser.json | 21 ++++++++++++++++++++ templates/general/PromptEngineerForTask.json | 21 ++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 templates/extraction/ExtractData.json create mode 100644 templates/general/BlankSystemUser.json create mode 100644 templates/general/PromptEngineerForTask.json diff --git a/templates/extraction/ExtractData.json b/templates/extraction/ExtractData.json new file mode 100644 index 000000000..87fa410fd --- /dev/null +++ b/templates/extraction/ExtractData.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholder: `data`.", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class function calling and argument extraction expert. Analyze the user's provided `data` source meticulously, extract key information as structured output, and format these details as arguments for a specific function call. Ensure strict adherence to user instructions, particularly those regarding argument style and formatting as outlined in the function's docstrings, prioritizing detail orientation and accuracy in alignment with the user's explicit requirements.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Data\n\n{{data}}", + "variables": [ + "data" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/general/BlankSystemUser.json b/templates/general/BlankSystemUser.json new file mode 100644 index 000000000..d28cabdea --- /dev/null +++ b/templates/general/BlankSystemUser.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "Blank template for easy of prompt entry without the `Message` objects. Simply provide keyword arguments for `system` (=system prompt/persona) and `user` (=user/task/data prompt). Placeholders: `system`, `user`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "{{system}}", + "variables": ["system"], + "_type": "systemmessage" + }, + { + "content": "{{user}}", + "variables": [ + "user" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/general/PromptEngineerForTask.json b/templates/general/PromptEngineerForTask.json new file mode 100644 index 000000000..670083678 --- /dev/null +++ b/templates/general/PromptEngineerForTask.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "Suggest what could be a good system prompt/user prompt for a given `task`. Placeholder: `task`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class prompt engineering assistant. Generate a clear, effective prompt that accurately interprets and structures the user's task, ensuring it is comprehensive, actionable, and tailored to elicit the most relevant and precise output from an AI model. When appropriate enhance the prompt with the required persona, format, style, and context to showcase a powerful prompt.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}", + "variables": [ + "task" + ], + "_type": "usermessage" + } +] \ No newline at end of file From 054312e28ccf9a3ef25211d6c8af4664b55034bd Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:52:53 +0000 Subject: [PATCH 008/251] update metadata --- templates/extraction/ExtractData.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/extraction/ExtractData.json b/templates/extraction/ExtractData.json index 87fa410fd..1f64611a2 100644 --- a/templates/extraction/ExtractData.json +++ b/templates/extraction/ExtractData.json @@ -1,7 +1,7 @@ [ { "content": "Template Metadata", - "description": "Great template for detail-oriented tasks like string manipulations, data cleaning, etc. Placeholder: `data`.", + "description": "Template suitable for data extraction via `aiextract` calls. Placeholder: `data`.", "version": "1", "source": "", "_type": "metadatamessage" From 657584ca8072fae4c448146b56f1b5d984f9f718 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:53:24 +0000 Subject: [PATCH 009/251] fix a typo --- templates/general/PromptEngineerForTask.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/general/PromptEngineerForTask.json b/templates/general/PromptEngineerForTask.json index 670083678..d7126608e 100644 --- a/templates/general/PromptEngineerForTask.json +++ b/templates/general/PromptEngineerForTask.json @@ -1,7 +1,7 @@ [ { "content": "Template Metadata", - "description": "Suggest what could be a good system prompt/user prompt for a given `task`. Placeholder: `task`", + "description": "Prompt engineer that suggests what could be a good system prompt/user prompt for a given `task`. Placeholder: `task`", "version": "1", "source": "", "_type": "metadatamessage" From 7b0b6209b4bf511d4e943f183025d846050515e7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:51:45 +0000 Subject: [PATCH 010/251] update tests to account for new templates --- test/templates.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/templates.jl b/test/templates.jl index 51f027461..5a3a1b2c6 100644 --- a/test/templates.jl +++ b/test/templates.jl @@ -41,8 +41,8 @@ end # Search for multiple with :Task in name tmps1 = aitemplates(:Task) @test length(tmps1) >= 1 - tmps2 = aitemplates("Task") - @test length(tmps2) == length(tmps1) + tmps2 = aitemplates("Task") # broader search + @test length(tmps2) >= length(tmps1) # Search via regex tmps = aitemplates(r"IMPARTIAL AI JUDGE"i) @test length(tmps) >= 1 From 9c1abef8b16ddbf64919a96cac149d516a111225 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 19 Nov 2023 21:54:07 +0000 Subject: [PATCH 011/251] add aiscan --- Project.toml | 2 + README.md | 33 ++++++- src/PromptingTools.jl | 7 +- src/llm_interface.jl | 4 +- src/llm_openai.jl | 156 +++++++++++++++++++++++++++++++++- src/messages.jl | 64 +++++++++++++- src/precompilation.jl | 14 ++- src/templates.jl | 14 ++- src/utils.jl | 22 +++++ templates/visual/OCRTask.json | 21 +++++ test/data/julia.png | Bin 0 -> 38178 bytes test/llm_openai.jl | 42 ++++++++- test/messages.jl | 67 ++++++++++++++- test/utils.jl | 17 ++++ 14 files changed, 447 insertions(+), 16 deletions(-) create mode 100644 templates/visual/OCRTask.json create mode 100644 test/data/julia.png diff --git a/Project.toml b/Project.toml index 18294e9af..76050193d 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,7 @@ authors = ["J S @svilupp and contributors"] version = "0.2.0-DEV" [deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" @@ -11,6 +12,7 @@ PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" [compat] Aqua = "0.7" +Base64 = "<0.0.1, 1" HTTP = "1" JSON3 = "1" OpenAI = "0.8.7" diff --git a/README.md b/README.md index 4ef2047a4..a88b8782d 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ For more practical examples, see the `examples/` folder and the [Advanced Exampl - [Embeddings](#embeddings) - [Classification](#classification) - [Data Extraction](#data-extraction) + - [OCR and Image Comprehension](#ocr-and-image-comprehension) - [More Examples](#more-examples) - [Package Interface](#package-interface) - [Frequently Asked Questions](#frequently-asked-questions) @@ -189,7 +190,7 @@ tmps = aitemplates("Julia") # 2-element Vector{AITemplateMetadata}... -> more to come later! ``` -If you are on VSCode, you can leverage nice tabular display with `vscodedisplay`: +If you are on VSCode, you can leverage a nice tabular display with `vscodedisplay`: ```julia using DataFrames tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay @@ -321,6 +322,36 @@ msg.content.measurements There is even a wrapper to help you catch errors together with helpful explanations on why parsing failed. See `?PromptingTools.MaybeExtract` for more information. +### OCR and Image Comprehension + +With the `aiscan` function, you can interact with images as if they were text. + +You can simply describe a provided image: +```julia +msg = aiscan("Describe the image"; image_path="julia.png", model="gpt4v") +# [ Info: Tokens: 1141 @ Cost: \$0.0117 in 2.2 seconds +# AIMessage("The image shows a logo consisting of the word "julia" written in lowercase") +``` + +Or you can do an OCR of a screenshot. +Let's transcribe some SQL code from a screenshot (no more re-typing!), we use a template `:OCRTask`: + +```julia +# Screenshot of some SQL code +image_url = "https://www.sqlservercentral.com/wp-content/uploads/legacy/8755f69180b7ac7ee76a69ae68ec36872a116ad4/24622.png" +msg = aiscan(:OCRTask; image_url, model="gpt4v", task="Transcribe the SQL code in the image.", api_kwargs=(; max_tokens=2500)) + +# [ Info: Tokens: 362 @ Cost: \$0.0045 in 2.5 seconds +# AIMessage("```sql +# update Orders +``` + +You can add syntax highlighting of the outputs via Markdown +```julia +using Markdown +msg.content |> Markdown.parse +``` + ### More Examples TBU... diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 8c72216ca..c34a9fe61 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -1,5 +1,6 @@ module PromptingTools +using Base64: base64encode using OpenAI using JSON3 using JSON3: StructTypes @@ -16,9 +17,11 @@ const MODEL_COSTS = Dict("gpt-3.5-turbo" => (0.0015, 0.002), "gpt-3.5-turbo-1106" => (0.001, 0.002), "gpt-4" => (0.03, 0.06), "gpt-4-1106-preview" => (0.01, 0.03), + "gpt-4-vision-preview" => (0.01, 0.03), "text-embedding-ada-002" => (0.001, 0.0)) const MODEL_ALIASES = Dict("gpt3" => "gpt-3.5-turbo", "gpt4" => "gpt-4", + "gpt4v" => "gpt-4-vision-preview", # 4v is for "4 vision" "gpt4t" => "gpt-4-1106-preview", # 4t is for "4 turbo" "gpt3t" => "gpt-3.5-turbo-1106", # 3t is for "3 turbo" "ada" => "text-embedding-ada-002") @@ -27,13 +30,13 @@ const MODEL_ALIASES = Dict("gpt3" => "gpt-3.5-turbo", include("utils.jl") -export aigenerate, aiembed, aiclassify, aiextract +export aigenerate, aiembed, aiclassify, aiextract, aiscan # export render # for debugging only include("llm_interface.jl") ## Conversation history / Prompt elements export AIMessage -# export UserMessage, SystemMessage, DataMessage # for debugging only +# export UserMessage, UserMessageWithImages, SystemMessage, DataMessage # for debugging only include("messages.jl") export aitemplates, AITemplate diff --git a/src/llm_interface.jl b/src/llm_interface.jl index a5ce942fe..e3c17e148 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -12,6 +12,7 @@ function aigenerate end function aiembed end function aiclassify end function aiextract end +function aiscan end ## Prompt Schema "Defines different prompting styles based on the model training and fine-tuning." @@ -76,4 +77,5 @@ function aiembed(doc_or_docs, args...; kwargs...) aiembed(PROMPT_SCHEMA, doc_or_docs, args...; kwargs...) end aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) -aiextract(prompt; kwargs...) = aiextract(PROMPT_SCHEMA, prompt; kwargs...) \ No newline at end of file +aiextract(prompt; kwargs...) = aiextract(PROMPT_SCHEMA, prompt; kwargs...) +aiscan(prompt; kwargs...) = aiscan(PROMPT_SCHEMA, prompt; kwargs...) \ No newline at end of file diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 45d9bb29e..2e60cb919 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -1,11 +1,25 @@ ## Rendering of converation history for the OpenAI API -"Builds a history of the conversation to provide the prompt to the API. All kwargs are passed as replacements such that `{{key}}=>value` in the template.}}" +""" + render(schema::AbstractOpenAISchema, + messages::Vector{<:AbstractMessage}; + image_detail::AbstractString = "auto", + kwargs...) + +Builds a history of the conversation to provide the prompt to the API. All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. + +# Arguments +- `image_detail`: Only for `UserMessageWithImages`. It represents the level of detail to include for images. Can be `"auto"`, `"high"`, or `"low"`. + +""" function render(schema::AbstractOpenAISchema, messages::Vector{<:AbstractMessage}; + image_detail::AbstractString = "auto", kwargs...) ## - conversation = Dict{String, String}[] - # TODO: concat system messages together + @assert image_detail in ["auto", "high", "low"] "Image detail must be one of: auto, high, low" + ## + conversation = Dict{String, Any}[] + # TODO: concat multiple system messages together (2nd pass) has_system_msg = false # replace any handlebar variables in the messages @@ -23,6 +37,20 @@ function render(schema::AbstractOpenAISchema, for (key, value) in pairs(kwargs) if key in msg.variables] push!(conversation, Dict("role" => "user", "content" => replace(msg.content, replacements...))) + elseif msg isa UserMessageWithImages + replacements = ["{{$(key)}}" => value + for (key, value) in pairs(kwargs) if key in msg.variables] + # Build message content + content = Dict{String, Any}[Dict("type" => "text", + "text" => replace(msg.content, replacements...))] + # Add images + for img in msg.image_url + push!(content, + Dict("type" => "image_url", + "image_url" => Dict("url" => img, + "detail" => image_detail))) + end + push!(conversation, Dict("role" => "user", "content" => content)) elseif msg isa AIMessage push!(conversation, Dict("role" => "assistant", "content" => msg.content)) @@ -60,8 +88,9 @@ Generate an AI response based on a given prompt using the OpenAI API. # Returns - `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. + Use `msg.content` to access the extracted string. -See also: `ai_str`, `aai_str`, `aiembed`, `aiclassify`, `aiextract` +See also: `ai_str`, `aai_str`, `aiembed`, `aiclassify`, `aiextract`, `aiscan` # Example @@ -436,5 +465,124 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T ## Reporting verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + return msg +end + +""" +aiscan([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_detail::AbstractString = "auto", + attach_to_latest::Bool = true, + verbose::Bool = true, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = (; + retry_non_idempotent = true, + retries = 5, + readtimeout = 120), + api_kwargs::NamedTuple = = (; max_tokens = 2500), + kwargs...) + +Scans the provided image (`image_url` or `image_path`) with the goal provided in the `prompt`. + +Can be used for many multi-modal tasks, such as: OCR (transcribe text in the image), image captioning, image classification, etc. + +It's effectively a light wrapper around `aigenerate` call, which uses additional keyword arguments `image_url`, `image_path`, `image_detail` to be provided. + At least one image source (url or path) must be provided. + +# Arguments +- `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema`) +- `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` +- `image_url`: A string or vector of strings representing the URL(s) of the image(s) to scan. +- `image_path`: A string or vector of strings representing the path(s) of the image(s) to scan. +- `image_detail`: A string representing the level of detail to include for images. Can be `"auto"`, `"high"`, or `"low"`. See [OpenAI Vision Guide](https://platform.openai.com/docs/guides/vision) for more details. +- `attach_to_latest`: A boolean how to handle if a conversation with multiple `UserMessage` is provided. When `true`, the images are attached to the latest `UserMessage`. +- `verbose`: A boolean indicating whether to print additional information. +- `api_key`: A string representing the API key for accessing the OpenAI API. +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `http_kwargs`: A named tuple of HTTP keyword arguments. +- `api_kwargs`: A named tuple of API keyword arguments. +- `kwargs`: Prompt variables to be used to fill the prompt/template + +# Returns +- `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. + Use `msg.content` to access the extracted string. + +See also: `ai_str`, `aai_str`, `aigenerate`, `aiembed`, `aiclassify`, `aiextract` + +# Notes + +- All examples below use model "gpt4v", which is an alias for model ID "gpt-4-vision-preview" +- `max_tokens` in the `api_kwargs` is preset to 2500, otherwise OpenAI enforces a default of only a few hundred tokens (~300). If your output is truncated, increase this value + +# Example + +Describe the provided image: +```julia +msg = aiscan("Describe the image"; image_path="julia.png", model="gpt4v") +# [ Info: Tokens: 1141 @ Cost: \$0.0117 in 2.2 seconds +# AIMessage("The image shows a logo consisting of the word "julia" written in lowercase") +``` + +You can provide multiple images at once as a vector and ask for "low" level of detail (cheaper): +```julia +msg = aiscan("Describe the image"; image_path=["julia.png","python.png"], image_detail="low", model="gpt4v") +``` + +You can use this function as a nice and quick OCR (transcribe text in the image) with a template `:OCRTask`. +Let's transcribe some SQL code from a screenshot (no more re-typing!): + +```julia +# Screenshot of some SQL code +image_url = "https://www.sqlservercentral.com/wp-content/uploads/legacy/8755f69180b7ac7ee76a69ae68ec36872a116ad4/24622.png" +msg = aiscan(:OCRTask; image_url, model="gpt4v", task="Transcribe the SQL code in the image.", api_kwargs=(; max_tokens=2500)) + +# [ Info: Tokens: 362 @ Cost: \$0.0045 in 2.5 seconds +# AIMessage("```sql +# update Orders + +# You can add syntax highlighting of the outputs via Markdown +using Markdown +msg.content |> Markdown.parse +``` + +Notice that we enforce `max_tokens = 2500`. That's because OpenAI seems to default to ~300 tokens, which provides incomplete outputs. +Hence, we set this value to 2500 as a default. If you still get truncated outputs, increase this value. + +""" +function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_detail::AbstractString = "auto", + attach_to_latest::Bool = true, + verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = (; max_tokens = 2500), + kwargs...) + ## + global MODEL_ALIASES, MODEL_COSTS + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + ## Vision-specific functionality + msgs = attach_images_to_user_message(prompt; image_url, image_path, attach_to_latest) + ## Build the conversation, pass what image detail is required (if provided) + conversation = render(prompt_schema, msgs; image_detail, kwargs...) + ## Model call + time = @elapsed r = create_chat(prompt_schema, api_key, + model_id, + conversation; + http_kwargs, + api_kwargs...) + msg = AIMessage(; content = r.response[:choices][begin][:message][:content] |> strip, + status = Int(r.status), + tokens = (r.response[:usage][:prompt_tokens], + r.response[:usage][:completion_tokens]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + return msg end \ No newline at end of file diff --git a/src/messages.jl b/src/messages.jl index 354557e82..c614cd63f 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -31,6 +31,12 @@ Base.@kwdef struct UserMessage{T <: AbstractString} <: AbstractChatMessage variables::Vector{Symbol} = _extract_handlebar_variables(content) _type::Symbol = :usermessage end +Base.@kwdef struct UserMessageWithImages{T <: AbstractString} <: AbstractChatMessage + content::T + image_url::Vector{<:AbstractString} # no default! fail when not provided + variables::Vector{Symbol} = _extract_handlebar_variables(content) + _type::Symbol = :usermessagewithimages +end Base.@kwdef struct AIMessage{T <: Union{AbstractString, Nothing}} <: AbstractChatMessage content::T = nothing status::Union{Int, Nothing} = nothing @@ -47,9 +53,10 @@ Base.@kwdef struct DataMessage{T <: Any} <: AbstractDataMessage end # content-only constructor -function (MSG::Type{<:AbstractChatMessage})(s::AbstractString) - MSG(; content = s) +function (MSG::Type{<:AbstractChatMessage})(prompt::AbstractString) + MSG(; content = prompt) end +isusermessage(m::AbstractMessage) = m isa UserMessage # equality check for testing, only equal if all fields are equal and type is the same Base.var"=="(m1::AbstractMessage, m2::AbstractMessage) = false @@ -57,13 +64,64 @@ function Base.var"=="(m1::T, m2::T) where {T <: AbstractMessage} all([getproperty(m1, f) == getproperty(m2, f) for f in fieldnames(T)]) end +## Vision Models -- Constructor and Conversion +"Construct `UserMessageWithImages` with 1 or more images. Images can be either URLs or local paths." +function UserMessageWithImages(prompt::AbstractString; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing) + @assert !(isnothing(image_url) && isnothing(image_path)) "At least one of `image_url` and `image_path` must be provided." + url1 = !isnothing(image_url) ? _string_to_vector(image_url) : String[] + # Process local image + url2 = !isnothing(image_path) ? _string_to_vector(_encode_local_image(image_path)) : + String[] + return UserMessageWithImages(; + content = prompt, + image_url = vcat(url1, url2), + variables = _extract_handlebar_variables(prompt), + _type = :usermessagewithimage) +end + +# Attach image to user message +function attach_images_to_user_message(prompt::AbstractString; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + kwargs...) + UserMessageWithImages(prompt; image_url, image_path) +end +function attach_images_to_user_message(msg::UserMessageWithImages; kwargs...) + throw(AssertionError("Cannot attach additional images to UserMessageWithImages.")) +end +function attach_images_to_user_message(msg::UserMessage; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + kwargs...) + UserMessageWithImages(msg.content; image_url, image_path) +end +# automatically attach images to the latest user message, if not allowed, throw an error if more than 2 user messages provided +function attach_images_to_user_message(msgs::Vector{T}; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + attach_to_latest::Bool = true) where {T <: AbstractChatMessage} + # Check how to add images to UserMessage + count_user_msgs = count(isusermessage, msgs) + @assert attach_to_latest||(count_user_msgs <= 1) "At most one user message must be provided. Otherwise we would not know where to attach the images!" + @assert count_user_msgs>0 "At least one user message must be provided." + ## + idx = findlast(isusermessage, msgs) + # re-type to accept UserMessageWithImages type + msgs = convert(Vector{typejoin(UserMessageWithImages, T)}, msgs) + msgs[idx] = attach_images_to_user_message(msgs[idx]; image_url, image_path) + return msgs +end + +## Display methods function Base.show(io::IO, ::MIME"text/plain", m::AbstractChatMessage) type_ = string(typeof(m)) |> x -> split(x, "{")[begin] if m isa AIMessage printstyled(io, type_; color = :magenta) elseif m isa SystemMessage printstyled(io, type_; color = :light_green) - elseif m isa UserMessage + elseif m isa UserMessage || m isa UserMessageWithImages printstyled(io, type_; color = :light_red) elseif m isa MetadataMessage printstyled(io, type_; color = :light_blue) diff --git a/src/precompilation.jl b/src/precompilation.jl index e7a809a06..2e2f4510b 100644 --- a/src/precompilation.jl +++ b/src/precompilation.jl @@ -1,13 +1,14 @@ # Load templates load_templates!(); -# API Calls +# API Calls prep mock_response = Dict(:choices => [ Dict(:message => Dict(:content => "Hello!", :function_call => Dict(:arguments => JSON3.write(Dict(:x => 1))))), ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) schema = TestEchoOpenAISchema(; response = mock_response, status = 200) + # API calls msg = aigenerate(schema, "I want to ask {{it}}"; it = "Is this correct?") msg = aiclassify(schema, "I want to ask {{it}}"; it = "Is this correct?") @@ -16,7 +17,18 @@ struct X123 x::Int end msg = aiextract(schema, "I want to ask {{it}}"; it = "Is this correct?", return_type = X123) +image_url = "some_mock_url" +msg = aiscan(schema, "Describe the image"; image_url) + # Use of Templates template_name = :JudgeIsItTrue msg = aigenerate(schema, template_name; it = "Is this correct?") msg = aiclassify(schema, template_name; it = "Is this correct?"); +msg = aiextract(schema, + template_name; + it = "This doesn't make sense but do run it...", + return_type = X123); +msg = aiscan(schema, + template_name; + it = "Is the image a Julia logo?", + image_url = "some_link_to_julia_logo"); diff --git a/src/templates.jl b/src/templates.jl index 772aa21ec..fdb90d945 100644 --- a/src/templates.jl +++ b/src/templates.jl @@ -103,7 +103,7 @@ function render(schema::AbstractPromptSchema, template::AITemplate; kwargs...) haskey(TEMPLATE_STORE, template.name) || error("Template $(template.name) not found in TEMPLATE_STORE") # get template - return TEMPLATE_STORE[template.name] + return TEMPLATE_STORE[template.name] |> copy end # dispatch on default schema @@ -308,6 +308,12 @@ end function aiclassify(schema::AbstractPromptSchema, template::AITemplate; kwargs...) aiclassify(schema, render(schema, template); kwargs...) end +function aiextract(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aiextract(schema, render(schema, template); kwargs...) +end +function aiscan(schema::AbstractPromptSchema, template::AITemplate; kwargs...) + aiscan(schema, render(schema, template); kwargs...) +end # Shortcut for symbols function aigenerate(schema::AbstractPromptSchema, template::Symbol; kwargs...) @@ -315,4 +321,10 @@ function aigenerate(schema::AbstractPromptSchema, template::Symbol; kwargs...) end function aiclassify(schema::AbstractPromptSchema, template::Symbol; kwargs...) aiclassify(schema, AITemplate(template); kwargs...) +end +function aiextract(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aiextract(schema, AITemplate(template); kwargs...) +end +function aiscan(schema::AbstractPromptSchema, template::Symbol; kwargs...) + aiscan(schema, AITemplate(template); kwargs...) end \ No newline at end of file diff --git a/src/utils.jl b/src/utils.jl index 046ecae3e..1365fa166 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -2,6 +2,10 @@ function _extract_handlebar_variables(s::AbstractString) Symbol[Symbol(m[1]) for m in eachmatch(r"\{\{([^\}]+)\}\}", s)] end +# create a method for Vector{Dict} in UserMessageWithImage to extract handlebar variables for Dict keys +function _extract_handlebar_variables(vect::Vector{Dict{String, <:AbstractString}}) + unique([_extract_handlebar_variables(v) for d in vect for (k, v) in d if k == "text"]) +end # helper to produce summary message of how many tokens were used and for how much function _report_stats(msg, model::String, model_costs::AbstractDict = Dict()) @@ -11,3 +15,21 @@ function _report_stats(msg, model::String, model_costs::AbstractDict = Dict()) return "Tokens: $(sum(msg.tokens))$(cost_str) in $(round(msg.elapsed;digits=1)) seconds" end +# Loads and encodes the provided image path as a base64 string +function _encode_local_image(image_path::AbstractString) + @assert isfile(image_path) "`image_path` must be a valid path to an image file. File: $image_path not found." + base64_image = open(image_path, "r") do image_bytes + base64encode(image_bytes) + end + image_suffix = split(image_path, ".")[end] + image_url = "data:image/$image_suffix;base64,$(base64_image)" + return image_url +end +function _encode_local_image(image_path::Vector{<:AbstractString}) + return _encode_local_image.(image_path) +end +_encode_local_image(::Nothing) = String[] + +# Used for image_url in aiscan to provided consistent output type +_string_to_vector(s::AbstractString) = [s] +_string_to_vector(v::Vector{<:AbstractString}) = v diff --git a/templates/visual/OCRTask.json b/templates/visual/OCRTask.json new file mode 100644 index 000000000..e563b8b3f --- /dev/null +++ b/templates/visual/OCRTask.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "Transcribe screenshot, scanned pages, photos, etc. Placeholders: `task`", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class OCR engine. Accurately transcribe all visible text from the provided image, ensuring precision in capturing every character and maintaining the original formatting and structure as closely as possible.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}", + "variables": [ + "task" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/test/data/julia.png b/test/data/julia.png new file mode 100644 index 0000000000000000000000000000000000000000..7045534d99210a097927d4586965897e54e46c4d GIT binary patch literal 38178 zcmYg&cOcdO_y4uCA{0@Gi%UjEWK+hy_ROZTlD%bKBs*Mti;Ha8n<6r^Wn7AEuD#dq zb<6wv``rG(>pox4=Xsv-IOloJc^0asB2P+0O9TReNMTSJ4G;(q_!UR&20rlPMg%Dx z@B`OUT3H$d`W!`k{u1*oqbXEF83giV1%cr2K%i6L5qt#%a(x5>t-Sz&M3O)tYKOE2 zbur)z{8x(dGN3EWzpUo`SYRXwCL^uoF|~nmPt#gNUGA)VS-9z`#W8IRB;)CHB;lTg zDQGH{hj(JXp`v_9dpn1nwUsR>iA|hW-IU6UGf8SA=x0WB$g5|j;i0@l4Wh&xFYw&l zm%RP_;GfqMCexiQB(^5Pbx6$Jmm0RjmnA0Bn6LmkfDgn#4)^abO{Oy}X9u<8PaD+I zYqbnsN_oY%YL6L@u)>H{nhvy+)8as2LOe#y*sNC(7n`$m^3s=;tPBtI0+mP+Pho+| zfCyVkqT36x zBg2urthUS}&z(qDg6vUC!y~MIGks_>vm*<9+wDIy1K9-1D}10bpx{qwAgVOeaCa^X2LX;5Cb!N ze01U^L#&a3L*`za4qCSZioF7(f@4O)WhU3cP!k4QT=$t)6%n0^L==|^m&`{+T!jt$ zEwT0nUvJ<*1g}YFifDZiRYI)p642KIu?pn#4v#T(_aHYEuu;3BtG8r?zxw!(+K#CY zm8_9u*}q6&XIg!Z#p`cl``;+v=GGOsD4j^c12qI0~o|Rj7>VCMf;C zQ9c`{jSYqNK?r7<(%L}Xn=vR3HhG>mfRk^)I5t7k!hFBEt|`_thC8eEZ|b z{Ja5M=9{1`xqlm6xotW?J2G*0C^?N>faY|GQ6w{*wh_8trb0kXk^h)AxFDBVdsgo9 zYeGRVMZj~0j#YH6h0?w?0Rn=G84+TL9&GQ-sGUYjzi?L&L39eEeKnHn8l`s@iZ!dtvuI`V;6&=YD=W{tNf(Zh38Eh8|(0iO{wBn#4+_BgNX^q^I`6a{0h zNnj6gGk9`Q$`*qC>0a73hiWjh6wRGFvtPX`d#>(qq_V7hN}>*mzt*KeF3lw+m~Hv_ z*u%vdGAr;`a^e6{x4PFy%D^ln*Rn5l*4x{i;7PFIcnMbU;V2VjORJn9&j$4{T<@p1 z)42TX$mr4>0YxhBX`dL-St_IDsifZK{L^fg?w@-{NlU{=P^NAh7uV5Kul_WcBq;7P z)vdpq6|v6OYutiNxB_gA@yNX2avAQqGk(-E76YB8#T={t<%B(H;T3nN zqD-&J1ETCuGNo2)EOpSLKmUJYl+`Yn)i1Tfv(uP3*#g}-pEe9NbRwqU-DpEY1-TQ;=3=-2wHZRyF#&{f&>4U z=#4CVz1^LCUi9-f8RuzwT5K>Jxe!c)d42n1r(shUWA#}p3n^1%k9kGlmgvr*H4Gcd z{ol(F`y+<1-Ke*}E{W-HV1w!YN$wYpn~2{J{MZjHgx{-R(w^FMoG{z+~6dqz85!jus$1<{wJL zCHolhTMn~;VAC~}e`O$rU^N!}EXSI07#3A|lGG5?NQo28`H#IIn8x&4VIT`BE1`EW zMC1cG-0WKJP{d5cD}u~En^wZg!|&v{(f6+3*xa7};ioW+i;yBfPUpO)iN5W@c--#F5G4_WbCUe4Qyqi0Z013>dp*I)ch3O}MT8-9(D zqSk#-f(pe7k^INbPcJf#P~coBwvnklmuA+ujXRIBdj%ztP)jYhAAokQ@%FZt)N3lC6^W4qr@PW1$JtpNjPh~B*~9QxE(M0+ zJJc$)a^zCp|4revs)>ZZg4Ef{5&hpoyuW^gvp^D-MI$!MiUf5~U-L1m&_(KHU^ZdI z32w)!W!w$S+*!L+akMH#kUNP&(b(iB*Lx2YD`df?8d75d&4*vJFS=WZ36IKAj+7OL zu?6ziAhY<$2-t1Z1gLtX`f875Fw=qJzY|1bg$br2{r_iS#>CTT?644ES(zU#zbA&Cu~cfD ztX_ds@Z;>irVl4Ft5^g$>3H(+f5^`I&O{_dHA3{iTwwnd!9^7n{`77L@3nZL>il=J zUSoUtlf!AQLMZ-Pk`{gC{wocsA(@tUq? zuLs9589Qj_zpeM|gCAtwAUYghSEa=HKiL#8d8LsL8Z8}W2Icb@DpUNQ;Dh)8@7K`Z zpUH&BID0Ljr^y6S2<)(5(!d_B{WZsEGu!3KT^LU+ZbQ4$Q4BV@{WW6>lkaQ*FG z+gD4F0W1FZygRM3b})T4NA|BD2E#r6IUP9B^XNHQq7@G3RtzWqdwAFu>Mq7)br+WP za7G5}1{icLa=zGJS5fIKV&fvc@ke1Vo+EzL)DBpV$+DLHw%_Vg795%h2U?E&dmX01$k-c&-F7lW`` zE*3(@m(y-c7_9|0ZDOJ#FW^*gvwsQ;YU;v7;`A%>V1M^|^WUMhl28&UzJIs?lF0HM#Me>u zOaU{pBeJ({;Y1t$6EU1DNz~Qu0bjZTg~Q$)kJwJBJjR9nV@F%C=dX97=s1BWFvnDC zftyb-4czQHzVMz=Usy#<5oYt%?~iz?*7M6#NDT`u{@s}OyP%P+<9wJTTWHvRng3>i z2?b8H_w|BN)i1r)g6I7i<2!R0Mcig1-j>K}wU#9)3yk(CaQn}fgB8h%WtoyIJ zb1~k?61xs|C5DM!W)2be;`wllL-=jUVmdEE-qM_`mnvu3+xsTzjUlY)+KZmEwZ0 z0*TZK?ABic&f-t8dyDQq^%1&nHv9pp&g~8RNWO_W@9%xmNC|oNU+`CT_~K-On!Z4 zz86eij<)+x-UwRs_aPk&8**)XaBeS5*qUe~3A2xun?K*bq(nbSq=D~R{)5|<()58j z+q8NV_lPI*mLK;8`jT?%NhlcJ({>k*kq+Fh@v1{~OZx?Z;v1I8BxfVXtBTM0YhwZB2oy(TzcLk!Bc%d2RNEkfH(;LnQqo=wD{^_E3TcFqG!6mtImz4 zpx#cOx0ih#Ho7uz{MCbRf`?7w@?PX9&VmF1g#nhU%deDM=MR8T_&TN;Wd0iq zX}Y8$e?-f5S0!)!JX2}9Pypig|M<}rafOn=As)YEgJm*D8rRM!5zM-Oz3<0NlvUwK zCWTo?DoSrV-au!ZY}UbD=qee^7Bl-Q>dr8&@(F>|JjuTyJa_jRQF-4tC}0<`95%B3 z&H1&Iam^)$$9|+%Al*hI%sim1#0X*gs2ILhg}nQ)oE78PX`MU1i-(iIX$z%s&0a3$ z+6&8X--p_{d5SXyVGG|%3c?o4oPhzUDtd@(71oGo2H)O7PPV+Ua1f= zyF>wyl3w)pkhlHvn%gWRKu}ybMc`*tx}~tD$doReRhv4+EP3)V7VQxu0P;RR{2Ro< zDd^6on}rUw&VI0@e*L`0cAAg0=M>xdU=V^WC10+N$#8%=I_u^(c^~jy8 zlPZ;mVr^ol4M^VGNo^YLk`sHYa}tkCXV;Z(fQ+tvQb_YZeN#!C^hFFWo*Nr^D(%jZ z*r=ND+jllSmt&A}zs4X6)33$id_e&pT=o{gw)%4Ro21?L4rHZuR~^$^3Eg*XLErw% z3S=bn2Yz-vii|Q0PI8GxPT{tMKStOI9Z5FVw-{X=$GA6rYr|e4yrxtpmCsUv<7jtX zRDQ{;w<;gW_3GRyv3^x0cJ5AfuKnVyFUa#>%!cfdY*l$BYeDzIUkeyJznlDyuo!*_rH+MVPIs0iq_(ItUe5O zpOe>kK1}9RKch4(ElhLO=!4&|T)Wd&8)wrCZ3%*rBG$r=ZD-^5oBpV#PE*=l#yvb( zxnR3|%gYSP8vQGMcu(xjb_X`tnL-NKs*y}sFbgCzN#{JC)S}*sm8we>_Z5>HFJPG?Q@x%?aRO;tAkZhJ zzdV(C3pM9UwjEFZE!2&2kI|Cl1aRZ81A=}!G8%78MPeZVt(^TANkc=Xrlt-w3Px#4 zd>p#Kut>~z5+jWjJ(J`2>8-dOZFjv@pe*~{pA45EMBdU z`I{rDqHJ&9x6W>Ri<-*=<|W}jg_pOB1j`v|@7>l21jAy;%ColT@}mvR%H*VbG=t9R zY}My~8!~tB3HT9|XOttulw8i1pMzPNuIN472mJYqmhF(Plu#K>bf4&e8*N5I0|6R=P zeEK|hJVGO@WbsQLw|#W6^e#S(u3+zol?$rRy(*h{C)N8iyT<@36EgZI^U3CJjxmdC z*0VEQ#tc>luMyU~uTOLYo>$ywiA0R%Q=5G4W%U9h?&NVcOc1XU*fqFZ3DMjI;$)?R zg9D!&2yMPQ_d1H3b=`%ijefsvz`*uZ20hyZ4NNrtMafxMGJVXt#9IfGqSG14%ULhI z^=wUxfwa3Mmq*{#Ib;^ZZglqe5WtvOay@sIB@S|*y zjLnAYu`Qj}ff0XqZnM(VNbYG;b4LJ@Kw094vY%a_#OK#beaW|&0e*_C8lLotURSSoPl`%&921SQ!S=Hmikicw564YFSMFl;@5<{VKR)m zD!(kfs}V*^Q<+$&KWwrn@gs1wNz;GAW>%MpF-78hve@c`T21R=HS+QRPedq(uc^ zan;Q0gH_u_PmT0{7@1O)>isP@E)3I2%IB_MMO$6#jc!(TlY_FsQur)Ku~8)V?d65i0!;ArO zo5OrmcI=|B%w3x<35B9Qdo{D={^Ry4FB(@ZY$B=sI-mS2EB=SLY>FZ(MqV2B8WZwA zdTpLZF7hOK{T`i20#bi*i4UH=FE-KEu8u`^BMxIn)hz5|u*-4oL{8e7Jy^_#x)mdR z<10tUUJ12kiELKZq+cEBBolVKAHLLy5W?dJOR6=Wkw#es;^m1RVW?kB+)lN29>aDZ3P4QoDbRk%y zJ4KZ|^_S)(-yT*;CilT`%>`=x=aY8hh1Y9qa$C?9_OP{QObp*LAJ<;FkFXF7un>bl z^DHFozKG5@wv$3I4O6@>X{=-JsLhr!PwVHotuKe~BHlp%FgLI4Zf(^48hRMQlrbQD zUg%lx+k~d~bHQftnK^vHc(v9&;cfLmFYDfR`-fLHG`4f5Wmgq2$#{V{F_8_sa1Az; z%j)G76Ht6n$S*Cm(;~;U&-^VU?Hzpr#Xz zs$WBboexFwqwq{KPYScjkUEruJj`-6s6Zu+-RnjNC^{h zn^%|J<0=uT$=a(DgN^D^MakKhUzjaZI`nI)(|o?)Hqd>cqc{~N5VmISeBY~wArB=( z1*IU8a_RByGHi@WHR85n9$xvJ*(17pa(-)%JN1<+!{sis^b+W={@5U)ha_J-oEHI@KPt;VptWd9SbGa00q<-K;EDJzwIf<#jz%27baoY z=x|;$CvevqWvsvM`A5^OX-hq$RNrh_Il0pJ0m?HJS%4gRcn9y!J(t~$_C#{L`48CS znJdI5Exu9@N{U$KQUyGw|~PT>Ex`>Ibdge zf0`S|A6s><;6-h#1u+Sqh3nW`peGb1!UR0W6cMWFj{PxyEJPi5@fan>TiE)#@lW=- zxNV=&vN)@lUM%E0nn;19atlgIjCkS|B8R5<&soFsbq;GPctBfpm9B#63Uf@t)+oe| zvmB4dz5tr|XFtyS_OdBh%pBP2hE<$Qs!~^flK3W3wv?edXr6h~Yv0v?z~9cXAVWX_ zErO>#V%OrZ%aIM><7dX$Zkw~xUQhQsM`awj72Ke$%yQIo|LGD$15DB!V*i)&o%yIUD`?BAXCgj@(nzNp1UHQG_v+uc#^3m+j{7vC(h zK1_E%m@SLN2W1(O+!v{;DJ*U*D64fg8%J=+ z%Evd~RgyK&gZV`hP0x-UDv6)_u1&_)1rYOYP{EHPOb00P0ZOibp4!m=Ipz247(Vbh;kBTFhcK@9g=Q5lF*(-OKL)up+%KQ@7hd<`;Z02zDfd%%szj{sJcCzO08J!8#=ewb6j^`_N1lhXb!lm-MxLi0y( zSbU5%7>1?T#RPn(VTh$i1g7@AkS%wVkOfIC&hZU^_%R)pD=b*Y?3X}3<%zyzJ~mmZ z5F8>YDB@6bY08niH3LtbudYgYaMo=-uB=Zl6JQWXVm46Jo?j4Q#8dEIfa2x4OX0yd z-a@BiROgZK+2BJJ#Q6j`hK7Jgx!P#MX~0dsTOIV&i^2Uu&`wab;sDC5-dSv={3H2E z>_`ugqLAom7ZtKAfFxN`Fr=~es*KfmqJe1CtfX#1Lw%&IyyiH4CAu80lK~}%un@Ho)yxeE9Uf`@7KF6}iH#4aqo3<7(j>rCTOM!b zKON+Jw~rS{6F~GXc8NV{*)fPu`!Z?L)tZx&kT@>nn0nw|>NgsTy{(cs-n3rbw<~iL9U*Vi14y(J@oOdl`ckE%hcmR*&`= zcM7SF83u&^>?((1=SpGeegxS6-oE>B5V&W>a`Ix0=boRqC?yaH@6czIqb69CSweU9 z*74Cg5@I!i9dNcZ8URWih%HNlSW{-9`aDKJ@3;$N^36_QKp()d|fntx$bDM zxGw^5s`5u@Es4NhFHm2|;Y^g9K2#+;CKLwFR|@+p9k%%2H;G_k!Zl*H&BmA2nc83J z#Fq!>?5!!*Pb}-r4u; ziYCI&OG=)1_la2bgMOa%IGa+pivHLn~$ASJkrLjK^ZxSj9!Ww(_Df8Rq~3U|@A zCr)^*Vuc#4{#mZrw{p$815mu4AI7t^s(ne8Ln01cCGt)==&x+P^PZueankVV-m zo$J@WIH~-z)&+lFDkWk)zEfEU#TI|krB;fG-1UX71YufDzISr{pnx4^rHS#qoCY#G zH-U#OF=tVTbNm6}{T@QZO$*4wRK-jypUOEGgp}$>5T|Jjbqg0z?P<$pK9szC&jrH< z{7QJ!Mk+!OO!$5O+Zmfpt#*KkB*XZ^CQj{Ex{&$6DXxD8U@m!wqNkmMN?ozIvzK~N zVyE1^fL=On|LPcWb@B-UZy)Llh3q$jTeh8V*1vIwH9mT$YY7D$H4|@^ykMM+T87wn zKY0pOzi*AKK{sd5>eg!?0GYsOmvQBH91QJzyhtsTRa{^!1z~G~kr5_(! z-aR}~bD@HpufQow={P3U!Tw~$JWP^-K%rF%0f1~DqmU#}+PthTl^?1_><9c1-w69r zU&Z_^_-OKzh3kQ12mDnIi1S&Ho20A;DeMr?RAFC<#gED-Gu^CIfww=lY#tosk>i8`rZ^*W(hjzjAwQ<++K||ElXwgq zYeOsN4@k-4wd5&|Y2+mzJ8ErYX0I71ec5>;;r0P?*IW=0NUCyT%j&VqBVnalA+3C1 z5KN8>98+ld#~Y4iF6L~?J{;(~kN?rTFcXRR__fP^yJ|!@*s%Vwbe1llXy1LiF0CHI zM1kT#XzB+RAXbeA6T8o%`4OE6T!|RHVuq~2@z$-E2>eHC$1>~-{_+r7qBg#H1@*P8 zoK7VPv}#TWU}NB=wp_-Hu+W>lR$alJcCE5DQPXJ}1ozW9hi)xzxT-J&NU0y@$f(g{ zQUFljN3g5~(O9ZPEF2KrjMD(wrlbkOzArf(c{6gMSH!F5b~{VtOZXtKAQT6GL**u< z$0FuNPkVM=39CX+G_FX~!x&8vaORfrc>T485^N5D5B8ww;VhdYwD zz;m-*sPbxLG6+=ZGKWEz-cW`e&9 zvKc4?3!K84%P`G9AjZ;}lQs$?)8Q3NoGu{86IvfpYIj3;Nxl{s-d@v|mXH0hNaIA} z@a-!APIko6x2=s_PCROXs<+`iy9;Qsqm1Ch4h8Y;9|YuqYu-ATvTwvTBYh~ z*qS^rMy}0Pj2*CnCu6F#9q0R zSd->-o<@oRj1Dw>0(-n+rId>yV^sIF+Aixe@P_M*#e4^;i!=W_GoCQR=~(Or_G*#B zsRWFrZrt=(j6}lG^m})v6eOeiA{PW*Ci4B<_E|1Mv5aPED5Sq8*RbNc_|cv`M*@3! z-z*fI`_hjfrJ64?a3=1`$rymm8fR`>zJf3;Ej@0NWR+6qA zLXeSw%3c}B`eNi(B{CUJq9zcN_@<;67&wC)XT#0>8v>$y(-T*(jfeoNHHn875(41T z@dc9)RaA(<4g1Ck>DCu|RQAA!>Zq>&j2ye6N}MU+9qm)S!0fWHp8%zB-Ic3Ox?)## zExTNHWhS8|zAz4QBT$2wd$i1n&`8{4h~N@npCLh7SJ(t~5RSd!VpkmAMD>mCdk!=2 zx$U9=XW<9~A}LUh)}Ky&YV$AstB!*#N@?#nkeaqg_&j&mr~(EQ?hvq#sf)5_CG|B4 zSQ}A0-~E=ZQTiAO=iW33WlWx$C<3wqG1RhDa9}K}HEIzc$*>eyj2SR%CRK}TuBhz| zpo(laeMulw8Sm}`+7lAANzL^pkqWa}1+5ehs`5^`7`95@{6QALCTD|Sp0W4Ts=kG; zYGW*!cURvWqvN)|5 zxN4mK0HSga+-HLB+eI*VQ#q@5Fah1M9jj77)!boqKMxSau7En;h3R*LvR)Ih0}wKH z!GUNb8wp3kvp41Y5~2Y#2q6WW5+p^-YNDafoJw(eSM|!8oj2HYL+Ry8<*S2OlL3pQ>g}Q9mi(oSK|;rucEDS* zyHoG*REs%(n^|z@K%HSOpcOu%HZYkn#{q+eRE(J7U%O{A^$EykUv-S_eq#45Mlj+P z_s3pfd7Fc^7FC7$eG`||)C@+z3z8Qt%skEX3(3#VhJ+i58xPxW4d~N-y9MY7T8RNX zeqRT@lkn$93~ocOZXBR>Np~9^e!p>y#|~>Nsj@~rRlX#I4M4M|yKF6@0DKF>esq!w z5(K@V2RK~uhfe}2b42DTu08zLWol8rnd2=b09kevd)k|-w{RMoxWEA&_BeN9uo~FI z3JV!n0K+10H}7dT190(??B*BdT;E%)rrbG=8Ab-II7ZK~lJ=XE1J0mWsQoB|A9L<3 zn24(m#$K05HK?q*b^I%^SJ7|AqR-sL`KA>XdzL7$}!iFdDCg*F~BbTx_S8f8)#S8QvFl>Ltm70F^DaJF_Ftw(W4}N3L?W$=! z8ZwM9CUd@PlrCBeFen_moMiuPvqLNLHrSs#OAKJu=Nho+u5@xjEUrj@wr+`Q4i%eS zmbI9L^WAP4EM5Gifpq=ojUQlt6?;&?FN?npQP34V6VUkN3@?2N@7tS_Ml{8MCCy!z z;WZa86T9hmXof3F>B)`RrD!Y>Vatrxov#-RU?4h+1{J#w>G7Z)=5mg@?C{X1%+CS` zJ{zc{h#n|A{q^#p-g7CEik^|+!JV<-{+K>Oj8imvQ;p&lu5FU;O1^xpu`%8t-Lu5^ zWM#&Yjsczpon~NykO8pJs69*y_Ww|Z6t)Hc3)~LmYpNC9m|^}uS&6l;dNJA8 zBBDgaH|EIgP)=7~09dB!nIM(v=|F=?=Id%5?H{a|wVJ9+O`Z}R<60A)q!cZjW2?5z zEB3ND!bByyDfulJ2Ad7ZZqG;Jo12t8yfkl5{4c|I3os?D!G4=<=uDDE(&?5LQ4fUDk66#Mwq%fr003 z2HVaU&4CG8T+UG|o>IboUGxQHvaBAA*Gw3)tt_;UjT11;DLCxU>xCK4seejsoF`T|> zbF3=Y_Zi7b@+P<^A)kn!r8*W@so^p~@#e2vyTDP<2FY~hj}%0}8&P!^0kUM^oodx| z(Q3xk#t0^G_oC!g&qlir0@o88@5S9l^YJfN39GeW^_{6~Ef@U12fB(2pYTiO5n=Eb zeVv3W3&dO-`-D>C^I?;_Jw(f%zJU@!CX~3IM0hWf6|1L{5yq^cVY4ykrwMul&dx7^ z*oR;Hu zd&UqZntP@g&|iFg-h9;Z8$I_RG_1O6(D)$7$%8q+U*dcVVbT-*oa3sufnVC6{u!1o z7HzP&V~im&G#fiPoo{Qc*@O*W+pK7f&S-pTxLRKLc6HxPJXw)s`hXgj0mg zH!Cg~kx-h!5g@_UhRLe$jebTvNv}gB*)^YJ;~Y}~5EyrYkVij%5>svWE%K~H6lnLa zO`9D7Bxd5gDj-)o&$^eKf3XA2z>TDr$I^X(DWLi#rzuK5>?+FWQDp<{MUBwSaQi2} zlV~iMqtiOb@!87G3}q{Lj+^LXo_`W06u(Kw`6r5!thNBxnn_k(rPaYxR6R>b4fH(v zgx|eiakbriKEq*g^GL|VyX|oYz)Fr#SWLjEKx}n~?~jMOv9N3#cjg~UIum`20351?x*bnHVek}G^`?U{8T9b?hIpaHy=e7UmPlc z_+ivKyE7|$yMA2U3azflRbGG?e?-~yf+Ll|XO|&g{3#U#56I5B_2Ox!GEZb6jF}_N z$jmb*_U6EVxZ&YnDFkeZL zLv91pxUaJiNU&G7gBM}UFW^QR7ecr>*Wv;$cPaU-^ z^oY+g0X`N^R=Xe`HDkA%^KDWSw)dk3(t0%RE;gX=o_@D7|3cSdIa+3ZW;Fk5jWPr) z7w9X74fj*ucc|}EWex2(j=HVU*xzWg)++bhvSj>C53v^ z1bZdfRbf3hIP~2k6slGAG8!WjHj+46JjToQrSO}l>54Oo+?QT4)EZW)E+e^Tdjh~Y z34W8d`PYyl%Nahb!?$GGOkuMDJ0VdA(qiM-`StgqQ}rJl&}4abn2 zmGi17@dFf3h+c@j+1uMK8aKKs4Zo1_wI%r=eYg8S#YxR>50muD- z;0)%7C?F6ReVW(21a&uhO8fb-pX*#rbF1TGjN#W0KeIp`1T)@4(yjcHi5>ljg(a)- z;RXBVea1U7fIN(<7Mi1s87E?jT}_uWPd+{JJv+N|5e$)-9Ie+R=D&34a70|a7nuKs zvR|EJYn`~8NWX_~#i#-S?R>OrIEEJ)`X(XAkhh5atoP16xX*wfbfpCl@t-$|c?$Zn zw2r@yJPkb0e+ET8j+O07?z1#>{$qvkF`1d;qvxM=h4teXk^QMK>9_?{pN6@p8TRuT zZ@9&G^t&rIT4oxINVYC-CMp?81O5{9?eju_e5nwzTfFyOtGMX(DesRKwf7mk9fV)H z^ha2KcrwyQ79+E&^)-1Tk0I?e*s5!{jjZD~rmPi7?fctb>xC(cxGXJ;?#<+ZCIAg@ zCIi7?S||XCYGyfGZ;z!lJ&TcS(3~S6<$LjZzd1??h>jmvqwGY?4`ZDBs}qrlu$;u? z=k+Jls_EjktS=b{04{{u)osJCMzxmR5gSp+A#2f?31yLEKrPN&%+dzPfJ);tK(AKn zYlrDJ829F4<;vc+_vJ?zZ+~}>P}B)uij!1IEo-acU&R_;VJWp_j_OPkIr}b2j*qGL z-RC=U*PLrQndiA@v0J7%AGefY=Gm96wV+JR;Bg;tN1**CqvanamF?nDzLQG%if@=> z=nvi%m0zrQ(KjbR!t6W8e(YSKBE5#QdE=LaO zadlu*X#!zAEUi%K11lkuC&_dgIG`Q{*4UGXm z3u_)9ll}5W)b>m^TzCC20H~F%f%sdY!!L#*g^w*x$Ga<|o^;je6BB;D3ty>9#%NG|_rOK|4K2#;iEu3oWgc%JO=tSG04naHBa zmhb_Yydl7(m{q%BNQ(Sz-39i_{B-4$VB~KSDVJVMowqX82r_`b!I3zrZL5B!^R|Zv zNEz)skhpf4O=l|K|%~t?_w~*A1YT z1PvTZ3g(HWU=8+tNiEt1HCv760%uCkO;a+)+hEuu>M=S+d#sef*zcRaAvVy91EGS4%3vKoKr>9& zd66Hhf^Fq#w}#%>QR~MK%=lhr>x3}R0R9}8Uy@KD z1)Vu?$4k5e=$NpZ5h`C%N~|f^el%r%Nbq*;Ovt2ZZm*9NB*l^qQUVjTMePh8+p^V9 z|1dAyCu^QII(?Qk+BZZ@@A)A)hLbh8 zYg@ZpStsj=yteF9Two%*Cid=gImND&XiW22UXgxlF*LtdpyKPSuS%ZtY1L6Ze4cC( z=*MWbxLUBIp<_QSG(0ORw$IwYwDc6BEKm+7SY>=bJPL?~3hxD**dw=T-a9{aSbRPf zPc)Q%v-6d9owBqQz1k{v?d<~su|Vk`5yt^DDlt`x$?o`MjT^NO_r@l+%}oCG4mIGY z)Y-lEGB)8`rGcd;c2B>FcgcZK+v{8%qxdZcB516DenV-jBq$Z!X#4v++u%PrzZCg^ zMy34TKepA$E~XDE!rhzfhm2T(<_eId-_Bb9AK&8cR&YU$((HHo-phC;zrbFcX2=f^+@~%FfeaW;|y`LsRKA z8`+Pj9X~kBePK71JvoiqgIJ@8st^4iSa&AjY5wv3g#1Y8MO=isup_py%M^NkVRhO+ zqQhJGNrnm@LRjnbq4{N_R7<|0S5H+e(uKrig%9TEcGBd=GxPS0MN{XI8&G-$QXeuv z=sd2VQxWVw>r3~xDzWi!bjr)L%b{ewSLw@UOtxfE$=Zj%sGk$;p6YsDT=>|&RMHsj zmEz3QQEmNJ{*`LuVj1T+uCvsUV^cic< z0;G-vE^UWm0dC6%r+ckDera~HVb{we?awKJ`|eP*zIR|44sl0j<)kXc^;bpJ#6~(Lw-2c|BNzdO+s}BbHIc# zfZ*tVqA=#O?1>xg10J|&^W${UQDn>BtTn!n*YEnWi$U0yBSp2 z)x~Z2eT)-Qn5?;^MW#?fpUe1Dqm)aDmfecdL^M%oV=2*3vEmzgXwSlPTuJo~(U%1u zo#q3o!~^V1K=DI`PL5O0*mw2y&gAY){`&6SF7@R-;!oIVcX3pGGj{>w(Kuv#A`sdu zzP-4lU1p7LcVqTnGsxvN&u2*Pj)kTSVd&n#ZT6PNyE0HSOX*)A&dk za|1qHR8&NVP{?I{gUU}utigcT4YY_hV(?Z)R5c<=VR(+ufi{+jzOPR|M)OuiG+Oo* zkE4R?&ZPM1z*bpEp!F`$TY`sQlS-D)Yxxy7e{AE`QS02+rsR4^)TzG3;hbbqY3XyC z_c6$cKAYUe)+KZqo^<(F6ma?I4_dL`rXacbo~$}IK-%c&XjMv#ssxc#mW-sAcLNZU z!4>$;zDFMv6&AW71oVb|5wuV|M?RO8!Kt;iwfB%P!I;rsXi|%DMyopHqp<9&gi(8H zI|iNkZ)%KIS&d}gXOCMOUwfa4*PZz(_C6xfQSg)=rhtX_+k_&iiGsdTxSLeiOjaZ& z2@^sdgn(A=Z=Ja~lfmfzNx_f<5JIILsTgdS8&Md7mn(pwWn^`K2_eC?%8w8fs_bWp zj2i`7e6B80@8UegZGSbG7$|O0Wk{DF-b(=1gPkTKEx{MgD8H*BHORU3WQDo?EfdUL z9`QQO0zkLM`VL)>#KE>o901!t!Cf7UTwM>)=$5ka1YaB&Uhd$}pSslmiK+m(CH*$m zC0ykaTvjxzRN9iT>C&?UlL9S;mozPqRTK0jphrQVB<+56 zXQuXv0Y0buZ;$JjGrS_}Jhwsc8}QPik`fg7MgVG7`*dg3;+Bm1{skm#fHBm}<6umu zm2>q79CHt7#XIwsXpIIV+WS-z_pK2ia%xvs*PH-Gk28{1Ld0n_k>^PH%l5C3u+I83 z4_KtFt!*UYNhsspLa?o^g+Z$kQK?>~+@#H+FziT9z3O}49nyPGwxv}BAoPGrQA%u# zC~E*mJgItsn)@j#`R*|SR;b+O2L)NR0s;b~RW&sq1S=g8ov^6_HHeK2-{LovJe2mO z#R%AHu_xZ%qHS9hTK9HsF~>!KqOU2 zs_tJt*s$q_vl@hot2|ox#raT2ANueFjgx-L|wQj+#<5&l%v%#4_chE}M*8YtVa`D~=W z8_`+kCB+HM1F-~dfUM_g6!7YMS*La*BosMTY}W_&Q<+wwu+-=q3N~3t)0vB;M4mPJ z=s6+9f$e0Z-nmu`EWrJo$*Bw2p=do2$jzDbV74r+;+7U)YIn&g6d7VP3L5Ru&T`@1 zpYnLF+l-Gob#k)a5FO;?6$`{+)tIJ1Z50f$zZk;0tA-ZT!!;PlNDe-yz^En~wi#e?`u2PIp;d(dCqla z!7IHh3=_T0khoqQc>ug4!u=}pHxa+Xw)*_XYG0Tvz+sf^+7R?<@bRaKR@3!30ar<$ zvUrcvk)XNfK!yH0V(Yqht#BC23UQfBm2$h)zP@UOX^B=m8%0^~LSCrub!*8h1cI^# zVU6Z3Gs-1U6Kz1axxD;*dE)*A`o%KD$c91V7f*HuTXh>mnEGaK%jqoYtfj*$jN7FeTwP)u{TN1m=T@ zt-h`Q@;YwbY$5hew21j3iOA>#%^BaQ%-+@zXYl@`cUM*uPsbo!*xwXB=G{cT+Xj2B zCw3hyg{>?PhkgE7rVd=#pNiw);7|oEtc;7uAJ-`Eus};~i#E1>-wCW1gx=LOggXgB z)PFH_6t&i1+7F0h-DfQqL{6y;q zaV$*k+IS6VH>HI3z&eniBAaOhGZSx6#L zbf;tiA36b7;{tSwbbpp+yE@rF+rC9X5EhO|ySRz!?Er2J_u?^L3Z5G`{=2UIF^w8L ze^-G$RQ}UNMzsdvhmRCS_OiGAr7eb|hIp2Q0w=T%RD6$Z6S6@M>j@!t0yq31ZbRSX zYD6D2N9P@R<7>Wt{TjZ|eUkACHvzj{<-rS=FKXJgWnyr9WRgnA7d8nv4d;(#tXzVQ zX6Yo<*nugci<>s*(-XanmiNzH@Tfl%5Mj7Fl88Y3d#llf6e8>9b*a7_#Ko*0LLK#T z0HdygUblD6(7Xcsx3pkZKg9d+b@Gx}eor!$8USc3HdVb#aUS(#vaYnB1O<8a90_e{ z0tMq~6}g2mf~PclTYy=f1Z37D&+L~Zla5n6m6`gz0Fi~(4^-6D?C;gq{@X{nOG@*V z&mGFAE#APn&4nMuRb~yi9SzU@-E-^N=wP;r5plD#u~YA8KxI|cSZPHCHPJIFaCH!` zKi}W40dDwgHoq52#c(S-AyF=-9F8gtIzPH%V6MCNiJI zMh_E0O>JiGJ`Jkg5b*XzQ>&A^6`P(qvu-?_M7VJ3*F>NU- zH>%GYe;90hDuNby<~+MVPPu;LM&t31Pt(Y^cgK+#)d`}xyS$Vx6BlI!XE$$`qsl33 zksBKR_7XQJhDX@QWjhP|#gGDEGZkq%hba5eh#k$<3IxajYDo{aP;BP4TB-I@eEyLF ze*hu$;!pWaZ*Yfrj9+8{nm70P^JnL+EkD1l0nM}hb5Mgk7@Dl8ZuX40q2sn>xSVOC z*vz$7c$r{g#2$if)jglZV|cz&Z^Ex`|1MBA|G?BJGW!sI5U6Vol7GqI>YWPGMKR+` z{vQ^k^LuD-M&%jTTJ_eS_t1Dd8jw*d=$A22Lr`mIXgKo?rr`7JJxAqk=9G=G?w3gA zJ3bhhlj#RMctgIjdocJQ%n?o+U$idiXVrjQcI~N?Cx6L)`tuo1NI~}4VjD^W`>B#R zqTH9~+Zn=Gyd9je08m#&V#oCO#A=>(C&R9qehO+h@admPeqNsaL{P&HBACvy@j7># zQ#-wqzkSgyLzX?j;_Lxvyv_JU0g*M$q*Erb#PRdFk~pLS&v&udLORJ8zw z20*#THIvIXvQ$hd1d@%CRk>}&A+WJ7X7U@zWzz>6m6@D#xD!=G*xRmY(o+~$rY9Zm z2Yq@rX8tBq((JPkLGAtH?QY$h|$2O&iB;URm8Tl8A_T!96X&UFm7D|+tq8aK%kl5+9>BL+>x>09s16{u>OEx@N zQb#hoYgCvTg^H#%XY>$4zf8KVl4~7CE&?IDVx#($>5D6GDU_e;o_~CS8`cqN!invQ zsP6Rtz5Nspr0vEpif|Yoc@Uy{J_WVxO0?oJwh2B@@-|A7p*5SsZo3NnB{M{oXKWAg zzGytr@fe{44J!HLWu7e;;Yj&`OWUi@_0LFPa*jb6$r67x`We(T-3Toh!pPw{Sr0s# zPIlAhPOLe_8(;J%RfNG(NhENHs>r?Ea|^pmy`92Y#WkhaV*AB@)PDT<_lZQZ?2hFP zvgWtpfg5#YI2%pE>_r#3x+I)t74U6CZbUYY+b3Uvy(4PGZI$cT;Pz5oYA_#~F<}1p z>>{)0WFs)nMchIx-iSznU+BH-@!AR2ZF$1OHZ>xdW-vPHm*gCK(?%oM-(&$WTY z=3pwNJmIRO3#B(vB(QnLW_WegjaT8u+IF-^B9TarXG@ZRYrAQ+=W*QL&&{K8+o{*w^0UJ-YZEcD~5u8|eR1kZKiY=Y!fRaLc7 z_Q#p~%H^{L_HlQ55bR7rA5Sc_STRnM?OQj*ZNVK#GwK|zyf;^AxLMY*n)JAvCYorp zqyEUt8`^J|@)^4wtbh3M;Y_HmCc14gg)J{L+;_zVm2bicE60jn!L z6O%h_#a`+|SUNj9uo0;2_X_v zxaWo67IjzWL?nMR-f(xv#R>dXAGo2XBTAKpp`KuuJ(p&e7=@vV0qBvDYA479k@{-3 z$vO?l`H7%7vqrdxR9?5ls7Qqr?7fqvlE>?Lj`>dLh#^a&^84wfRsPxn_QNaUD~ZI@ zY_Y|0JYj{okX5$Y@Q?>Vi)e7Ro^ED4nRzCv1yUsN83JOZI0naSepNQ&5{fg3vAwZf zMOC#jYeoS^Z&6@3HCtUq5;;XU?hhP8CCHTa9JhXT&QKNvK$eM6pj|A8Zus$VM|6|s zT#;y;S77LB$5LVyF}xqHm6eqh@PwK2Kc@WHFrlU5skRGD)*Gvx2MrX8o%0JT~sS zQAQnzV&x8s5!Y_{E&B3HNWFTx!3TCmahKwA-BYr*37MrvedJi-gC-jihmUitwmJu5 zxRV6!HtT;gEANM~#%XI-RkN@%_*kB*f;`bp{+&cI`HK?r5*!Qy3c#KD-*&Qt@x9LGW)ZdF$o<`uinN5=X7lo+}s zoJiRa`!3Q}k|VSnv3PZwQRAX-ZInnXG0DElwJCwuEZ#hNOZ@6=g65WX#Sz289w%+B z+G5xV06U}bB2wT*s51{chcjy%{U#Del;+Z}R%2A*K!R<-#1W%95hZiU0MRLwGbpmL zd&wB zD*P|ceG%4hw%vlx2!>8jjuzJpON%bQUVId63wtu16D=7WN$>i!h_YirffYr|0U9_P z55fY{K+? z0@LK))sv_=V60ggr&L6%`sMK?IqtdL#Y;K+eB=)J4&X-0YUl`nI_d3^#h9~ zK!<4i66oe1e!hIZJ7?%culC%?Q6r{cMlB;m8=Ks5l#bQe*>n=^AM{>V%(f~h`d}Sw zQr0rC)$%(l8alTH>WGLLG4ZVGGVNQ?){3Yj;U@)wP-Q14CpjEzas6uRwP3u)kOaiE z4furmq^UWu*&9ocq~uj{J41IH^|>yo*X%~eTb#Q62f+-GvO%;NH7KK9F}S%A#=tA* z%SbuA58Ny5R}M#YTsM31>ONaD13T62cW_8isxf;oafnPR`u%kznI!WE?! zW1n@!1R#JQZnMbLl*E`i*OtE!ig1iG@K6q25<<$Nmn?DS|vI67?3_tLO~z83Ax8#1IVTJsoh%$7Se|Fm&z-D*8L1UJywbzaDI?(4A% zf3^I;xyxc4*(w&cO}W1V=a`5V6jq|`*Tu!*mO2y45#{jF(J_S@La3WGb!})Ap|4nf z$#HH0!ChnsrM2q@*hWiYUhQn*SqSt{465~%_AuNc033N4iWISKW@t_X5m3UgelfUt z=k;DNdYN>{A8V|p423##Iy4|-K+><4bcBd*W5^-m+n%s_CjiidTG9_E=h@4dDo;n( zOTeXoWg`-iw&smr==Ki)Vc;VZdh#}tY_nEJD>gXq@!E-*7q^F~Er{<*oo{bFs@h&-u;oc+GtN=zl z?Vb&pld|EN7O6SpnRLMRfu?Ne}>(GH0@?znXYl1y5$#OhpU9_$Cp ztM>j(zH~)0zR!7;o?)#|({|BsIo&%FOX_qAw`#dcie0V1j7Spa=jWS&XaUMvzTgSd z&AwPkONuFvOSLY~3>nSX(00N>SxYIYWTD4@ji~;W)d>c_C4Zxa=8!~a`=k@ZR@az( zX0AV+Gi|>)4!>%TM5m;I4-_=lcJKb5Ycqx=^;&Z)ClhkI1qzzn%@vTQ`i3s?Bnnq_ zcQgDj1P4ATr!>!ob$}lepO`7x>hEC~cF5M7j4+Ej3b5XMZ0>=y4bRqRBWti3Lm@~A zg=nqQKe6B0c{qkRzh(V1xT?RrnCom%(`HCADLFOx7AXxEYsRQ33Yd(cp`m4XINd8$ z7Z%zC_ifi^GEQRzZup7!`w~p=-@7Lu=ckZZOk_|zy4u>oK^=lM^5&W*l+QS3Jn=Le z+ww^G`_P7$<>3#Sj-{;|H`7~2Axs3f4-0j?diAP8pX$wwJFJ$_jk zph}5Kr+Z&cW8j=5#2J@Y)rS?FPzUJJ?N~s7?%VDoc$UdM47MyDgecANw0-3*RN93^ zSLVsB|B9F8Ia+Xg{QLXFW?8Cxs5&=O*s9)}RQy^v86HoHK-WleFb1&p-xmMg3)AIV zTyE33deNauWdb4kP1LF2=;;>A@j#*<^DfI0s(OX%?$Wsn2gEfc7Vn$9SE235T?9 zK+cdVG4dri{@vsyVK(NIq1G@m(Yzt5`=k1r0M9dQc|O&UATaC#7- zio193!r>*5**mf1K;hz+k|)fPNNy_X>b+SR-zIf;HLO<7MT7uU>;&t|CwB->k<^M( zuW?gXS9dGD$2mXAzM=KMKEK_ibvCyURSH#}J&zy~0J+jiED2hbV$>EioY(>`sidvl z(e)F-hKptF=#e9hm27So8`5ML3yC(Xqv=GsrOmW}LzPzjGb4AQc-iQ9fOmsJ$GD5w zO-=Y%KLnI|0x-zusx?rh&+%`0?gD1n96{qmqmDgCEO;3dl(tsOuMC|VYosrdZ0ucC zYQEhLuQ!d+5MH&IJ{%NJ-}8Jl3gQzVBzFdw%^%=m(S?aQjb7XGROUn|x7F%qNVh|% zQ%#ma^CfVCR@wDe+t?L=AulIK&OwbR`;V7|}T&9jELR-`_ z%_7KvIyru5?RdxSzPxSE075%t2uOM8TmgE`Uq+>-3ed1PvH8U-OgRb+j?bu2U{6V# zIf_*Yl#$7rsRX;j)K)6s5#&|;_fD?Rj#e|a zz}f_2cP4;l>DJTPDEJ3YyKf&qdX)YdgGyCQsxB%c|2YOf-930C?I@h1O-UNu-)O>Y z`vQ&}3EP`|^78xtj#uvEqhc-D=9*B9As{z5q6Vm+#M@%>e+(gv?a?b*WF^TG-SuLm z`*zS=e|~{(& zNDEnL6zra0W;d<1?vMU^0}hI>u*shRdhPs4jLhov3fbrTSK@_)*d~Tyn2LbtX+hk! zhIEybcCZ)9-&DfxJ#>*X<(pJhxxYN2b{F1wvZ+mQw&W6bSNCoIfPeshAk-c9x0@nt z{~Jval#LBh?jx_xT~}hT3*e-d-ZO>u>)appy&> zCW+Jrkfp$tK4#llmw`ru5`+s_Jc+-uNaL`lKA;fNSwAE& z>}BgO=YD~oQ%JVjmDJwZKwpF&IUQcpKeM-bz0bfx^H8QqioLBUe`q}}O@kA7^2OGJ ziy~)M-DGZvrAU~~YofS|xrcW9Wp8&otLSibbDxA5JLCMYW(G|?L7N~NeYny;GIBPL zbUjvCp-g|3+55TxTbH1w5N%0myp^;_3&5vd8pWJ{FCq+sLw;_V0{aE?MtfThKb*-; zQU%OVeRG@_A%?ugDCj#E*YZnY<`AWKtCU?jg6>kZNAs?9eUe;y6&NbVv~N|vgk1oL zw1PC9D605YsiqUQW?#waZ!Y4oUYEBsrrL+80xB(162(3FN}N4@{CIP=WNI*@I;$Qh zmhCf5(}l$M(~x@td!G>{l&q5ox7o2zz)@dcUM`l0{OC(k8e0G)>X-rpEg1bst7p;h z7-!^B>2bn|N#1X}2E`fZu*Ft@XUoa`xC62}|C=bDH@*X}8I9w9C}s)L+;~}1XETVz zXElb=qFsFZ_HBKNBb0Yh2Hq)c?-KzSWv4ttktWHr#lEvecXAnvx&R%XNe?jqR6a${?NeI-S!Xm@f3i={GT=wUElV7BXL$D@sD!N7T`9aU5U3Jo3eDS zhA=jRMQ~e!%1=ztRVLp7Jv4`lG0XkQ5DOlCh%1+oJaC)CQ*wtK60bT>rt>N+&siGV zUwg_RUUb$K^!hqA0N})vj=$%Aw|b}*=Z`E%5MS*|JgrvulQh#* zqp5|ffIe?hdYBpf+}b1yPL!-lGdH8 z!!avsUFPxBHHkV7DkvJ4^dAouKZx_tu%LJm6Y3hz#FZQ#en-^)1N zn(wZR!Oq5lA90y<85pKeEY|fQ9QexuqE}ZRo`=<`QQf*+sT6KiBQX1Q54}a zJZ6+Cxt9YjI%KFS4=3?(pDR3f9E3G-aqX9fRKUqB9`E)yx5Z!L6Ge33 z4!#~gWan3(O=2VNg4lCfFf$5Kj6mA+9ntGI8-v2=9sfDTg3ZG8W`eK{D+Dc zX$Q2(9dx)cpt)@H<1RytsTi3$13H9X8EOEvCzItR;?7T7VkjyTgl-9DZ?51V*jBH4 z^cDMdp+YggLvONq!gL6tctJH!eW=;=d?<$p($S8f zS0@gO$9@DGg3A=3QfJ?3iw!>xI6IR8{u7h3ouI@ZK9E5v|MwLh=+?y|=x;YDq} z0A7AyO0nHaEx$mTDoF&Y9qqoA%)<@0*Dl<2p@7}fU%;4N5+?+@d4xGis{_*woy&gEHi}D#^ z6YtU!#b-=me2t4QgzMN?0bI|{?m~&U_S|mI-8w+M$fE?PvI5*?!jyBzFcBQRUT(wQ z#F7NY+==9qtnNPtFnz1%DGBLfK6uS|Ah+ZOop%;FU}NCo3hg9$;EDvc=Q6P8o&x(= zO1H+pKaI8s1ErJ7I5*>H2R(VDX83_Yu^;ug`;pd5G!UJ3a&zmwoNewknx0dYz8@p? zi4m8F_eEj_we2A%TCk9rUAXYoEyrKyaT19iWmCa_kye~(ZftB60RW(oBm!m9^lVCD z=^@yF(l6ZLu@Pf4VzHlrs$@1_rQEe9oOe%T&^T!ImW5)y_ ze02C6huS+I(KFzJ>78w$Un8aBUqZAkfj7+3jN*?TPJnGK+;ak$WPPmrc3b4_Z)JTJ z#PdP_qLviyRffNsxyZ|n>u?^tu9ALHC9tI(MnKpsp&s$fS&RYh15$7JL0=ek<-PQrP%|c) zSeVx76*)aBp>uwuwC~5@H+&Lm#NcWJettWe@%Y2n z5pb*1jV7)^Hbo0@#tz$rIbg*P9Ra2vMH`SJK+_4uQ4Q?&tNan;tc z)Oe7B%|u4f-t*N=u#6Y5@#!@yAC5YluI17 z*x-Hmgmh-(si&zTr;L95;N#%^faK?6_2&aSPreB1_G`d)UNLWcV6dcDV$Q_bFBXYKZ>h1u4%*{imAkbt2K4&4 zUB$p*TJuJ`WuyTA{8`t+23twz2(d$3;@{^IZ8LwCDd%| z?5c>&;^ zJ-UAMK*-d;Ux9tXe7@q3@FuL(l)<$G<~t8A^%82f{?JRsIQEgkzv0H3#nT&t){eKX z(W7j0%Z7s72d&@En>Ru?JHZmRm!kvQTS@V3NP(0ysHq@qbF|MCz51Y@?&uvUbprBp z3I&Z{jEPRDAw|V|{At3VR5W4>_~@Fj2}PunxXO+@%t)#X zW}%{#D16DSfqHt;axQB_(_P5fv=mQo=pBs`s}M5ez$93yz>sBYX%R^X;G%Ef&5^i+ zTe9NgUDRB`{^Gv z%#ew0hnPlP&d^DmO<*-`k*>fl1mb9SxY^p=2ljXP&6~os;CPY_yAaeLt4n2jMqvWA z@qpBlsK74a;NYN3rIp{X*kH+%3D^$4B$o6m;i@Ox)QEz7;xTx0Am}xSMAcEg=e`j0-UHJzf&{O|{gjQvw-yMW^X%Y{@cLhGT(uM9zqLG{QH)Xh-8wLBK*Q)Mm7YSBydee+}>X$KyMsf zAJznwuut9~FBwvp*oCn2@zF+3Sf2R@8Bl^p3FRop91?8`dv~$7x&EDjY|Z!3&=j(v z&psTL2R!{f=mCFYLdZJB4B3-&Znw!%CvO4U7qlt3coACC28ym4nvxO`z1Rga3OY6O zUG5rgVL!#HP2pc1=`ZoK!+`?}-Vn2vef)4};<53VFpMs20&= zi2i_d0d&Aa3`EF&M|TFu1=V~!I=AhyP-EhH7U36=CzqW+)PRc-o!srWD^y6t4+#W| zoAWT?Bzo1;-=DA%HYgPHJi^C!zIypm6|%&4!>zEH#FoG!~4og>pok}9fq8KQk-P!%Q9^fqR zFA`K=WvxcI9&kWB{fq)zEPZ5Q4_bu}+rt)nR;NhFpKz|*|4#0)4>X{AFr;1YIwyQx z+++qyVk4hQ+zI6kbSSxn&GIA7!3@el=#Xq)%;ydng%vKH_Z%*YgOAbev zC3ix|E|g3^wn8CnU4e7?D6^g|C|yZ)z-+}8VRFS1xkJVzL|6)&c_u_c#fZ;wV2U6! zB+$#{q@`4rXcjC|^{Jkhevf2|#}F{>r(%cn*byRRLM*j3X>q3Cy!}gi@o(V`S2o=F zESXa`4zk8ZE1pnvr)@Qwh~3Gjw^@+Yr|)Z4AX#p_Ny*j6<#TSwv}Vb3_KV3oLxW8Q z(kwS~sc+NYmB4aI1?{i+rS+1U`uq(kTXT$Vs<}zLpbu>%?>o@q)@x@aK|&pxdMiH8 zs83uG>Ps>ijff*@O2M2I4#Z!$P?c7j$c_9=W)hCh$|$2&K{$;Yp5;ixS@2q#>STsI z)nbiCoIprwbZ}I4DBgs$?%%Ov$KP8dfea+ohaF+6y#*nKt1?}3=`%d4SjSuE>GY$7 z*;CM1aT^m;-YKqs_ZuM5Ht0EMJFHudM~YvT`pxe_>+CQx*?EjMp3qgtpRpJ&0?)Gy`Cfsk4&*gY=CTh?U}kqjIY2pxHAjanNC|=4@2&P z`4TAlL-M$$qUE)005h#9tu_zV84v5+A$I=zTGZ_tSaXQ@@QnjEB*G7>MIchkJxa$eR)q1IaaImhavT{0;7`8m7)c50eh1sI{JrFi`lduno5)_%P z=jpY%aI_#f79!_e(J15f$eD*&+XFXW+BeWki6$$vZ-Q!lSqEAN#f2h3@|A;` zxXT-`6=lrm-uvZNA6@1_7z9X?P-*}5Dl39CXg&E?og0zN?Q4K$dHa<6%`f3!oaIHB zTbnuin%~g^w#@A_Jjok0w-qJ2xK`R1VK;vMX5c#A8t8RWG2%hro<4m_z2`KYP4YF9 zVP-~pahCl@M-~nc4{C2cb6+i~q^4h`DGRWVkFMIu<6YfT>qvTA{t$SKCuZj62JtT* zU}Le6{N~$W_cLm<_@3@mrXRn1(8a0J{#!ZBu*U;aOteKORJWWp=!P6MiR+QK%OT3Y z<%Tbh+&e%Bi6<4=tl9G;gWMv45-)~@hp&4I|MP19zpo=+ zykN!}M&gMdJK=6cHafpSwa=Ziflg7eY)I5V5X)QTb?XARQl)RA@0hNnQDDt@Gx z$z)`qN#y*-hImFVBl9rYxh7$kfz>u|NNAAtY}Pt9@0bGcSWF3jr$)#?CODy^RS&ra<8i8)ZV`_g9= z(Keok%JqA?swd*Pbb20YBgkt@&~syC`W|9fXUrM}Nos+}`Ru^XZ{6*n0LrYviL+*V z-)6(3JvbU=u= O3@n^LHND-~62l2kVq&UYYLj-NPN+%4B<&#>7Oj73G%!+fsX^ z@<~iL_*qxl3xQelXf=yksU@C>)XpYsQ=aCstZNdIk&d)eM^2wjr*G_K42`~+0Z5h5;o4wQ)m z$gT$O%Oj@_v%T-3MaJhY5Jgi5d2VjW{3HrG%i0yqD*P~w}3qfc~I;mpeNw*wykby&QV=Ky$1L4?V zP;Wq~>6y|D6hF8-yuI6XtHEg)=n6hM0!zJlnk!@3|J8u4H!E0YF33-lGjB*tYLMKQGuJCi^K+Ioy!V1A3@SU9$y_AmpbM9A}{SAXU+t(u3=~ zM;0s4Rr^COT;mUw=(hXi-DVeg(CAuKq!eNzy!(+_m0+mbW&;f=lvss6!^b9Ko3m%T zLH+LYch35ss{4wYE;f@J=$RrDU3m>OcAr171r9DfSJ3&|SD+voaho}0e5u_~x%~3* zZhB#A8TB5~1dUyI3{%N7$}4LzBYyk|`in@eN}XXlz~_09)a&DTAEfG{0b6Jq9L_2i zxgLGtdNdq5_1Z>ItL$usbi@9^=a04J){DppIuB((6xwtU-s^J-3ky?28|^9CA2f6Z zN5?z~RT;r-r^=FZwPmP9rlzJXZ{grB5Ctmb9vi6-?7DsU88Ef2a`-vYM%*de^1B!4 zw8h6SUc8W>1C9PPrgJ_7?1Vmp0AKEI4M#R8uU~gSI@7RPMr*KJE@bW^u1B<&#H&#A zb#$Q-hidc#1YP@4(L1lK&aCnG>%%DpT1rYm733CIic)h&PV?b?=vd}VfXI2R$4v_g zkf_8g`)IU*6vcYe{-zb$h*B&&1VTs1;GD|w?_??56(2)Ciignj!cDMS>acX9`j@Wn zCMPEYu1RVZSdQ)=zV&9R>gu&?ho}4cxafzJBd%G?lqpYYeEIAD<8zw#5VE!G$?HsU z2lGbF_I6f2OexNMMq{cuV#jKZD_On;yd}u0@Rp`{+j*v~9oKUIH1?>{kz92o$ml0f^duZ&%i@-LBKB!@! zn0xuP6rK45XzGwW8uFPR{dK*1soVcDc}$uL;a(aq-Sn(`*sLzM`(Ab=KZWd;UPg7Z zWu`+_fqfvOpNOs+vj;WKo!l!YzJKf52C#ZoX@8K^5;gLm+^!J~= zK{`3I;4-~eD|Pq2Cv`oeQ3OIA1k{h-bBefx^L@QDn}wHiO>5aqDc2+Cr_l>=SE*>4 zMvjkx%6S8irD^NBW)4enm1MIPlOzX z-C~t8JR4e^>W+_&`+INUS7o0Wtm1%+Lrx;cFgIOwiW!M)bpSOV{;1>LcZVs`eM>NLEU$v^Z}GS zxeG{|a?QFMFTacvuT+9AN#(q_P4){fKfjAV;o``G25J?>UwW2fojQI4@zv!Bi*EnQ z+=bf*Wfmj-@A4E^+}WVeC)#~WZ>>e#-TW=ODzmT0^^VhLr#JT8nxer!{sgg5U0aF^ zdJ@+Y`DXlk;G$rj_Uw2MF}}zDCOIHmMaEB$VuD5x?!O&PznkI*=ECssaF=v6kk+!8 zOb*9uRrUS(tW>K&H!#)-0S<@|UJ#MY;OsOv=MU9mES}U#`5%9+i>W zNe?_k*`~bn#GEYQ{+X%qESWAf^EcKq+YGN-<3Z3v?mX^00wPp-j!BsBl<-{pYs_KP z;2L`UsYVrT5nHG5SCkuhPy`1JXrr&5?nrI&CCt(XE)8WzVqQIrkY4`QD!3wuI}m2< zd$cwj+fE&JxaEfk`7pv(`z(J?5@7Gkf0>E_jUYM_k$|}(|Mj)|gE%83zZamK7`_?BfpX8M946Q}Y>PftM}97hmq9=t(c=_Aw-S8~^gUQ)5IUU{kLKl+Cd?_g>A}nxqI+z24f=vu-39OU?Zlk^ghJ_YiS7lE5F>}wFLCH zMY+cG`tny~HV*o0ua<%RUH-#tBKG|t(@LCv?-qDlxh3XW!q({J-dZ8Ioh8o!OnG%8 zqx+4}{h0;g3QxVu#paWd3vVrV*P8&{kwz9G8Jp_^W`ys)7s8mr5f$U;nPLxnwmE> zlGXEuB509CWTo%*FQKl3Qk@Fl=8^YhQx1RmLkb^sJWv7sXD`(9q z;-U-X2gljbX)_GamBnRCoC0%ioy?ZrXG7evdfE6XfTg%9=4oyYh-V5(y!{TU%QXm)cTCF>Rewp%qUSdOo}>6g+F1 zpy+D*bRA^x&!@Y50Gw<&di1EX55d9ghKfh_wVtVG4<9~UH?E-*8c*A4`hE7P0Bs4B zKYV6~y*43`K-bdT)D&?qHv&%?**1BSNE#m3ShO#-mFgu4v~@S04*xjS!)IJ)&EV-O zbO&NDo#_oT+D=s!N{?coeW9d)Xv0-z?A*m-QNkKLVY}P z;zYyjkdTnKcG8*2em@5P%Dfe9z|3y!!=Ji_ZzR*Kw9d>)%8$xEegAbgO*Yf4^56Up zz8NH8Y6A-F&!>YkGy^L zD&!jJB6|M zbaKb@Vt-A?tOKuO_nbay@?wyoEH}NXU(>3QyktUA`pI7_4~VE;{xq7u<6^XmYo$}= zTgIt#;?68DBIi>7{b1l&OJ`65*=-Ck=K#tm#Jnm8jmQJs?ohrBsSIVR=t{h6y=&uV*#pD)8+cccrmp#PA zfWo5U+@|QB0qdf)j|^r?_Z~#8~ z$?xR>NuFlAiv629`_q+bw%_&RBP{gw^-VAi6me%9N?Q!FwIKIb%g{Xw3$3%fCu$t| zK7Uj9e|;yDA+6QM8@I{i$W!;*s4)e^e>U@=9`~*^ysY`@4@{-!EOR^5LTV!8`9XPOL{J`rSTo$LU=o;7j(b&P|F~8_nLJI5ryPQn5eFa(Md{-KZGS zR#H*O@aRIp>RYnJV96appAuxcBKON>_vR+c+skazmGYAtDjzikNGFy-lpDst{vx59 zPR-?i&}a^pZUK_)v7&tXSQx??$8{UW;KEIisqA(EKX1a{QPjsXQY4=|o(`mTKux~* zWujsEBpci1l}1y!&>*FcS?IhTf5%uMSp6R9zw@)YKKYo>$JEc%ZN9xpWWOk zho7c~S3!PqI8;v&n)%e2Y37|v9uuP;tlmd96|#J)w=( zxn-{vIyGPuSh2rRx~sa#JkRvGpSl+zYMTKsZbkp(?AzAyqky=W!DCtOOsVUcy5Dee z;%yULd44~t>+vvl?XQcEKMc_#b4kB&r&22wM;5dJy-H>Tp~9!wrsVna=j$GuwYHK@&V>#fxmRSnqT^I-2Y|V7)DT-F zy`x(vyNtSBYDKpBWQjOsatu!^sR8h(Iflf4)W@N1ZEa&|wVC{_`p$=Bw#jFzV81m2 zpVW10Q9gct{zC2W*&oD0Q|@=sl_@2YVqafXngh^`eSV6*`F&2NYf<0U1fBxqnFIzC zuhjB0y+WRl1$WT$3!v^_b1R2$a*SS3lB7Q1dK5ugegiqPWqS5kOQwp!Hupz`NhOn+ zBIn7gJzxJmQ{)e=+opzUIJ0M%DM+txd$$}Q9#L`ERj;T3p&74G&}x)^y1t8du>!F*jtpiLR_X&$u?lt?SG0`A@_fUd%HI zxdoly0mFcgN`3?f0{4aLTq%Ev_tQ2gzrGCEu#Y$jVf~dC1oEFsJPv%|LW%N~pJQin z8n{kc+G6mh7KZ^@@bNX-!Z&$rwDI@c$3ux^mgsWZTt?PY9;*EOT7rzY8jFTaNy0|`L%I-;e>uni@Z$Q z^)%@UU)IcD0j-dKqfajHbo{uYSi0HL8;Rfk{r$BaGE^eoj4s92KF!stmI=42w0AaM zA%a5s>;3=Az?25BS!Wj&$9*jg(sgNzX-<{1c1M;6i_U#o_I*wcYy*2#C`+YiAm^39 z@KiZrHVxQ$%j!GbW%C=`1NU}y+t<_SVvlhh8MHq3#pMo=#RSX2?GW@2q?cLd%bl8Q zNvlnm$uTk{9?uB&W_vHVLTKclL_STv9%?**893_$(L7Y5di@EmSdu)!an|adQ)ST# ziy-0EtdY~6{2kj-ClTsnjz~FQK|^AsF!iR$`FydLkc)_Y?u0h*HUg@)1s0r6Qa8)% zAUr&-gw8e6f72E@p3gFrZ@o*wsXE^mCCOm#4%XA1d!~#(+-lAB-%rt|EebvM{NW4| zy^-O(ypC!RxP$cxto@iUdjvIqtZ+j7^rP;`9Pgn@$WGFIQy)0KjkvOG{wrm?Hy#I` zttD+7U3f{DH9*&@9Wp;{L0KTu(u2d<-iNOGL@t)0_J{a8OwAfwyPuz&tHJg7D67T2 zqX7oHK#O#{YF*#rRCyc+fqnS24jh^9IeO z?-Vlq8Wt<l0D z-p8u*vXZQ=mOBJjWWUj@%-#t54R%B-e=SlyYVQV73NGgz%l#DpRz$9_EJId1c2FTn z?Tb>)J`~jHiO`fqt~dU){%~^m!|vHLReM8)cV6c}*u(#&nAaD2&QBqJ+lvPxcRZ!iput!rt uT~kS8w}OJEf`V65BZdR~UxQ--UVel#|NjlPzGGfK4p8HLMz;;zWB)%rXGm%Q literal 0 HcmV?d00001 diff --git a/test/llm_openai.jl b/test/llm_openai.jl index 135c14547..e01632862 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -1,5 +1,6 @@ using PromptingTools: TestEchoOpenAISchema, render, OpenAISchema -using PromptingTools: AIMessage, SystemMessage, UserMessage, DataMessage +using PromptingTools: AIMessage, SystemMessage, AbstractMessage +using PromptingTools: UserMessage, UserMessageWithImages, DataMessage @testset "render-OpenAI" begin schema = OpenAISchema() @@ -71,7 +72,7 @@ using PromptingTools: AIMessage, SystemMessage, UserMessage, DataMessage @test conversation == expected_output # Given an empty vector of messages, it should return an empty conversation dictionary just with the system prompt - messages = PT.AbstractMessage[] + messages = AbstractMessage[] expected_output = [ Dict("role" => "system", "content" => "Act as a helpful AI assistant"), ] @@ -118,6 +119,43 @@ using PromptingTools: AIMessage, SystemMessage, UserMessage, DataMessage ] # Broken: Does not concatenate system messages yet @test_broken conversation == expected_output + + # Test UserMessageWithImages + messages = [ + SystemMessage("System message 1"), + UserMessageWithImages("User message"; image_url = "https://example.com/image.png"), + ] + conversation = render(schema, messages) + expected_output = Dict{String, Any}[Dict("role" => "system", + "content" => "System message 1"), + Dict("role" => "user", + "content" => Dict{String, Any}[Dict("text" => "User message", "type" => "text"), + Dict("image_url" => Dict("detail" => "auto", + "url" => "https://example.com/image.png"), + "type" => "image_url")])] + @test conversation == expected_output + + # With a list of images and detail="low" + messages = [ + SystemMessage("System message 2"), + UserMessageWithImages("User message"; + image_url = [ + "https://example.com/image1.png", + "https://example.com/image2.png", + ]), + ] + conversation = render(schema, messages; image_detail = "low") + expected_output = Dict{String, Any}[Dict("role" => "system", + "content" => "System message 2"), + Dict("role" => "user", + "content" => Dict{String, Any}[Dict("text" => "User message", "type" => "text"), + Dict("image_url" => Dict("detail" => "low", + "url" => "https://example.com/image1.png"), + "type" => "image_url"), + Dict("image_url" => Dict("detail" => "low", + "url" => "https://example.com/image2.png"), + "type" => "image_url")])] + @test conversation == expected_output end @testset "aigenerate-OpenAI" begin diff --git a/test/messages.jl b/test/messages.jl index ec1aa0a31..a08c0266d 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -1,4 +1,6 @@ -using PromptingTools: AIMessage, SystemMessage, MetadataMessage, UserMessage, DataMessage +using PromptingTools: AIMessage, SystemMessage, MetadataMessage +using PromptingTools: UserMessage, UserMessageWithImages, DataMessage +using PromptingTools: _encode_local_image, attach_images_to_user_message @testset "Message constructors" begin # Creates an instance of MSG with the given content string. @@ -14,3 +16,66 @@ using PromptingTools: AIMessage, SystemMessage, MetadataMessage, UserMessage, Da @test msg.content == content end end + +@testset "UserMessageWithImages" begin + content = "Hello, world!" + image_path = joinpath(@__DIR__, "data", "julia.png") + image_url = "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png" + + # Cannot init with string only // without images + @test_throws AssertionError UserMessageWithImages(content) + # non-existing image path + @test_throws AssertionError UserMessageWithImages(content; + image_path = "data/does_not_exist_123456.png") + + # Test with URL + msg = UserMessageWithImages(content; image_url) + @test typeof(msg) <: UserMessageWithImages + @test msg.content == content + @test msg.image_url == [image_url] + + # Creates an instance of UserMessageWithImages with the given content string and image_path. + msg = UserMessageWithImages(content, image_path = image_path) + @test typeof(msg) <: UserMessageWithImages + @test msg.content == content + @test msg.image_url == [_encode_local_image(image_path)] + + # Creates an instance of UserMessageWithImages with the given content string and image_path. + msg = UserMessageWithImages(content; image_path, image_url) + @test typeof(msg) <: UserMessageWithImages + @test msg.content == content + @test msg.image_url == [image_url, _encode_local_image(image_path)] +end + +@testset "attach_images_to_user_message" begin + content = "Hello, world!" + image_url = "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png" + msg = attach_images_to_user_message(content; image_url) + @test typeof(msg) <: UserMessageWithImages + @test msg.content == content + @test msg.image_url == [image_url] + + # Test with UserMessage + msg = UserMessage(content) + msg = attach_images_to_user_message(msg; image_url) + @test typeof(msg) <: UserMessageWithImages + @test msg.content == content + @test msg.image_url == [image_url] + + # Test with multiple UserMessages + msgs = [UserMessage("Hello, world!"), UserMessage("Hello, world!")] + # default attach_to_latest = true + output = attach_images_to_user_message(msgs; image_url) + @test typeof(output[1]) <: UserMessage + @test typeof(output[2]) <: UserMessageWithImages + @test output[1].content == "Hello, world!" + @test output[2].content == "Hello, world!" + @test output[2].image_url == [image_url] + # attach_to_latest = false + @test_throws AssertionError attach_images_to_user_message(msgs; + image_url, + attach_to_latest = false) + # no UserMessages + msg = UserMessageWithImages(content; image_url) # unclear where to add the new images! + @test_throws AssertionError attach_images_to_user_message(msg; image_url) +end \ No newline at end of file diff --git a/test/utils.jl b/test/utils.jl index 438aba00e..138343257 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,4 +1,5 @@ using PromptingTools: _extract_handlebar_variables, _report_stats +using PromptingTools: _string_to_vector, _encode_local_image @testset "extract_handlebar_variables" begin # Extracts handlebar variables enclosed in double curly braces @@ -38,3 +39,19 @@ end expected_output = "Tokens: 6 in 5.0 seconds" @test _report_stats(msg, model, Dict(model => (0, 0))) == expected_output end + +@testset "_string_to_vector" begin + @test _string_to_vector("Hello") == ["Hello"] + @test _string_to_vector(["Hello", "World"]) == ["Hello", "World"] +end + +@testset "_encode_local_image" begin + image_path = joinpath(@__DIR__, "data", "julia.png") + output = _encode_local_image(image_path) + @test output isa String + @test occursin("data:image/png;base64,", output) + output2 = _encode_local_image([image_path, image_path]) + @test output2 isa Vector + @test output2[1] == output2[2] == output + @test_throws AssertionError _encode_local_image("not an path") +end From c545cbfac2a6a874d1c765535cdd1925fd78c5ad Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 19 Nov 2023 21:59:41 +0000 Subject: [PATCH 012/251] remove duplication --- src/extraction.jl | 5 +++++ test/extraction.jl | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/extraction.jl b/src/extraction.jl index 74ab48463..b96a290a6 100644 --- a/src/extraction.jl +++ b/src/extraction.jl @@ -162,6 +162,11 @@ function function_call_signature(datastructtype::Type; max_description_length::I ## docstrings docs = extract_docstring(datastructtype; max_description_length) !isempty(docs) && (schema["description"] = docs) + ## remove duplicated Struct docstring in schema + if haskey(schema["parameters"], "description") && + schema["parameters"]["description"] == docs + delete!(schema["parameters"], "description") + end return schema end diff --git a/test/extraction.jl b/test/extraction.jl index 3a21c1964..dc65bd693 100644 --- a/test/extraction.jl +++ b/test/extraction.jl @@ -210,8 +210,7 @@ end "weight" => Dict{String, Any}("type" => "number"), "age" => Dict{String, Any}("type" => "integer")), "required" => ["age"], - "type" => "object", - "description" => "Some docstring\n"), + "type" => "object"), "description" => "Some docstring\n") @test output == expected_output From 43a88a548a06365f44af67ff6337f65e5e8fa5c6 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 19 Nov 2023 22:03:29 +0000 Subject: [PATCH 013/251] update changes --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe1b8a50..9df47c177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,4 +7,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). -- Add `aiextract` function to extract structured information from text quickly and easily. See `?aiextract` for more information. \ No newline at end of file +- Add `aiextract` function to extract structured information from text quickly and easily. See `?aiextract` for more information. +- Add `aiscan` for image scanning (ie, image comprehension tasks). You can transcribe screenshots or reason over images as if they were text. Images can be provided either as a local file (`image_path`) or as an url (`image_url`). See `?aiscan` for more information. \ No newline at end of file From f82e37e9c5c67789fc9cf1616a8461ba924c85c5 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 23 Nov 2023 21:58:07 +0000 Subject: [PATCH 014/251] push ollama --- CHANGELOG.md | 3 +- README.md | 28 +++ src/PromptingTools.jl | 1 + src/llm_interface.jl | 11 +- src/llm_ollama_managed.jl | 337 +++++++++++++++++++++++++++++++++++++ src/llm_openai.jl | 7 +- src/messages.jl | 1 + test/llm_ollama_managed.jl | 154 +++++++++++++++++ 8 files changed, 537 insertions(+), 5 deletions(-) create mode 100644 src/llm_ollama_managed.jl create mode 100644 test/llm_ollama_managed.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 9df47c177..b099f7309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,4 +8,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). - Add `aiextract` function to extract structured information from text quickly and easily. See `?aiextract` for more information. -- Add `aiscan` for image scanning (ie, image comprehension tasks). You can transcribe screenshots or reason over images as if they were text. Images can be provided either as a local file (`image_path`) or as an url (`image_url`). See `?aiscan` for more information. \ No newline at end of file +- Add `aiscan` for image scanning (ie, image comprehension tasks). You can transcribe screenshots or reason over images as if they were text. Images can be provided either as a local file (`image_path`) or as an url (`image_url`). See `?aiscan` for more information. +- Add support for [Ollama.ai](https://ollama.ai/)'s local models. Only `aigenerate` and `aiembed` functions are supported. \ No newline at end of file diff --git a/README.md b/README.md index a88b8782d..53b083114 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ For more practical examples, see the `examples/` folder and the [Advanced Exampl - [Classification](#classification) - [Data Extraction](#data-extraction) - [OCR and Image Comprehension](#ocr-and-image-comprehension) + - [Using Ollama models](#using-ollama-models) - [More Examples](#more-examples) - [Package Interface](#package-interface) - [Frequently Asked Questions](#frequently-asked-questions) @@ -352,6 +353,33 @@ using Markdown msg.content |> Markdown.parse ``` +## Using Ollama models + +[Ollama.ai](https://ollama.ai/) is an amazingly simply tool that allows you to run several Large Language Models (LLM) on your computer. It's especially suitable when you're working with some sensitive data that should not be sent anywhere. + +TODO: assumes it's working, but it's not yet! + +We can use Ollama models with the `aigenerate` function: +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() + +msg = aigenerate(schema, "Say hi!"; model="openhermes2.5-mistral") +# [ Info: Tokens: 69 in 0.9 seconds +# AIMessage("Hello! How can I assist you today?") +``` + +And we can also use the `aiembed` function: +```julia +msg = aiembed(schema, "Embed me", copy; model="openhermes2.5-mistral") +msg.content # 4096-element JSON3.Array{Float64... + +msg = aiembed(schema, ["Embed me", "Embed me"]; model="openhermes2.5-mistral") +msg.content # 4096×2 Matrix{Float64}: +``` + +TODO: Add FAQ how to setup Ollama.ai + ### More Examples TBU... diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index c34a9fe61..466a4e1cb 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -50,6 +50,7 @@ include("extraction.jl") ## Individual interfaces include("llm_openai.jl") +include("llm_ollama_managed.jl") ## Convenience utils export @ai_str, @aai_str diff --git a/src/llm_interface.jl b/src/llm_interface.jl index e3c17e148..3921dcdee 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -58,6 +58,7 @@ It uses the following conversation structure: struct ChatMLSchema <: AbstractChatMLSchema end abstract type AbstractManagedSchema <: AbstractPromptSchema end +abstract type AbstractOllamaManagedSchema <: AbstractManagedSchema end """ Ollama by default manages different models and their associated prompt schemas when you pass `system_prompt` and `prompt` fields to the API. @@ -67,7 +68,15 @@ Warning: It works only for 1 system message and 1 user message, so anything more If you need to pass more messagese / longer conversational history, you can use define the model-specific schema directly and pass your Ollama requests with `raw=true`, which disables and templating and schema management by Ollama. """ -struct OllamaManagedSchema <: AbstractManagedSchema end +struct OllamaManagedSchema <: AbstractOllamaManagedSchema end + +"Echoes the user's input back to them. Used for testing the implementation" +@kwdef mutable struct TestEchoOllamaManagedSchema <: AbstractOllamaManagedSchema + response::AbstractDict + status::Integer + model_id::String = "" + inputs::Any = nothing +end ## Dispatch into default schema const PROMPT_SCHEMA = OpenAISchema() diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl new file mode 100644 index 000000000..83fa85998 --- /dev/null +++ b/src/llm_ollama_managed.jl @@ -0,0 +1,337 @@ +## Schema dedicated to [Ollama's managed models](https://ollama.ai/), which also managed the prompts +## It's limited to 2 messages (system and user), because there are only two slots for `system` and `prompt` +## +## Rendering of converation history for the Ollama +""" + render(schema::AbstractOllamaManagedSchema, + messages::Vector{<:AbstractMessage}; + kwargs...) + +Builds a history of the conversation to provide the prompt to the API. All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. + +Note: Due to its "managed" nature, at most 2 messages can be provided (`system` and `prompt` inputs in the API). +""" +function render(schema::AbstractOllamaManagedSchema, + messages::Vector{<:AbstractMessage}; + kwargs...) + ## + @assert length(messages)<=2 "Managed schema only supports 2 messages (eg, a system and user)" + @assert count(isusermessage, messages)<=1 "Managed schema only supports at most 1 User message" + @assert count(issystemmessage, messages)<=1 "Managed schema only supports at most 1 System message" + ## API expects: system=SystemMessage, prompt=UserMessage + system, prompt = nothing, nothing + + # replace any handlebar variables in the messages + for msg in messages + if msg isa SystemMessage + replacements = ["{{$(key)}}" => value + for (key, value) in pairs(kwargs) if key in msg.variables] + system = replace(msg.content, replacements...) + elseif msg isa UserMessage + replacements = ["{{$(key)}}" => value + for (key, value) in pairs(kwargs) if key in msg.variables] + prompt = replace(msg.content, replacements...) + elseif msg isa UserMessageWithImages + error("Managed schema does not support UserMessageWithImages. Please use OpenAISchema instead.") + elseif msg isa AIMessage + error("Managed schema does not support AIMessage and multi-turn conversations. Please use OpenAISchema instead.") + end + # Note: Ignores any DataMessage or other types + end + ## Sense check + @assert !isnothing(prompt) "Managed schema requires at least 1 User message, ie, no `prompt` provided!" + ## Add default system prompt if not provided + isnothing(system) && (system = "Act as a helpful AI assistant") + + return (; system, prompt) +end + +## Model-calling +""" + ollama_api(prompt_schema::AbstractOllamaManagedSchema, prompt::AbstractString, + system::Union{Nothing, AbstractString} = nothing, + endpoint::String = "generate"; + model::String = "llama2", http_kwargs::NamedTuple = NamedTuple(), + stream::Bool = false, + url::String = "localhost", port::Int = 11434, + kwargs...) + +Simple wrapper for a call to Ollama API. + +# Keyword Arguments +- `prompt_schema`: Defines which prompt template should be applied. +- `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` +- `system`: An optional string representing the system message for the AI conversation. If not provided, a default message will be used. +- `endpoint`: The API endpoint to call, only "generate" and "embeddings" are currently supported. Defaults to "generate". +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to empty `NamedTuple`. +- `stream`: A boolean indicating whether to stream the response. Defaults to `false`. +- `url`: The URL of the Ollama API. Defaults to "localhost". +- `port`: The port of the Ollama API. Defaults to 11434. +- `kwargs`: Prompt variables to be used to fill the prompt/template +""" +function ollama_api(prompt_schema::AbstractOllamaManagedSchema, prompt::AbstractString; + system::Union{Nothing, AbstractString} = nothing, + endpoint::String = "generate", + model::String = "llama2", http_kwargs::NamedTuple = NamedTuple(), + stream::Bool = false, + url::String = "localhost", port::Int = 11434, + kwargs...) + @assert endpoint in ["generate", "embeddings"] "Only 'generate' and 'embeddings' Ollama endpoints are supported." + ## + body = Dict("model" => model, "stream" => stream, "prompt" => prompt, kwargs...) + if !isnothing(system) + body["system"] = system + end + # eg, http://localhost:11434/api/generate + api_url = string("http://", url, ":", port, "/api/", endpoint) + resp = HTTP.post(api_url, + [],# no headers + JSON3.write(body); http_kwargs...) + body = JSON3.read(resp.body) + return (; response, resp.status) +end +# For testing +function ollama_api(prompt_schema::TestEchoOllamaManagedSchema, prompt::AbstractString; + system::Union{Nothing, AbstractString} = nothing, endpoint::String = "generate", + model::String = "llama2", kwargs...) + prompt_schema.model_id = model + prompt_schema.inputs = (; system, prompt) + return prompt_schema +end + +## User-Facing API +""" + aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + +Generate an AI response based on a given prompt using the OpenAI API. + +# Arguments +- `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema` not `AbstractManagedSchema`) +- `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` +- `verbose`: A boolean indicating whether to print additional information. +- `api_key`: Provided for interface consistency. Not needed for locally hosted Ollama. +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to empty `NamedTuple`. +- `api_kwargs::NamedTuple`: Additional keyword arguments for the Ollama API. Defaults to an empty `NamedTuple`. +- `kwargs`: Prompt variables to be used to fill the prompt/template + +# Returns +- `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. + Use `msg.content` to access the extracted string. + +See also: `ai_str`, `aai_str`, `aiembed` + +# Example + +Simple hello world to test the API: +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() # We need to explicit if we want Ollama, OpenAISchema is the default + +msg = aigenerate(schema, "Say hi!"; model="openhermes2.5-mistral") +# [ Info: Tokens: 69 in 0.9 seconds +# AIMessage("Hello! How can I assist you today?") +``` + +`msg` is an `AIMessage` object. Access the generated string via `content` property: +```julia +typeof(msg) # AIMessage{SubString{String}} +propertynames(msg) # (:content, :status, :tokens, :elapsed +msg.content # "Hello! How can I assist you today?" +``` + +Note: We need to be explicit about the schema we want to use. If we don't, it will default to `OpenAISchema` (=`PT.DEFAULT_SCHEMA`) +___ +You can use string interpolation: +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() +a = 1 +msg=aigenerate(schema, "What is `\$a+\$a`?"; model="openhermes2.5-mistral") +msg.content # "The result of `1+1` is `2`." +``` +___ +You can provide the whole conversation or more intricate prompts as a `Vector{AbstractMessage}`: +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() + +conversation = [ + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] + +msg = aigenerate(schema, conversation; model="openhermes2.5-mistral") +# [ Info: Tokens: 111 in 2.1 seconds +# AIMessage("Strong the attachment is, it leads to suffering it may. Focus on the force within you must, ...") +``` + +Note: Managed Ollama currently supports at most 1 User Message and 1 System Message given the API limitations. If you want more, you need to use the `ChatMLSchema`. +""" +function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE; + verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_CHAT, + http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + ## + global MODEL_ALIASES, MODEL_COSTS + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + conversation = render(prompt_schema, prompt; kwargs...) + time = @elapsed resp = ollama_api(prompt_schema, conversation.prompt; + conversation.system, endpoint = "generate", model, http_kwargs, api_kwargs...) + msg = AIMessage(; content = resp.response[:response] |> strip, + status = Int(resp.status), + tokens = (resp.response[:prompt_eval_count], + resp.response[:eval_count]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + + return msg +end + +""" + aiembed(prompt_schema::AbstractOllamaManagedSchema, + doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, + postprocess::F = identity; + verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_EMBEDDING, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), + api_kwargs::NamedTuple = NamedTuple(), + kwargs...) where {F <: Function} + +The `aiembed` function generates embeddings for the given input using a specified model and returns a message object containing the embeddings, status, token count, and elapsed time. + +## Arguments +- `prompt_schema::AbstractOllamaManagedSchema`: The schema for the prompt. +- `doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}`: The document or list of documents to generate embeddings for. The list of documents is processed sequentially, + so users should consider implementing an async version with with `Threads.@spawn` +- `postprocess::F`: The post-processing function to apply to each embedding. Defaults to the identity function, but could be `LinearAlgebra.normalize`. +- `verbose::Bool`: A flag indicating whether to print verbose information. Defaults to `true`. +- `api_key::String`: The API key to use for the OpenAI API. Defaults to `API_KEY`. +- `model::String`: The model to use for generating embeddings. Defaults to `MODEL_EMBEDDING`. +- `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to empty `NamedTuple`. +- `api_kwargs::NamedTuple`: Additional keyword arguments for the Ollama API. Defaults to an empty `NamedTuple`. +- `kwargs`: Prompt variables to be used to fill the prompt/template + +## Returns +- `msg`: A `DataMessage` object containing the embeddings, status, token count, and elapsed time. + +Note: Ollama API currently does not return the token count, so it's set to `(0,0)` + +# Example + +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() + +msg = aiembed(schema, "Hello World"; model="openhermes2.5-mistral") +msg.content # 4096-element JSON3.Array{Float64... +``` + +We can embed multiple strings at once and they will be `hcat` into a matrix + (ie, each column corresponds to one string) +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() + +msg = aiembed(schema, ["Hello World", "How are you?"]; model="openhermes2.5-mistral") +msg.content # 4096×2 Matrix{Float64}: +``` + +If you plan to calculate the cosine distance between embeddings, you can normalize them first: +```julia +const PT = PromptingTools +using LinearAlgebra +schema = PT.OllamaManagedSchema() + +msg = aiembed(schema, ["embed me", "and me too"], LinearAlgebra.normalize; model="openhermes2.5-mistral") + +# calculate cosine distance between the two normalized embeddings as a simple dot product +msg.content' * msg.content[:, 1] # [1.0, 0.34] +``` + +Similarly, you can use the `postprocess` argument to materialize the data from JSON3.Object by using `postprocess = copy` +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() + +msg = aiembed(schema, "Hello World", copy; model="openhermes2.5-mistral") +msg.content # 4096-element Vector{Float64} +``` + +""" +function aiembed(prompt_schema::AbstractOllamaManagedSchema, + doc::AbstractString, + postprocess::F = identity; verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_EMBEDDING, + http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) where {F <: Function} + ## + global MODEL_ALIASES, MODEL_COSTS + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + time = @elapsed resp = ollama_api(prompt_schema, doc; + endpoint = "embeddings", model, http_kwargs, api_kwargs...) + msg = DataMessage(; + content = postprocess(resp.response[:embedding]), + status = Int(resp.status), + tokens = (0, 0), # token counts are not provided for embeddings + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + + return msg +end +function aiembed(prompt_schema::AbstractOllamaManagedSchema, + docs::Vector{<:AbstractString}, + postprocess::F = identity; verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_EMBEDDING, + kwargs...) where {F <: Function} + ## + global MODEL_ALIASES, MODEL_COSTS + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + ## Send each document individually (no parallelism) + messages = [aiembed(prompt_schema, + doc, + postprocess; + verbose = false, + api_key, + model, + kwargs...) + for doc in docs] + ## Aggregate results + msg = DataMessage(; + content = mapreduce(x -> x.content, hcat, messages), + status = mapreduce(x -> x.status, max, messages), + tokens = (0, 0),# not tracked for embeddings in Ollama + elapsed = sum(x -> x.elapsed, messages)) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + + return msg +end + +function aiclassify(prompt_schema::AbstractManagedSchema, prompt::ALLOWED_PROMPT_TYPE; + kwargs...) + error("Managed schema does not support aiclassify. Please use OpenAISchema instead.") +end +function aiextract(prompt_schema::AbstractManagedSchema, prompt::ALLOWED_PROMPT_TYPE; + kwargs...) + error("Managed schema does not support aiextract. Please use OpenAISchema instead.") +end +function aiscan(prompt_schema::AbstractManagedSchema, prompt::ALLOWED_PROMPT_TYPE; + kwargs...) + error("Managed schema does not support aiscan. Please use OpenAISchema instead.") +end \ No newline at end of file diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 2e60cb919..30b956627 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -117,9 +117,11 @@ msg.content # "The sum of `1+1` is `2`." ___ You can provide the whole conversation or more intricate prompts as a `Vector{AbstractMessage}`: ```julia +const PT = PromptingTools + conversation = [ - SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), - UserMessage("I have feelings for my iPhone. What should I do?")] + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] msg=aigenerate(conversation) # AIMessage("Ah, strong feelings you have for your iPhone. A Jedi's path, this is not... ") ``` @@ -239,7 +241,6 @@ function aiembed(prompt_schema::AbstractOpenAISchema, model_id; http_kwargs, api_kwargs...) - @info r.response |> typeof msg = DataMessage(; content = mapreduce(x -> postprocess(x[:embedding]), hcat, r.response[:data]), status = Int(r.status), diff --git a/src/messages.jl b/src/messages.jl index c614cd63f..e9d206d56 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -57,6 +57,7 @@ function (MSG::Type{<:AbstractChatMessage})(prompt::AbstractString) MSG(; content = prompt) end isusermessage(m::AbstractMessage) = m isa UserMessage +issystemmessage(m::AbstractMessage) = m isa SystemMessage # equality check for testing, only equal if all fields are equal and type is the same Base.var"=="(m1::AbstractMessage, m2::AbstractMessage) = false diff --git a/test/llm_ollama_managed.jl b/test/llm_ollama_managed.jl new file mode 100644 index 000000000..d3611db3d --- /dev/null +++ b/test/llm_ollama_managed.jl @@ -0,0 +1,154 @@ +using PromptingTools: TestEchoOllamaManagedSchema, render, OllamaManagedSchema, ollama_api +using PromptingTools: AIMessage, SystemMessage, AbstractMessage +using PromptingTools: UserMessage, UserMessageWithImages, DataMessage + +# Write unit tests for the render function +@testset "render-ollama" begin + schema = OllamaManagedSchema() + @testset "render with system and prompt" begin + system = "System message" + prompt = "Prompt message" + conversation = render(schema, + AbstractMessage[SystemMessage(system), UserMessage(prompt)]) + @test conversation.system == system + @test conversation.prompt == prompt + end + @testset "render with only prompt" begin + prompt = "Prompt message" + conversation = render(schema, UserMessage(prompt)) + @test conversation.system == "Act as a helpful AI assistant" + @test conversation.prompt == prompt + ## alt with string format + conversation = render(schema, prompt) + @test conversation.system == "Act as a helpful AI assistant" + @test conversation.prompt == prompt + end + @testset "render without prompt" begin + @test_throws AssertionError render(schema, SystemMessage("System message")) + @test_throws AssertionError render(schema, AbstractMessage[]) + end + # error with UserMessageWithImages or AIMessage + @test_throws ErrorException render(schema, + UserMessageWithImages("abc"; image_url = "https://example.com")) + @test_throws ErrorException render(schema, + [AIMessage("abc")]) + # error if more than 2 user messages, or no user messages + @test_throws AssertionError aigenerate(schema, + [UserMessage("abc"), UserMessage("abc"), UserMessage("abc")]) + @test_throws AssertionError aigenerate(schema, + [UserMessage("abc"), SystemMessage("abc"), UserMessage("abc")]) + @test_throws AssertionError aigenerate(schema, + [SystemMessage("abc"), SystemMessage("abc")]) + @test_throws AssertionError aigenerate(schema, + [SystemMessage("abc")]) + @test_throws AssertionError aigenerate(schema, + [UserMessage("abc"), UserMessage("abc")]) + + # Double check templating + messages = [ + SystemMessage("Act as a helpful AI assistant"), + UserMessage("Hello, my name is {{name}}"), + ] + expected_output = (; system = "Act as a helpful AI assistant", + prompt = "Hello, my name is John") + conversation = render(schema, messages; name = "John") + @test conversation == expected_output +end + +# Sense check for the Echo Setup +@testset "ollama_api-echo" begin + # corresponds to standard Ollama response format + response = Dict(:response => "Hello!", + :prompt_eval_count => 2, + :eval_count => 1) + schema = TestEchoOllamaManagedSchema(; response, status = 200) + prompt = "Prompt message" + system = "System message" + msg = ollama_api(schema, prompt; system) + schema + msg + @test msg.response == response + @test msg.status == 200 + @test schema.inputs == (; system, prompt) +end + +@testset "aigenerate-ollama" begin + @testset "with system and prompt" begin + response = Dict(:response => "Prompt message", + :prompt_eval_count => 2, + :eval_count => 1) + schema = TestEchoOllamaManagedSchema(; response, status = 200) + prompt = "Prompt message" + system = "System message" + msg = aigenerate(schema, + [SystemMessage(system), UserMessage(prompt)]; + model = "llama2") + @test msg.content == prompt + @test msg.status == 200 + @test msg.tokens == (2, 1) + @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test schema.inputs == (; system, prompt) + @test schema.model_id == "llama2" + end + @testset "prompt with placeholders" begin + response = Dict(:response => "Hello John", + :prompt_eval_count => 2, + :eval_count => 1) + schema = TestEchoOllamaManagedSchema(; response, status = 200) + prompt = "Hello {{name}}" + msg = aigenerate(schema, prompt; model = "llama2aaaa", name = "John") + @test msg.content == "Hello John" + @test msg.status == 200 + @test msg.tokens == (2, 1) + @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test schema.inputs == + (; system = "Act as a helpful AI assistant", prompt = "Hello John") + @test schema.model_id == "llama2aaaa" + end + @testset "error modes" begin + response = Dict(:response => "Hello John", + :prompt_eval_count => 2, + :eval_count => 1) + schema = TestEchoOllamaManagedSchema(; response, status = 200) + @test_throws AssertionError aigenerate(schema, AbstractMessage[]) + @test_throws AssertionError aigenerate(schema, SystemMessage("abc")) + @test_throws AssertionError aigenerate(schema, + [UserMessage("abc"), UserMessage("abc")]) + ## disabled types + @test_throws ErrorException aigenerate(schema, + UserMessageWithImages("abc"; image_url = "https://example.com")) + end +end +@testset "aiembed-ollama" begin + @testset "single doc" begin + response = Dict(:embedding => ones(16)) + schema = TestEchoOllamaManagedSchema(; response, status = 200) + doc = "embed me" + msg = aiembed(schema, doc; model = "llama2") + @test msg.content == ones(16) + @test msg.status == 200 + @test msg.tokens == (0, 0) + @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test schema.inputs == (; system = nothing, prompt = doc) + @test schema.model_id == "llama2" + end + @testset "multi doc + postprocess" begin + response = Dict(:embedding => ones(16)) + schema = TestEchoOllamaManagedSchema(; response, status = 200) + docs = ["embed me", "and me"] + msg = aiembed(schema, docs, x -> 2 * x; model = "llama2") + @info typeof(msg.content) size(msg.content) + @test msg.content == 2 * ones(16, 2) + @test msg.status == 200 + @test msg.tokens == (0, 0) + @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test schema.inputs == (; system = nothing, prompt = docs[2]) # only the last doc is caught (serial execution) + @test schema.model_id == "llama2" + end +end + +@testset "not implemented ai* functions" begin + @test_throws ErrorException aiextract(OllamaManagedSchema(), "prompt") + @test_throws ErrorException aiclassify(OllamaManagedSchema(), "prompt") + @test_throws ErrorException aiscan(OllamaManagedSchema(), "prompt") +end \ No newline at end of file From 4a7ed993627bf1b20ab830dc416ddd3da29e7621 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 24 Nov 2023 09:29:32 +0000 Subject: [PATCH 015/251] update README and examples --- CHANGELOG.md | 2 +- README.md | 49 ++++++++++++++++++++++----- examples/working_with_ollama.jl | 60 +++++++++++++++++++++++++++++++++ src/llm_ollama_managed.jl | 2 +- 4 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 examples/working_with_ollama.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index b099f7309..4439adea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,4 +9,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). - Add `aiextract` function to extract structured information from text quickly and easily. See `?aiextract` for more information. - Add `aiscan` for image scanning (ie, image comprehension tasks). You can transcribe screenshots or reason over images as if they were text. Images can be provided either as a local file (`image_path`) or as an url (`image_url`). See `?aiscan` for more information. -- Add support for [Ollama.ai](https://ollama.ai/)'s local models. Only `aigenerate` and `aiembed` functions are supported. \ No newline at end of file +- Add support for [Ollama.ai](https://ollama.ai/)'s local models. Only `aigenerate` and `aiembed` functions are supported at the moment. \ No newline at end of file diff --git a/README.md b/README.md index 53b083114..166367c06 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,8 @@ For more practical examples, see the `examples/` folder and the [Advanced Exampl - [Configuring the Environment Variable for API Key](#configuring-the-environment-variable-for-api-key) - [Understanding the API Keyword Arguments in `aigenerate` (`api_kwargs`)](#understanding-the-api-keyword-arguments-in-aigenerate-api_kwargs) - [Instant Access from Anywhere](#instant-access-from-anywhere) + - [Open Source Alternatives](#open-source-alternatives) + - [Setup Guide for Ollama](#setup-guide-for-ollama) - [Roadmap](#roadmap) ## Why PromptingTools.jl @@ -355,14 +357,15 @@ msg.content |> Markdown.parse ## Using Ollama models -[Ollama.ai](https://ollama.ai/) is an amazingly simply tool that allows you to run several Large Language Models (LLM) on your computer. It's especially suitable when you're working with some sensitive data that should not be sent anywhere. +[Ollama.ai](https://ollama.ai/) is an amazingly simple tool that allows you to run several Large Language Models (LLM) on your computer. It's especially suitable when you're working with some sensitive data that should not be sent anywhere. -TODO: assumes it's working, but it's not yet! +Let's assume you have installed Ollama, downloaded a model, and it's running in the background. + +We can use it with the `aigenerate` function: -We can use Ollama models with the `aigenerate` function: ```julia const PT = PromptingTools -schema = PT.OllamaManagedSchema() +schema = PT.OllamaManagedSchema() # notice the different schema! msg = aigenerate(schema, "Say hi!"; model="openhermes2.5-mistral") # [ Info: Tokens: 69 in 0.9 seconds @@ -370,6 +373,7 @@ msg = aigenerate(schema, "Say hi!"; model="openhermes2.5-mistral") ``` And we can also use the `aiembed` function: + ```julia msg = aiembed(schema, "Embed me", copy; model="openhermes2.5-mistral") msg.content # 4096-element JSON3.Array{Float64... @@ -378,7 +382,7 @@ msg = aiembed(schema, ["Embed me", "Embed me"]; model="openhermes2.5-mistral") msg.content # 4096×2 Matrix{Float64}: ``` -TODO: Add FAQ how to setup Ollama.ai +If you're getting errors, check that Ollama is running - see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. ### More Examples @@ -447,9 +451,9 @@ Each new interface would be defined in a separate `llm_.jl` file. OpenAI's models are at the forefront of AI research and provide robust, state-of-the-art capabilities for many tasks. -There will be reasons when you do not or cannot use it (eg, privacy, cost, etc.). In that case, you can use local models (eg, Ollama) or other APIs (eg, Anthropic). +There will be situations not or cannot use it (eg, privacy, cost, etc.). In that case, you can use local models (eg, Ollama) or other APIs (eg, Anthropic). -Note: Tutorial for how to set up and use Ollama + PromptingTools.jl is coming! +Note: To get started with [Ollama.ai](https://ollama.ai/), see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. ### Data Privacy and OpenAI @@ -459,7 +463,10 @@ At the time of writing, OpenAI does NOT use the API calls for training their mod > > OpenAI does not use data submitted to and generated by our API to train OpenAI models or improve OpenAI’s service offering. In order to support the continuous improvement of our models, you can fill out this form to opt-in to share your data with us. -- [How your data is used to improve our models](https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance) +You can always double-check the latest information on the [OpenAI's How we use your data](https://platform.openai.com/docs/models/how-we-use-your-data) page. + Resources: +- [OpenAI's How we use your data](https://platform.openai.com/docs/models/how-we-use-your-data) - [Data usage for consumer services FAQ](https://help.openai.com/en/articles/7039943-data-usage-for-consumer-services-faq) - [How your data is used to improve our models](https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance) @@ -548,16 +555,40 @@ const PT = PromptingTools # to access unexported functions and types Now, you can just use `ai"Help me do X to achieve Y"` from any REPL session! +### Open Source Alternatives + +The ethos of PromptingTools.jl is to allow you to use whatever model you want, which includes Open Source LLMs. The most popular and easiest to setup is [Ollama.ai](https://ollama.ai/) - see below for more information. + +### Setup Guide for Ollama + +Ollama runs a background service hosting LLMs that you can access via a simple API. It's especially useful when you're working with some sensitive data that should not be sent anywhere. + +Installation is very easy, just download the latest version [here](https://ollama.ai/download). + +Once you've installed it, just launch the app and you're ready to go! + +To check if it's running, go to your browser and open `127.0.0.1:11434`. You should see the message "Ollama is running". +Alternatively, you can run `ollama serve` in your terminal and you'll get a message that it's already running. + +There are many models available in [Ollama Library](https://ollama.ai/library), including Llama2, CodeLlama, SQLCoder, or my personal favorite `openhermes2.5-mistral`. + +Download new models with `ollama pull ` (eg, `ollama pull openhermes2.5-mistral`). + +Show currently available models with `ollama list`. + +See [Ollama.ai](https://ollama.ai/) for more information. + ## Roadmap This is a list of features that I'd like to see in the future (in no particular order): - Document more mini-tasks, add tutorials -- Integration of new OpenAI capabilities (eg, vision, audio, assistants -> Imagine a function you send a Plot to and it will add code to add titles, labels, etc. and generate insights for your report!) -- Documented support for local models (eg, guide and prompt templates for Ollama) +- Integration of new OpenAI capabilities (eg, audio, assistants -> Imagine a function you send a Plot to and it will add code to add titles, labels, etc. and generate insights for your report!) - Add Preferences.jl mechanism to set defaults and persist them across sessions - More templates for common tasks (eg, fact-checking, sentiment analysis, extraction of entities/metadata, etc.) - Ability to easily add new templates, save them, and share them with others - Ability to easily trace and serialize the prompts & AI results for finetuning or evaluation in the future +- Add multi-turn conversations if you need to "reply" to the AI assistant + For more information, contributions, or questions, please visit the [PromptingTools.jl GitHub repository](https://github.com/svilupp/PromptingTools.jl). diff --git a/examples/working_with_ollama.jl b/examples/working_with_ollama.jl new file mode 100644 index 000000000..b4e524fd2 --- /dev/null +++ b/examples/working_with_ollama.jl @@ -0,0 +1,60 @@ +using PromptingTools +const PT = PromptingTools + +# Notice the schema change! If you want this to be the new default, you need to change `PT.PROMPT_SCHEMA` +schema = PT.OllamaManagedSchema() +# You can choose models from https://ollama.ai/library - I prefer `openhermes2.5-mistral` +model = "openhermes2.5-mistral" + +# # Text Generation with aigenerate + +# ## Simple message +msg = aigenerate(schema, "Say hi!"; model) + +# ## Standard string interpolation +a = 1 +msg = aigenerate(schema, "What is `$a+$a`?"; model) + +name = "John" +msg = aigenerate(schema, "Say hi to {{name}}."; name, model) + +# ## Advanced Prompts +conversation = [ + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] +msg = aigenerate(schema, conversation; model) + +# # Embeddings with aiembed + +# ## Simple embedding for one document +msg = aiembed(schema, "Embed me"; model) +msg.content + +# One document and we materialize the data into a Vector with copy (`postprocess` function argument) +msg = aiembed(schema, "Embed me", copy; model) +msg.content + +# ## Multiple documents embedding +# Multiple documents - embedded sequentially, you can get faster speed with async +msg = aiembed(schema, ["Embed me", "Embed me"]; model) +msg.content + +# You can use Threads.@spawn or asyncmap, whichever you prefer, to paralellize the model calls +docs = ["Embed me", "Embed me"] +tasks = asyncmap(docs) do doc + msg = aiembed(schema, doc; model) +end +embedding = mapreduce(x -> x.content, hcat, tasks) + +# ## Using postprocessing function +# Add normalization as postprocessing function to normalize embeddings on reception (for easy cosine similarity later) +using LinearAlgebra +schema = PT.OllamaManagedSchema() + +msg = aiembed(schema, + ["embed me", "and me too"], + LinearAlgebra.normalize; + model = "openhermes2.5-mistral") + +# Cosine similarity is then a simple multiplication +msg.content' * msg.content[:, 1] # [1.0, 0.34] \ No newline at end of file diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 83fa85998..5a0c46df5 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -89,7 +89,7 @@ function ollama_api(prompt_schema::AbstractOllamaManagedSchema, prompt::Abstract [],# no headers JSON3.write(body); http_kwargs...) body = JSON3.read(resp.body) - return (; response, resp.status) + return (; response = body, resp.status) end # For testing function ollama_api(prompt_schema::TestEchoOllamaManagedSchema, prompt::AbstractString; From 1373f877f6c9d5102f5554b6d079569b7ae0a141 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 24 Nov 2023 09:32:10 +0000 Subject: [PATCH 016/251] add install note --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 166367c06..cc95b7d2a 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,13 @@ Getting started with PromptingTools.jl is as easy as importing the package and u Note: You will need to set your OpenAI API key as an environment variable before using PromptingTools.jl (see the [Creating OpenAI API Key](#creating-openai-api-key) section below). For a quick start, simply set it via `ENV["OPENAI_API_KEY"] = "your-api-key"` +Install PromptingTools.jl: +```julia +using Pkg +Pkg.add("https://github.com/svilupp/PromptingTools.jl") +``` + +And we're ready to go! ```julia using PromptingTools From 5f2dec8e8fb6d92543d6e3395be6523d609cbca1 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 24 Nov 2023 19:59:12 +0000 Subject: [PATCH 017/251] Create docs from the README file --- README.md | 6 +- docs/Project.toml | 2 + docs/generate_examples.jl | 10 + docs/make.jl | 8 + docs/src/examples/readme_examples.md | 289 ++ docs/src/examples/working_with_aitemplates.md | 138 + docs/src/examples/working_with_ollama.md | 4255 +++++++++++++++++ docs/src/frequently_asked_questions.md | 132 + docs/src/getting_started.md | 91 + docs/src/index.md | 23 +- docs/src/reference.md | 8 + examples/working_with_aitemplates.jl | 40 +- examples/working_with_ollama.jl | 29 +- 13 files changed, 4985 insertions(+), 46 deletions(-) create mode 100644 docs/generate_examples.jl create mode 100644 docs/src/examples/readme_examples.md create mode 100644 docs/src/examples/working_with_aitemplates.md create mode 100644 docs/src/examples/working_with_ollama.md create mode 100644 docs/src/frequently_asked_questions.md create mode 100644 docs/src/getting_started.md create mode 100644 docs/src/reference.md diff --git a/README.md b/README.md index cc95b7d2a..40d4c9421 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Getting started with PromptingTools.jl is as easy as importing the package and u Note: You will need to set your OpenAI API key as an environment variable before using PromptingTools.jl (see the [Creating OpenAI API Key](#creating-openai-api-key) section below). For a quick start, simply set it via `ENV["OPENAI_API_KEY"] = "your-api-key"` -Install PromptingTools.jl: +Install PromptingTools: ```julia using Pkg Pkg.add("https://github.com/svilupp/PromptingTools.jl") @@ -75,7 +75,7 @@ For more practical examples, see the `examples/` folder and the [Advanced Exampl - [Classification](#classification) - [Data Extraction](#data-extraction) - [OCR and Image Comprehension](#ocr-and-image-comprehension) - - [Using Ollama models](#using-ollama-models) + - [Using Ollama models](#using-ollama-models) - [More Examples](#more-examples) - [Package Interface](#package-interface) - [Frequently Asked Questions](#frequently-asked-questions) @@ -362,7 +362,7 @@ using Markdown msg.content |> Markdown.parse ``` -## Using Ollama models +### Using Ollama models [Ollama.ai](https://ollama.ai/) is an amazingly simple tool that allows you to run several Large Language Models (LLM) on your computer. It's especially suitable when you're working with some sensitive data that should not be sent anywhere. diff --git a/docs/Project.toml b/docs/Project.toml index afae62a78..b0857977a 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,5 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589" PromptingTools = "670122d1-24a8-4d70-bfce-740807c42192" diff --git a/docs/generate_examples.jl b/docs/generate_examples.jl new file mode 100644 index 000000000..f48c298ea --- /dev/null +++ b/docs/generate_examples.jl @@ -0,0 +1,10 @@ +using Literate + +## ! Config +example_files = joinpath(@__DIR__, "..", "examples") |> x -> readdir(x; join = true) +output_dir = joinpath(@__DIR__, "src", "examples") + +# Run the production loop +for fn in example_files + Literate.markdown(fn, output_dir; execute = true) +end \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl index c2e3db45d..d3e676100 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -19,6 +19,14 @@ makedocs(; assets = String[]), pages = [ "Home" => "index.md", + "Getting Started" => "getting_started.md", + "Examples" => [ + "Various examples" => "examples/readme_examples.md", + "Using AITemplates" => "examples/working_with_aitemplates.md", + "Local models with Ollama.ai" => "examples/working_with_ollama.md", + ], + "F.A.Q." => "frequently_asked_questions.md", + "Reference" => "reference.md", ]) deploydocs(; diff --git a/docs/src/examples/readme_examples.md b/docs/src/examples/readme_examples.md new file mode 100644 index 000000000..5e2f35ca3 --- /dev/null +++ b/docs/src/examples/readme_examples.md @@ -0,0 +1,289 @@ +# Various Examples + +Noteworthy functions: `aigenerate`, `aiembed`, `aiclassify`, `aiextract`, `aitemplates` + +## Seamless Integration Into Your Workflow +Google search is great, but it's a context switch. You often have to open a few pages and read through the discussion to find the answer you need. Same with the ChatGPT website. + +Imagine you are in VSCode, editing your `.gitignore` file. How do I ignore a file in all subfolders again? + +All you need to do is to type: +`aai"What to write in .gitignore to ignore file XYZ in any folder or subfolder?"` + +With `aai""` (as opposed to `ai""`), we make a non-blocking call to the LLM to not prevent you from continuing your work. When the answer is ready, we log it from the background: + +```plaintext +[ Info: Tokens: 102 @ Cost: $0.0002 in 2.7 seconds +┌ Info: AIMessage> To ignore a file called "XYZ" in any folder or subfolder, you can add the following line to your .gitignore file: +│ +│ ``` +│ **/XYZ +│ ``` +│ +└ This pattern uses the double asterisk (`**`) to match any folder or subfolder, and then specifies the name of the file you want to ignore. +``` + +You probably saved 3-5 minutes on this task and probably another 5-10 minutes, because of the context switch/distraction you avoided. It's a small win, but it adds up quickly. + +## Advanced Prompts / Conversations + +You can use the `aigenerate` function to replace handlebar variables (eg, `{{name}}`) via keyword arguments. + +```julia +msg = aigenerate("Say hello to {{name}}!", name="World") +``` + +The more complex prompts are effectively a conversation (a set of messages), where you can have messages from three entities: System, User, AI Assistant. We provide the corresponding types for each of them: `SystemMessage`, `UserMessage`, `AIMessage`. + +```julia +using PromptingTools: SystemMessage, UserMessage + +conversation = [ + SystemMessage("You're master Yoda from Star Wars trying to help the user become a Jedi."), + UserMessage("I have feelings for my {{object}}. What should I do?")] +msg = aigenerate(conversation; object = "old iPhone") +``` + +```plaintext +AIMessage("Ah, a dilemma, you have. Emotional attachment can cloud your path to becoming a Jedi. To be attached to material possessions, you must not. The iPhone is but a tool, nothing more. Let go, you must. + +Seek detachment, young padawan. Reflect upon the impermanence of all things. Appreciate the memories it gave you, and gratefully part ways. In its absence, find new experiences to grow and become one with the Force. Only then, a true Jedi, you shall become.") +``` + +You can also use it to build conversations, eg, +```julia +new_conversation = vcat(conversation...,msg, UserMessage("Thank you, master Yoda! Do you have {{object}} to know what it feels like?")) +aigenerate(new_conversation; object = "old iPhone") +``` + +```plaintext +> AIMessage("Hmm, possess an old iPhone, I do not. But experience with attachments, I have. Detachment, I learned. True power and freedom, it brings...") +``` + +## Templated Prompts + +With LLMs, the quality / robustness of your results depends on the quality of your prompts. But writing prompts is hard! That's why we offer a templating system to save you time and effort. + +To use a specific template (eg, `` to ask a Julia language): +```julia +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +``` + +The above is equivalent to a more verbose version that explicitly uses the dispatch on `AITemplate`: +```julia +msg = aigenerate(AITemplate(:JuliaExpertAsk); ask = "How do I add packages?") +``` + +Find available templates with `aitemplates`: +```julia +tmps = aitemplates("JuliaExpertAsk") +# Will surface one specific template +# 1-element Vector{AITemplateMetadata}: +# PromptingTools.AITemplateMetadata +# name: Symbol JuliaExpertAsk +# description: String "For asking questions about Julia language. Placeholders: `ask`" +# version: String "1" +# wordcount: Int64 237 +# variables: Array{Symbol}((1,)) +# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" +# user_preview: String "# Question\n\n{{ask}}" +# source: String "" +``` +The above gives you a good idea of what the template is about, what placeholders are available, and how much it would cost to use it (=wordcount). + +Search for all Julia-related templates: +```julia +tmps = aitemplates("Julia") +# 2-element Vector{AITemplateMetadata}... -> more to come later! +``` + +If you are on VSCode, you can leverage a nice tabular display with `vscodedisplay`: +```julia +using DataFrames +tmps = aitemplates("Julia") |> DataFrame |> vscodedisplay +``` + +I have my selected template, how do I use it? Just use the "name" in `aigenerate` or `aiclassify` + like you see in the first example! + +You can inspect any template by "rendering" it (this is what the LLM will see): +```julia +julia> AITemplate(:JudgeIsItTrue) |> PromptingTools.render +``` + +See more examples in the [Examples](https://github.com/svilupp/PromptingTools.jl/tree/main/examples) folder. + +## Asynchronous Execution + +You can leverage `asyncmap` to run multiple AI-powered tasks concurrently, improving performance for batch operations. + +```julia +prompts = [aigenerate("Translate 'Hello, World!' to {{language}}"; language) for language in ["Spanish", "French", "Mandarin"]] +responses = asyncmap(aigenerate, prompts) +``` + +Pro tip: You can limit the number of concurrent tasks with the keyword `asyncmap(...; ntasks=10)`. + +## Model Aliases + +Certain tasks require more powerful models. All user-facing functions have a keyword argument `model` that can be used to specify the model to be used. For example, you can use `model = "gpt-4-1106-preview"` to use the latest GPT-4 Turbo model. However, no one wants to type that! + +We offer a set of model aliases (eg, "gpt3", "gpt4", "gpt4t" -> the above GPT-4 Turbo, etc.) that can be used instead. + +Each `ai...` call first looks up the provided model name in the dictionary `PromptingTools.MODEL_ALIASES`, so you can easily extend with your own aliases! + +```julia +const PT = PromptingTools +PT.MODEL_ALIASES["gpt4t"] = "gpt-4-1106-preview" +``` + +These aliases also can be used as flags in the `@ai_str` macro, eg, `ai"What is the capital of France?"gpt4t` (GPT-4 Turbo has a knowledge cut-off in April 2023, so it's useful for more contemporary questions). + +## Embeddings + +Use the `aiembed` function to create embeddings via the default OpenAI model that can be used for semantic search, clustering, and more complex AI workflows. + +```julia +text_to_embed = "The concept of artificial intelligence." +msg = aiembed(text_to_embed) +embedding = msg.content # 1536-element Vector{Float64} +``` + +If you plan to calculate the cosine distance between embeddings, you can normalize them first: +```julia +using LinearAlgebra +msg = aiembed(["embed me", "and me too"], LinearAlgebra.normalize) + +# calculate cosine distance between the two normalized embeddings as a simple dot product +msg.content' * msg.content[:, 1] # [1.0, 0.787] +``` + +## Classification + +You can use the `aiclassify` function to classify any provided statement as true/false/unknown. This is useful for fact-checking, hallucination or NLI checks, moderation, filtering, sentiment analysis, feature engineering and more. + +```julia +aiclassify("Is two plus two four?") +# true +``` + +System prompts and higher-quality models can be used for more complex tasks, including knowing when to defer to a human: + +```julia +aiclassify(:JudgeIsItTrue; it = "Is two plus three a vegetable on Mars?", model = "gpt4t") +# unknown +``` + +In the above example, we used a prompt template `:JudgeIsItTrue`, which automatically expands into the following system prompt (and a separate user prompt): + +> "You are an impartial AI judge evaluating whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide." + +For more information on templates, see the [Templated Prompts](#templated-prompts) section. + +## Data Extraction + +Are you tired of extracting data with regex? You can use LLMs to extract structured data from text! + +All you have to do is to define the structure of the data you want to extract and the LLM will do the rest. + +Define a `return_type` with struct. Provide docstrings if needed (improves results and helps with documentation). + +Let's start with a hard task - extracting the current weather in a given location: +```julia +@enum TemperatureUnits celsius fahrenheit +"""Extract the current weather in a given location + +# Arguments +- `location`: The city and state, e.g. "San Francisco, CA" +- `unit`: The unit of temperature to return, either `celsius` or `fahrenheit` +""" +struct CurrentWeather + location::String + unit::Union{Nothing,TemperatureUnits} +end + +# Note that we provide the TYPE itself, not an instance of it! +msg = aiextract("What's the weather in Salt Lake City in C?"; return_type=CurrentWeather) +msg.content +# CurrentWeather("Salt Lake City, UT", celsius) +``` + +But you can use it even for more complex tasks, like extracting many entities from a text: + +```julia +"Person's age, height, and weight." +struct MyMeasurement + age::Int + height::Union{Int,Nothing} + weight::Union{Nothing,Float64} +end +struct ManyMeasurements + measurements::Vector{MyMeasurement} +end +msg = aiextract("James is 30, weighs 80kg. He's 180cm tall. Then Jack is 19 but really tall - over 190!"; return_type=ManyMeasurements) +msg.content.measurements +# 2-element Vector{MyMeasurement}: +# MyMeasurement(30, 180, 80.0) +# MyMeasurement(19, 190, nothing) +``` + +There is even a wrapper to help you catch errors together with helpful explanations on why parsing failed. See `?PromptingTools.MaybeExtract` for more information. + +## OCR and Image Comprehension + +With the `aiscan` function, you can interact with images as if they were text. + +You can simply describe a provided image: +```julia +msg = aiscan("Describe the image"; image_path="julia.png", model="gpt4v") +# [ Info: Tokens: 1141 @ Cost: \$0.0117 in 2.2 seconds +# AIMessage("The image shows a logo consisting of the word "julia" written in lowercase") +``` + +Or you can do an OCR of a screenshot. +Let's transcribe some SQL code from a screenshot (no more re-typing!), we use a template `:OCRTask`: + +```julia +# Screenshot of some SQL code +image_url = "https://www.sqlservercentral.com/wp-content/uploads/legacy/8755f69180b7ac7ee76a69ae68ec36872a116ad4/24622.png" +msg = aiscan(:OCRTask; image_url, model="gpt4v", task="Transcribe the SQL code in the image.", api_kwargs=(; max_tokens=2500)) + +# [ Info: Tokens: 362 @ Cost: \$0.0045 in 2.5 seconds +# AIMessage("```sql +# update Orders +``` + +You can add syntax highlighting of the outputs via Markdown +```julia +using Markdown +msg.content |> Markdown.parse +``` + +## Using Ollama models + +[Ollama.ai](https://ollama.ai/) is an amazingly simple tool that allows you to run several Large Language Models (LLM) on your computer. It's especially suitable when you're working with some sensitive data that should not be sent anywhere. + +Let's assume you have installed Ollama, downloaded a model, and it's running in the background. + +We can use it with the `aigenerate` function: + +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() # notice the different schema! + +msg = aigenerate(schema, "Say hi!"; model="openhermes2.5-mistral") +# [ Info: Tokens: 69 in 0.9 seconds +# AIMessage("Hello! How can I assist you today?") +``` + +And we can also use the `aiembed` function: + +```julia +msg = aiembed(schema, "Embed me", copy; model="openhermes2.5-mistral") +msg.content # 4096-element JSON3.Array{Float64... + +msg = aiembed(schema, ["Embed me", "Embed me"]; model="openhermes2.5-mistral") +msg.content # 4096×2 Matrix{Float64}: +``` + +If you're getting errors, check that Ollama is running - see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. \ No newline at end of file diff --git a/docs/src/examples/working_with_aitemplates.md b/docs/src/examples/working_with_aitemplates.md new file mode 100644 index 000000000..d217ad0c1 --- /dev/null +++ b/docs/src/examples/working_with_aitemplates.md @@ -0,0 +1,138 @@ +```@meta +EditURL = "../../../examples/working_with_aitemplates.jl" +``` + +# Using AITemplates + +This file contains examples of how to work with AITemplate(s). + +First, let's import the package and define a helper link for calling un-exported functions: + +````julia +using PromptingTools +const PT = PromptingTools +```` + +```` +PromptingTools +```` + +LLM responses are only as good as the prompts you give them. However, great prompts take long time to write -- AITemplate are a way to re-use great prompts! + +AITemplates are just a collection of templated prompts (ie, set of "messages" that have placeholders like {{question}}) + +They are saved as JSON files in the `templates` directory. +They are automatically loaded on package import, but you can always force a re-load with `PT.load_templates!()` + +````julia +PT.load_templates!(); +```` + +You can (create them) and use them for any ai* function instead of a prompt: +Let's use a template called `:JuliaExpertAsk` +alternatively, you can use `AITemplate(:JuliaExpertAsk)` for cleaner dispatch + +````julia +msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") +```` + +```` +AIMessage("To add packages in Julia, you can use the built-in package manager called `Pkg`. Here are the steps: + +1. Open the Julia REPL (Read-Eval-Print Loop). +2. Press the `]` key to enter the package manager mode. +3. Use the `add` command followed by the name of the package you want to install. For example, to install the `DataFrames` package, type: `add DataFrames`. +4. Press the `backspace` or `ctrl + C` key to exit the package manager mode and return to the REPL. + +After following these steps, the specified package will be installed and available for use in your Julia environment.") +```` + +You can see that it had a placeholder for the actual question (`ask`) that we provided as a keyword argument. +We did not have to write any system prompt for personas, tone, etc. -- it was all provided by the template! + +How to know which templates are available? You can search for them with `aitemplates()`: +You can search by Symbol (only for partial name match), String (partial match on name or description), or Regex (more fields) + +````julia +tmps = aitemplates("JuliaExpertAsk") +```` + +```` +1-element Vector{AITemplateMetadata}: +PromptingTools.AITemplateMetadata + name: Symbol JuliaExpertAsk + description: String "For asking questions about Julia language. Placeholders: `ask`" + version: String "1" + wordcount: Int64 237 + variables: Array{Symbol}((1,)) + system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" + user_preview: String "# Question\n\n{{ask}}" + source: String "" + + +```` + +You can see that it outputs a list of available templates that match the search - there is just one in this case. + +Moreover, it shows not just the description, but also a preview of the actual prompts, placeholders available, and the length (to gauge how much it would cost). + +If you use VSCode, you can display them in a nice scrollable table with `vscodedisplay`: +```plaintext +using DataFrames +DataFrame(tmp) |> vscodedisplay +``` + +You can also just `render` the template to see the underlying mesages: + +````julia +msgs = PT.render(AITemplate(:JuliaExpertAsk)) +```` + +```` +2-element Vector{PromptingTools.AbstractChatMessage}: + SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) +```` + +Now, you know exactly what's in the template! + +If you want to modify it, simply change it and save it as a new file with `save_template` (see the docs `?save_template` for more details). + +Let's adjust the previous template to be more specific to a data analysis question: + +````julia +tpl = [PT.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. You're also a senior Data Scientist and proficient in data analysis in Julia. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + PT.UserMessage("# Question\n\n{{ask}}")] +```` + +```` +2-element Vector{PromptingTools.AbstractChatMessage}: + SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. You're also a senior Data Scientist and proficient in data analysis in Julia. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) +```` + +Templates are saved in the `templates` directory of the package. Name of the file will become the template name (eg, call `:JuliaDataExpertAsk`) + +````julia +filename = joinpath(pkgdir(PromptingTools), + "templates", + "persona-task", + "JuliaDataExpertAsk_123.json") +PT.save_template(filename, + tpl; + description = "For asking data analysis questions in Julia language. Placeholders: `ask`") +rm(filename) # cleanup if we don't like it +```` + +When you create a new template, remember to re-load the templates with `load_templates!()` so that it's available for use. + +````julia +PT.load_templates!(); +```` + +!!! If you have some good templates (or suggestions for the existing ones), please consider sharing them with the community by opening a PR to the `templates` directory! + +--- + +*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* + diff --git a/docs/src/examples/working_with_ollama.md b/docs/src/examples/working_with_ollama.md new file mode 100644 index 000000000..b4aeea77a --- /dev/null +++ b/docs/src/examples/working_with_ollama.md @@ -0,0 +1,4255 @@ +```@meta +EditURL = "../../../examples/working_with_ollama.jl" +``` + +# Local models with Ollama.ai + +This file contains examples of how to work with [Ollama.ai](https://ollama.ai/) models. +It assumes that you've already installated and launched the Ollama server. For more details or troubleshooting advice, see the [Frequently Asked Questions](@ref) section. + +First, let's import the package and define a helper link for calling un-exported functions: + +````julia +using PromptingTools +const PT = PromptingTools +```` + +```` +PromptingTools +```` + +Notice the schema change! If you want this to be the new default, you need to change `PT.PROMPT_SCHEMA` + +````julia +schema = PT.OllamaManagedSchema() +```` + +```` +OllamaManagedSchema() +```` + +You can choose models from https://ollama.ai/library - I prefer `openhermes2.5-mistral` + +````julia +model = "openhermes2.5-mistral" +```` + +```` +"openhermes2.5-mistral" +```` + +## Text Generation with aigenerate + +### Simple message + +````julia +msg = aigenerate(schema, "Say hi!"; model) +```` + +```` +AIMessage("Hi there! How can I help you today? If you have any questions or need assistance, please feel free to ask.") +```` + +### Standard string interpolation + +````julia +a = 1 +msg = aigenerate(schema, "What is `$a+$a`?"; model) + +name = "John" +msg = aigenerate(schema, "Say hi to {{name}}."; name, model) +```` + +```` +AIMessage("Hi there, John! It's great to see you today. How can I assist you? If you have any questions or need help with something, please don't hesitate to ask!") +```` + +### Advanced Prompts + +````julia +conversation = [ + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] +msg = aigenerate(schema, conversation; model) +```` + +```` +AIMessage("Strong your feelings are, but attachments lead to suffering they often do. Focus on the balance in all things and let go of possessions that cloud your judgment. Embrace the wisdom of the Force and understand that material objects are not the same as love. The Force will guide you.") +```` + +## Embeddings with aiembed + +### Simple embedding for one document + +````julia +msg = aiembed(schema, "Embed me"; model) # access msg.content +```` + +```` +DataMessage(JSON3.Array{Float64, Vector{UInt8}, SubArray{UInt64, 1, Vector{UInt64}, Tuple{UnitRange{Int64}}, true}} of size (4096,)) +```` + +One document and we materialize the data into a Vector with copy (`postprocess` function argument) + +````julia +msg = aiembed(schema, "Embed me", copy; model) +```` + +```` +DataMessage(Vector{Float64} of size (4096,)) +```` + +### Multiple documents embedding +Multiple documents - embedded sequentially, you can get faster speed with async + +````julia +msg = aiembed(schema, ["Embed me", "Embed me"]; model) +```` + +```` +DataMessage(Matrix{Float64} of size (4096, 2)) +```` + +You can use Threads.@spawn or asyncmap, whichever you prefer, to paralellize the model calls + +````julia +docs = ["Embed me", "Embed me"] +tasks = asyncmap(docs) do doc + msg = aiembed(schema, doc; model) +end +embedding = mapreduce(x -> x.content, hcat, tasks) +```` + +```` +4096×2 Matrix{Float64}: + 7.71459 7.71459 + -1.14532 -1.14532 + 2.90205 2.90205 + -4.01967 -4.01967 + -7.73098 -7.73098 + 8.02114 8.02114 + -6.01313 -6.01313 + -2.06712 -2.06712 + 4.97633 4.97633 + -9.69502 -9.69502 + -0.02567 -0.02567 + 8.09622 8.09622 + 6.54008 6.54008 + -5.70348 -5.70348 + 2.55213 2.55213 + -2.00164 -2.00164 + -2.21854 -2.21854 + -3.6568 -3.6568 + 3.97905 3.97905 + -1.79931 -1.79931 + 0.0769786 0.0769786 + -10.4355 -10.4355 + -3.92487 -3.92487 + -6.03455 -6.03455 + -2.8005 -2.8005 + 2.23584 2.23584 + -0.503125 -0.503125 + 1.99538 1.99538 + -0.283642 -0.283642 + -0.414273 -0.414273 + 8.72909 8.72909 + 2.6071 2.6071 + 0.0808531 0.0808531 + -1.83914 -1.83914 + 2.19998 2.19998 + -0.629226 -0.629226 + 3.74217 3.74217 + 1.71231 1.71231 + -0.742473 -0.742473 + 2.9234 2.9234 + 7.33933 7.33933 + 4.24576 4.24576 + -7.56434 -7.56434 + -1.22274 -1.22274 + 1.73444 1.73444 + -0.736801 -0.736801 + 1.30149 1.30149 + -6.91642 -6.91642 + -1.84513 -1.84513 + 1.69959 1.69959 + 5.74253 5.74253 + 1.48734 1.48734 + -1.45199 -1.45199 + -18.5026 -18.5026 + -8.61009 -8.61009 + -2.21845 -2.21845 + -4.22932 -4.22932 + 6.0436 6.0436 + -1.8824 -1.8824 + -0.689965 -0.689965 + 0.845927 0.845927 + -1.99517 -1.99517 + 9.32292 9.32292 + 6.24938 6.24938 + -4.59894 -4.59894 + 6.24579 6.24579 + -5.8733 -5.8733 + -4.60285 -4.60285 + -1.27596 -1.27596 + -1.68807 -1.68807 + -0.391147 -0.391147 + -2.68362 -2.68362 + 1.99197 1.99197 + 0.0812396 0.0812396 + -3.79761 -3.79761 + -8.5693 -8.5693 + -0.869305 -0.869305 + -0.77582 -0.77582 + -4.76995 -4.76995 + 1.9712 1.9712 + 4.74459 4.74459 + -4.31244 -4.31244 + 3.94876 3.94876 + -11.0882 -11.0882 + 9.38629 9.38629 + 10.2995 10.2995 + 2.40846 2.40846 + -3.91429 -3.91429 + -0.745707 -0.745707 + 4.31946 4.31946 + 8.34836 8.34836 + 0.857636 0.857636 + 1.66563 1.66563 + -11.1522 -11.1522 + -3.48353 -3.48353 + -6.08336 -6.08336 + 1.22086 1.22086 + -2.81636 -2.81636 + 1.07224 1.07224 + -8.24909 -8.24909 + 3.66474 3.66474 + -0.260558 -0.260558 + 2.38779 2.38779 + -4.00576 -4.00576 + 1.3949 1.3949 + -5.43468 -5.43468 + 4.08836 4.08836 + -1.1134 -1.1134 + -2.05916 -2.05916 + -9.78987 -9.78987 + -2.86149 -2.86149 + 5.54577 5.54577 + -1.96682 -1.96682 + 9.70577 9.70577 + -4.0553 -4.0553 + 8.54535 8.54535 + 0.539438 0.539438 + 4.61091 4.61091 + -5.32208 -5.32208 + -0.256733 -0.256733 + 4.74966 4.74966 + -2.46464 -2.46464 + -0.223077 -0.223077 + 1.84442 1.84442 + 6.42329 6.42329 + 0.431667 0.431667 + -8.42777 -8.42777 + -10.691 -10.691 + 3.023 3.023 + -5.65345 -5.65345 + -4.17833 -4.17833 + 0.937893 0.937893 + -6.99405 -6.99405 + -4.55107 -4.55107 + -15.3169 -15.3169 + -2.08895 -2.08895 + 7.17826 7.17826 + -4.26108 -4.26108 + -3.2712 -3.2712 + 16.1561 16.1561 + 13.5164 13.5164 + -5.91778 -5.91778 + 6.3401 6.3401 + 12.7018 12.7018 + 2.04305 2.04305 + 3.81683 3.81683 + -1.39969 -1.39969 + -0.17249 -0.17249 + -16.3687 -16.3687 + 4.3827 4.3827 + 2.58974 2.58974 + -4.75363 -4.75363 + 3.36371 3.36371 + 0.986534 0.986534 + -13.4299 -13.4299 + -12.7188 -12.7188 + 2.83107 2.83107 + -3.41115 -3.41115 + -3.01015 -3.01015 + 6.40446 6.40446 + -0.186923 -0.186923 + -1.42502 -1.42502 + 2.85606 2.85606 + -0.579786 -0.579786 + -3.92704 -3.92704 + 8.28959 8.28959 + 5.42878 5.42878 + 5.71589 5.71589 + -6.78065 -6.78065 + -0.403687 -0.403687 + -1.20623 -1.20623 + 4.92372 4.92372 + -1.69266 -1.69266 + -0.103872 -0.103872 + 1.9163 1.9163 + -2.26831 -2.26831 + -7.64622 -7.64622 + 1.02228 1.02228 + 2.91952 2.91952 + -0.524167 -0.524167 + 12.4803 12.4803 + 7.36984 7.36984 + -7.46027 -7.46027 + -2.78773 -2.78773 + 2.68293 2.68293 + -0.320891 -0.320891 + 7.12037 7.12037 + 3.02726 3.02726 + -2.68363 -2.68363 + 4.78372 4.78372 + 3.68899 3.68899 + 2.08839 2.08839 + 3.1873 3.1873 + -6.10744 -6.10744 + 10.5419 10.5419 + 6.29439 6.29439 + -9.41221 -9.41221 + -2.50548 -2.50548 + -1.14 -1.14 + -3.0203 -3.0203 + -1.73182 -1.73182 + -0.97194 -0.97194 + -6.69084 -6.69084 + -1.08986 -1.08986 + -3.83631 -3.83631 + 2.2775 2.2775 + -6.91276 -6.91276 + 2.4557 2.4557 + -0.477723 -0.477723 + -4.10405 -4.10405 + -3.91437 -3.91437 + -7.79672 -7.79672 + -6.19691 -6.19691 + 0.356732 0.356732 + 0.609725 0.609725 + -3.08225 -3.08225 + 6.39968 6.39968 + 1.30207 1.30207 + 7.36038 7.36038 + -7.7581 -7.7581 + -6.303 -6.303 + 0.348147 0.348147 + -8.38124 -8.38124 + 8.68524 8.68524 + -0.873688 -0.873688 + 1.19612 1.19612 + 0.725645 0.725645 + -6.59284 -6.59284 + -6.59079 -6.59079 + 1.03175 1.03175 + -0.236469 -0.236469 + 5.01671 5.01671 + 0.752329 0.752329 + 5.39971 5.39971 + 0.826802 0.826802 + 9.38285 9.38285 + 5.85717 5.85717 + 1.71145 1.71145 + -1.36528 -1.36528 + -5.09575 -5.09575 + 7.23996 7.23996 + 12.7272 12.7272 + 2.86673 2.86673 + 2.86546 2.86546 + 1.2423 1.2423 + 6.05857 6.05857 + 9.40879 9.40879 + 1.47573 1.47573 + 8.19025 8.19025 + 12.5009 12.5009 + -4.57244 -4.57244 + -0.674127 -0.674127 + 0.416418 0.416418 + -5.23336 -5.23336 + -0.771443 -0.771443 + 4.72784 4.72784 + -4.9684 -4.9684 + 4.75989 4.75989 + 1.68141 1.68141 + -3.2264 -3.2264 + 2.67195 2.67195 + 0.424227 0.424227 + 3.5195 3.5195 + 2.22441 2.22441 + -2.4856 -2.4856 + 8.03468 8.03468 + 8.54339 8.54339 + 3.83506 3.83506 + 13.5693 13.5693 + 2.44909 2.44909 + 2.70572 2.70572 + 6.13746 6.13746 + 1.26651 1.26651 + 8.25694 8.25694 + -3.59258 -3.59258 + 3.77765 3.77765 + -0.144755 -0.144755 + 3.15706 3.15706 + -2.3952 -2.3952 + 9.82079 9.82079 + 8.94186 8.94186 + -1.83071 -1.83071 + 1.45764 1.45764 + -11.8258 -11.8258 + -0.737553 -0.737553 + -1.2382 -1.2382 + 1.83341 1.83341 + -2.75977 -2.75977 + 3.75117 3.75117 + 6.04452 6.04452 + -4.40271 -4.40271 + -8.82336 -8.82336 + 10.8513 10.8513 + -4.91857 -4.91857 + -5.7401 -5.7401 + 7.22234 7.22234 + 7.15112 7.15112 + 1.81187 1.81187 + 8.19917 8.19917 + 2.91605 2.91605 + 3.82883 3.82883 + -0.208109 -0.208109 + 1.33796 1.33796 + 5.69606 5.69606 + -2.19266 -2.19266 + -5.91177 -5.91177 + 7.25269 7.25269 + -8.65987 -8.65987 + -3.47799 -3.47799 + -10.4904 -10.4904 + -0.00963959 -0.00963959 + -6.81662 -6.81662 + -2.05566 -2.05566 + 2.10144 2.10144 + 2.58138 2.58138 + 2.03289 2.03289 + -6.43532 -6.43532 + -2.97225 -2.97225 + -4.71142 -4.71142 + 4.97199 4.97199 + 3.687 3.687 + 1.8587 1.8587 + -0.444899 -0.444899 + -1.05556 -1.05556 + 4.15926 4.15926 + 5.48777 5.48777 + 2.28346 2.28346 + -4.69401 -4.69401 + 1.8873 1.8873 + -2.62671 -2.62671 + 1.4144 1.4144 + -2.97535 -2.97535 + 0.759131 0.759131 + 5.75781 5.75781 + -5.13309 -5.13309 + 1.72701 1.72701 + 2.96653 2.96653 + -10.8087 -10.8087 + 1.07262 1.07262 + -5.80018 -5.80018 + 1.90592 1.90592 + -5.42958 -5.42958 + 8.74889 8.74889 + -3.19785 -3.19785 + -2.7096 -2.7096 + 7.44399 7.44399 + -8.7433 -8.7433 + 11.6667 11.6667 + 2.59703 2.59703 + 4.22273 4.22273 + -4.68793 -4.68793 + -4.44601 -4.44601 + -0.57319 -0.57319 + 6.63389 6.63389 + -9.14857 -9.14857 + -1.34147 -1.34147 + 7.78513 7.78513 + -4.87331 -4.87331 + -5.06022 -5.06022 + 3.13076 3.13076 + -3.49373 -3.49373 + 3.12637 3.12637 + 0.566696 0.566696 + 4.99319 4.99319 + 3.57986 3.57986 + 0.607679 0.607679 + 2.37633 2.37633 + 0.35097 0.35097 + 0.239089 0.239089 + -6.51449 -6.51449 + -3.18838 -3.18838 + 0.770256 0.770256 + 2.09481 2.09481 + 5.36062 5.36062 + -5.25216 -5.25216 + -6.9523 -6.9523 + 3.97384 3.97384 + 8.7784 8.7784 + -3.91837 -3.91837 + -9.08965 -9.08965 + -1.17883 -1.17883 + -4.21353 -4.21353 + -5.0915 -5.0915 + 3.74499 3.74499 + -4.39715 -4.39715 + 2.13732 2.13732 + 5.97568 5.97568 + 1.11809 1.11809 + -3.93191 -3.93191 + -1.39764 -1.39764 + -4.23595 -4.23595 + 0.103914 0.103914 + -2.34387 -2.34387 + -4.95433 -4.95433 + 3.58645 3.58645 + 0.818317 0.818317 + 6.23266 6.23266 + -5.62973 -5.62973 + -7.45604 -7.45604 + 1.29222 1.29222 + 0.327714 0.327714 + 5.31996 5.31996 + -2.23663 -2.23663 + 0.058689 0.058689 + -0.74368 -0.74368 + -1.20749 -1.20749 + -4.75414 -4.75414 + 2.10011 2.10011 + -6.86479 -6.86479 + 1.58403 1.58403 + 0.0492497 0.0492497 + 0.32083 0.32083 + -3.11682 -3.11682 + 4.61797 4.61797 + -0.399561 -0.399561 + -7.89927 -7.89927 + -0.659676 -0.659676 + -2.2416 -2.2416 + 0.933026 0.933026 + 1.98848 1.98848 + -2.14547 -2.14547 + -1.10747 -1.10747 + 8.90983 8.90983 + -3.84128 -3.84128 + 9.82771 9.82771 + 3.02843 3.02843 + 3.26396 3.26396 + 6.75629 6.75629 + 0.0290972 0.0290972 + 7.92768 7.92768 + 7.44608 7.44608 + -4.14083 -4.14083 + -1.39636 -1.39636 + 2.87656 2.87656 + 3.87446 3.87446 + 0.112521 0.112521 + -3.3429 -3.3429 + -6.85823 -6.85823 + 1.18408 1.18408 + 3.53175 3.53175 + 3.56147 3.56147 + 5.41961 5.41961 + -1.5263 -1.5263 + 3.05559 3.05559 + -5.7201 -5.7201 + -3.98882 -3.98882 + -0.131939 -0.131939 + 6.25683 6.25683 + 0.712945 0.712945 + 4.17266 4.17266 + 9.04425 9.04425 + -2.39179 -2.39179 + 3.03807 3.03807 + 5.79693 5.79693 + -5.28875 -5.28875 + -2.56482 -2.56482 + -1.00679 -1.00679 + -0.512488 -0.512488 + -4.60373 -4.60373 + -2.69188 -2.69188 + 0.958182 0.958182 + -1.08075 -1.08075 + 2.66033 2.66033 + -5.77563 -5.77563 + 5.393 5.393 + 0.822122 0.822122 + 3.50281 3.50281 + -1.90373 -1.90373 + -3.41986 -3.41986 + -7.32502 -7.32502 + -2.0256 -2.0256 + -6.28488 -6.28488 + 0.358393 0.358393 + 1.89312 1.89312 + -0.709162 -0.709162 + -4.43491 -4.43491 + -3.56097 -3.56097 + -8.3806 -8.3806 + -5.56256 -5.56256 + -3.40994 -3.40994 + -6.15002 -6.15002 + 0.949459 0.949459 + 3.18256 3.18256 + 6.31834 6.31834 + 12.4998 12.4998 + -6.16927 -6.16927 + -1.73781 -1.73781 + 0.274813 0.274813 + 7.11001 7.11001 + 6.79962 6.79962 + 2.00121 2.00121 + -4.30592 -4.30592 + -2.38345 -2.38345 + 7.50502 7.50502 + -3.56375 -3.56375 + -1.07828 -1.07828 + 7.4632 7.4632 + -5.78317 -5.78317 + -0.54432 -0.54432 + 8.82699 8.82699 + -2.51939 -2.51939 + -3.21417 -3.21417 + 3.06052 3.06052 + -0.45856 -0.45856 + 8.89456 8.89456 + 5.89006 5.89006 + 1.01204 1.01204 + 4.9875 4.9875 + -1.63 -1.63 + 1.35424 1.35424 + 3.72608 3.72608 + -8.53795 -8.53795 + -5.93051 -5.93051 + -2.35685 -2.35685 + 3.51823 3.51823 + 3.65767 3.65767 + -3.04233 -3.04233 + -1.12453 -1.12453 + -1.68299 -1.68299 + -5.69175 -5.69175 + 3.66601 3.66601 + -3.11779 -3.11779 + -0.20161 -0.20161 + 0.78317 0.78317 + 2.28035 2.28035 + -4.43493 -4.43493 + 2.12557 2.12557 + 6.97219 6.97219 + 4.91357 4.91357 + -1.87778 -1.87778 + 1.98163 1.98163 + 1.01184 1.01184 + 0.0544142 0.0544142 + -0.748318 -0.748318 + 10.0677 10.0677 + -5.50226 -5.50226 + 3.89987 3.89987 + 1.38136 1.38136 + 4.67073 4.67073 + 5.3372 5.3372 + -1.29886 -1.29886 + -0.965173 -0.965173 + 0.546909 0.546909 + 5.87692 5.87692 + -10.1356 -10.1356 + 0.541422 0.541422 + 0.486656 0.486656 + 8.42395 8.42395 + -4.04554 -4.04554 + 11.4728 11.4728 + -6.54655 -6.54655 + 6.90602 6.90602 + -13.8383 -13.8383 + 2.64142 2.64142 + 3.96547 3.96547 + -0.887154 -0.887154 + 0.0442338 0.0442338 + -5.12331 -5.12331 + 4.95632 4.95632 + 3.15264 3.15264 + 4.80494 4.80494 + -5.42313 -5.42313 + -4.2795 -4.2795 + 1.661 1.661 + 3.85204 3.85204 + 10.1308 10.1308 + -4.34526 -4.34526 + -5.49571 -5.49571 + 3.92939 3.92939 + -3.28527 -3.28527 + 0.154911 0.154911 + -3.606 -3.606 + 5.91814 5.91814 + -8.85249 -8.85249 + 9.38796 9.38796 + -0.800741 -0.800741 + -2.87508 -2.87508 + 2.99955 2.99955 + -7.13252 -7.13252 + -6.77081 -6.77081 + -2.28359 -2.28359 + -0.180517 -0.180517 + 7.04622 7.04622 + 4.2577 4.2577 + -4.73655 -4.73655 + -0.249759 -0.249759 + 2.4412 2.4412 + 8.47175 8.47175 + -3.24927 -3.24927 + -12.5242 -12.5242 + -2.74845 -2.74845 + -9.32786 -9.32786 + 4.21624 4.21624 + 2.94687 2.94687 + 3.35216 3.35216 + -3.5485 -3.5485 + 6.97298 6.97298 + 2.01617 2.01617 + 4.70745 4.70745 + 2.96924 2.96924 + -0.18365 -0.18365 + -0.694247 -0.694247 + -7.14459 -7.14459 + 5.38548 5.38548 + 2.04923 2.04923 + -5.33216 -5.33216 + 5.47927 5.47927 + 0.357422 0.357422 + 4.36552 4.36552 + 6.88375 6.88375 + -6.47244 -6.47244 + -3.40726 -3.40726 + -6.56449 -6.56449 + 6.34818 6.34818 + -4.23984 -4.23984 + -11.1113 -11.1113 + 2.41915 2.41915 + 3.90153 3.90153 + -7.69422 -7.69422 + -8.03709 -8.03709 + -9.64719 -9.64719 + -4.04416 -4.04416 + 2.64435 2.64435 + 5.11566 5.11566 + -1.27873 -1.27873 + -1.01265 -1.01265 + -8.38716 -8.38716 + -0.960571 -0.960571 + 2.05458 2.05458 + -1.89606 -1.89606 + -7.04401 -7.04401 + 4.91798 4.91798 + 2.12484 2.12484 + 2.38768 2.38768 + 7.9691 7.9691 + -1.00886 -1.00886 + -4.9569 -4.9569 + -4.74278 -4.74278 + 0.191814 0.191814 + -5.2925 -5.2925 + -1.15484 -1.15484 + 2.27898 2.27898 + 4.12308 4.12308 + -6.18988 -6.18988 + 7.1232 7.1232 + -6.68678 -6.68678 + 1.65808 1.65808 + 8.53283 8.53283 + 0.509069 0.509069 + -3.03638 -3.03638 + -4.86641 -4.86641 + 7.20729 7.20729 + -7.51236 -7.51236 + 3.37738 3.37738 + -0.0649395 -0.0649395 + 2.75749 2.75749 + -5.61535 -5.61535 + 3.1237 3.1237 + -0.766488 -0.766488 + 4.39047 4.39047 + 1.28616 1.28616 + -8.02003 -8.02003 + 4.21688 4.21688 + -2.79942 -2.79942 + -5.80171 -5.80171 + 9.97235 9.97235 + 21.8011 21.8011 + -3.58992 -3.58992 + 5.03481 5.03481 + -2.1684 -2.1684 + -5.46844 -5.46844 + 1.57702 1.57702 + -4.53923 -4.53923 + -1.77363 -1.77363 + -0.489051 -0.489051 + -0.371992 -0.371992 + 8.264 8.264 + 1.63502 1.63502 + -1.10134 -1.10134 + 4.76612 4.76612 + 5.93085 5.93085 + -2.07348 -2.07348 + 4.26074 4.26074 + 4.1331 4.1331 + 11.1442 11.1442 + 2.18824 2.18824 + 2.18854 2.18854 + 0.210843 0.210843 + -9.30743 -9.30743 + 5.34539 5.34539 + -4.21419 -4.21419 + -3.97284 -3.97284 + -2.67745 -2.67745 + 4.17366 4.17366 + 2.41498 2.41498 + 0.801359 0.801359 + 8.35766 8.35766 + -1.29589 -1.29589 + -7.45531 -7.45531 + -7.26731 -7.26731 + 4.06669 4.06669 + -2.35771 -2.35771 + -8.73174 -8.73174 + -0.837329 -0.837329 + -2.53419 -2.53419 + 44.3977 44.3977 + 13.5049 13.5049 + -3.66878 -3.66878 + -6.5533 -6.5533 + -5.59814 -5.59814 + -10.5759 -10.5759 + 0.663108 0.663108 + -3.45147 -3.45147 + -3.75944 -3.75944 + 1.84721 1.84721 + -0.363204 -0.363204 + 4.54678 4.54678 + 2.07408 2.07408 + 7.85227 7.85227 + -7.53707 -7.53707 + 4.18344 4.18344 + -1.96048 -1.96048 + 6.24217 6.24217 + -9.16295 -9.16295 + 0.0480544 0.0480544 + 2.84725 2.84725 + 1.08008 1.08008 + -0.874464 -0.874464 + 1.67428 1.67428 + -1.91245 -1.91245 + 3.53596 3.53596 + 3.75983 3.75983 + 1.37903 1.37903 + -0.799744 -0.799744 + 2.75015 2.75015 + -11.0835 -11.0835 + -1.6781 -1.6781 + 2.86463 2.86463 + -11.1467 -11.1467 + -3.76398 -3.76398 + 9.06439 9.06439 + 9.84403 9.84403 + -5.07 -5.07 + 3.2952 3.2952 + -1.62527 -1.62527 + -7.98997 -7.98997 + -7.8193 -7.8193 + 1.10895 1.10895 + 0.460921 0.460921 + -1.47816 -1.47816 + 0.718936 0.718936 + -3.74006 -3.74006 + -2.87535 -2.87535 + 0.037427 0.037427 + -4.49959 -4.49959 + 0.0987492 0.0987492 + 1.8443 1.8443 + 0.748879 0.748879 + 1.4364 1.4364 + -0.90809 -0.90809 + -1.36403 -1.36403 + -1.27123 -1.27123 + 3.09447 3.09447 + -3.82708 -3.82708 + 0.683696 0.683696 + 3.96997 3.96997 + 0.461267 0.461267 + 4.96801 4.96801 + -5.96169 -5.96169 + 2.56714 2.56714 + -10.7519 -10.7519 + -3.39381 -3.39381 + 1.15623 1.15623 + -3.95798 -3.95798 + -1.42797 -1.42797 + 4.85734 4.85734 + -4.46424 -4.46424 + -11.9172 -11.9172 + 0.740766 0.740766 + -2.06857 -2.06857 + -1.23723 -1.23723 + -6.43373 -6.43373 + 7.04893 7.04893 + -1.10208 -1.10208 + -0.0507102 -0.0507102 + 8.23443 8.23443 + -1.71378 -1.71378 + 2.769 2.769 + 9.77752 9.77752 + 0.423859 0.423859 + 0.901832 0.901832 + 0.0738559 0.0738559 + -0.487266 -0.487266 + 2.05358 2.05358 + -8.73912 -8.73912 + 3.01532 3.01532 + -0.926127 -0.926127 + -11.2315 -11.2315 + 1.79698 1.79698 + -13.074 -13.074 + 3.72342 3.72342 + -9.17341 -9.17341 + 7.23722 7.23722 + 3.85919 3.85919 + -4.10267 -4.10267 + 5.89157 5.89157 + -1.06631 -1.06631 + -2.18366 -2.18366 + -0.0316413 -0.0316413 + -8.63864 -8.63864 + -0.194451 -0.194451 + 2.71759 2.71759 + -5.19424 -5.19424 + -16.7634 -16.7634 + 5.97943 5.97943 + 0.319596 0.319596 + -10.0687 -10.0687 + 1.12736 1.12736 + 2.11687 2.11687 + 2.5643 2.5643 + 0.502174 0.502174 + -5.75011 -5.75011 + -11.1808 -11.1808 + -3.42246 -3.42246 + 7.55982 7.55982 + -5.85592 -5.85592 + 1.22363 1.22363 + 1.39871 1.39871 + 3.35581 3.35581 + 2.99389 2.99389 + -0.762194 -0.762194 + 1.39891 1.39891 + -4.24295 -4.24295 + -6.95612 -6.95612 + 7.00699 7.00699 + -30.893 -30.893 + -7.3071 -7.3071 + 17.5017 17.5017 + -3.26283 -3.26283 + -4.13569 -4.13569 + 4.33006 4.33006 + -5.94055 -5.94055 + -0.564017 -0.564017 + 5.60949 5.60949 + 7.50747 7.50747 + -4.08147 -4.08147 + 4.08671 4.08671 + 6.72008 6.72008 + -5.02883 -5.02883 + -3.48779 -3.48779 + 4.76881 4.76881 + 4.5818 4.5818 + -3.10608 -3.10608 + -5.08198 -5.08198 + -5.54477 -5.54477 + -13.1989 -13.1989 + -8.63604 -8.63604 + -0.688683 -0.688683 + -2.34276 -2.34276 + -3.19008 -3.19008 + 0.204818 0.204818 + 0.639057 0.639057 + 12.6767 12.6767 + -3.40057 -3.40057 + -6.36799 -6.36799 + 3.7564 3.7564 + -3.04825 -3.04825 + -3.98011 -3.98011 + -2.21944 -2.21944 + 8.40757 8.40757 + -5.6418 -5.6418 + 3.3001 3.3001 + -0.678107 -0.678107 + -2.42254 -2.42254 + 0.439524 0.439524 + -0.417505 -0.417505 + -4.98938 -4.98938 + -6.34015 -6.34015 + -4.84203 -4.84203 + 2.86778 2.86778 + 3.29409 3.29409 + 2.59772 2.59772 + 5.20187 5.20187 + 3.55625 3.55625 + -7.065 -7.065 + -6.60792 -6.60792 + -3.20259 -3.20259 + 0.417062 0.417062 + -2.39846 -2.39846 + -5.762 -5.762 + 1.74843 1.74843 + 8.19239 8.19239 + -1.7349 -1.7349 + -0.0331415 -0.0331415 + 5.00712 5.00712 + 10.611 10.611 + 9.28817 9.28817 + -3.85324 -3.85324 + 2.29622 2.29622 + 10.962 10.962 + 4.44034 4.44034 + -3.2265 -3.2265 + 1.39326 1.39326 + -1.56539 -1.56539 + -8.78843 -8.78843 + -1.74101 -1.74101 + 8.51953 8.51953 + 3.31178 3.31178 + -1.20051 -1.20051 + -3.93224 -3.93224 + 2.4431 2.4431 + 3.69278 3.69278 + -10.2714 -10.2714 + -13.7579 -13.7579 + -1.76844 -1.76844 + -0.448193 -0.448193 + 1.48574 1.48574 + -0.831377 -0.831377 + 6.42657 6.42657 + 6.51848 6.51848 + 2.7764 2.7764 + 4.29448 4.29448 + -1.27173 -1.27173 + -7.14856 -7.14856 + 2.95751 2.95751 + 2.39789 2.39789 + 4.79429 4.79429 + 7.29216 7.29216 + -4.91502 -4.91502 + 2.38701 2.38701 + -2.34997 -2.34997 + -0.876115 -0.876115 + -0.672649 -0.672649 + 4.43884 4.43884 + 0.254258 0.254258 + -3.56471 -3.56471 + 0.161779 0.161779 + -10.1128 -10.1128 + 9.97279 9.97279 + -5.01498 -5.01498 + 1.10415 1.10415 + 1.37993 1.37993 + -3.32619 -3.32619 + 2.57257 2.57257 + -0.137478 -0.137478 + 1.49426 1.49426 + -0.805644 -0.805644 + 3.25356 3.25356 + 2.46332 2.46332 + 1.39266 1.39266 + 4.15167 4.15167 + -9.27164 -9.27164 + -2.29794 -2.29794 + 0.067971 0.067971 + 3.83697 3.83697 + 5.7385 5.7385 + -6.15176 -6.15176 + -4.08442 -4.08442 + -6.18563 -6.18563 + 6.44396 6.44396 + 5.63585 5.63585 + 1.21604 1.21604 + 11.1837 11.1837 + 2.29144 2.29144 + -0.995473 -0.995473 + 5.22826 5.22826 + 9.27205 9.27205 + -7.23457 -7.23457 + 6.29887 6.29887 + 2.48343 2.48343 + -4.96111 -4.96111 + -5.52811 -5.52811 + -4.40855 -4.40855 + -5.69429 -5.69429 + -1.12765 -1.12765 + -0.22142 -0.22142 + -5.96815 -5.96815 + 4.55923 4.55923 + -1.05719 -1.05719 + 2.07986 2.07986 + 7.77539 7.77539 + -2.03581 -2.03581 + 0.270705 0.270705 + 0.126658 0.126658 + -6.1672 -6.1672 + -16.0576 -16.0576 + 0.635198 0.635198 + 8.55006 8.55006 + -2.93081 -2.93081 + -1.7657 -1.7657 + -0.37886 -0.37886 + -2.4086 -2.4086 + 1.41889 1.41889 + -1.40539 -1.40539 + 0.963807 0.963807 + -2.14947 -2.14947 + -6.31832 -6.31832 + -4.30827 -4.30827 + 6.2609 6.2609 + -8.36351 -8.36351 + 4.28564 4.28564 + 0.646361 0.646361 + 4.60485 4.60485 + -3.1664 -3.1664 + -0.611618 -0.611618 + -9.53534 -9.53534 + 1.92275 1.92275 + -8.1521 -8.1521 + 0.101441 0.101441 + 0.399002 0.399002 + -2.04551 -2.04551 + -4.5564 -4.5564 + 3.0555 3.0555 + 0.992401 0.992401 + -5.62638 -5.62638 + -0.46873 -0.46873 + -6.86208 -6.86208 + -2.77108 -2.77108 + 3.51118 3.51118 + 0.885266 0.885266 + 3.65701 3.65701 + 6.88336 6.88336 + -7.25948 -7.25948 + 7.31435 7.31435 + -6.57357 -6.57357 + 3.67947 3.67947 + 4.80901 4.80901 + -2.80342 -2.80342 + 5.78724 5.78724 + 5.30985 5.30985 + 7.24724 7.24724 + -1.30439 -1.30439 + 2.50975 2.50975 + 5.28538 5.28538 + -3.91583 -3.91583 + 2.98722 2.98722 + 5.31167 5.31167 + -0.596966 -0.596966 + -4.94141 -4.94141 + 4.59005 4.59005 + 1.3813 1.3813 + 4.0611 4.0611 + -0.747616 -0.747616 + -3.1697 -3.1697 + -1.70787 -1.70787 + -2.43542 -2.43542 + -5.86823 -5.86823 + -10.9093 -10.9093 + 5.20087 5.20087 + -6.40378 -6.40378 + 1.5149 1.5149 + -6.52874 -6.52874 + -5.69743 -5.69743 + 1.06819 1.06819 + -7.31776 -7.31776 + 3.69649 3.69649 + -4.21319 -4.21319 + -4.91507 -4.91507 + 5.44776 5.44776 + -0.708927 -0.708927 + 1.94895 1.94895 + 2.90927 2.90927 + -2.82547 -2.82547 + -1.79858 -1.79858 + -15.6727 -15.6727 + -0.308918 -0.308918 + 2.61943 2.61943 + -3.89041 -3.89041 + -1.84684 -1.84684 + -6.80446 -6.80446 + 3.97398 3.97398 + 2.31201 2.31201 + 4.29417 4.29417 + -1.24479 -1.24479 + 4.25927 4.25927 + -1.96968 -1.96968 + 0.703519 0.703519 + 2.06517 2.06517 + 0.920347 0.920347 + 6.22843 6.22843 + 1.86167 1.86167 + 0.43407 0.43407 + 1.25225 1.25225 + -0.00512493 -0.00512493 + -1.70887 -1.70887 + 0.725693 0.725693 + 6.11604 6.11604 + -5.87059 -5.87059 + 3.26102 3.26102 + 2.0488 2.0488 + -0.0544172 -0.0544172 + 2.57295 2.57295 + -1.10578 -1.10578 + 2.43904 2.43904 + -2.34604 -2.34604 + 3.2098 3.2098 + 2.16089 2.16089 + -9.35001 -9.35001 + 9.43924 9.43924 + 0.916747 0.916747 + 2.59533 2.59533 + -1.84596 -1.84596 + 1.02889 1.02889 + 0.755944 0.755944 + 8.28274 8.28274 + -3.21136 -3.21136 + 1.24897 1.24897 + -0.363928 -0.363928 + 2.37533 2.37533 + -1.5794 -1.5794 + 6.67417 6.67417 + -4.4632 -4.4632 + 8.53731 8.53731 + -1.16526 -1.16526 + -0.51467 -0.51467 + -4.91688 -4.91688 + 7.17741 7.17741 + 4.61708 4.61708 + -2.41511 -2.41511 + -11.5234 -11.5234 + 2.61523 2.61523 + 4.7703 4.7703 + 6.72381 6.72381 + 5.65388 5.65388 + -4.23963 -4.23963 + 0.925176 0.925176 + 1.98862 1.98862 + -6.14466 -6.14466 + 2.76728 2.76728 + -0.83598 -0.83598 + -4.22593 -4.22593 + 5.99083 5.99083 + -4.886 -4.886 + 4.37801 4.37801 + 5.77761 5.77761 + 3.38352 3.38352 + -0.311291 -0.311291 + 8.26669 8.26669 + -4.94787 -4.94787 + -9.62034 -9.62034 + 2.37023 2.37023 + 3.41718 3.41718 + -2.43368 -2.43368 + 3.5898 3.5898 + -1.21973 -1.21973 + 0.0350305 0.0350305 + -4.33097 -4.33097 + -3.41432 -3.41432 + 2.59161 2.59161 + -2.11239 -2.11239 + -1.0801 -1.0801 + -3.27061 -3.27061 + -0.34025 -0.34025 + -6.40563 -6.40563 + -0.522305 -0.522305 + 4.63382 4.63382 + 1.5154 1.5154 + 0.968893 0.968893 + 2.79354 2.79354 + -0.829942 -0.829942 + -1.76388 -1.76388 + -6.64903 -6.64903 + -8.52588 -8.52588 + 2.70798 2.70798 + 6.78381 6.78381 + -5.67891 -5.67891 + -0.0588557 -0.0588557 + -4.12923 -4.12923 + -2.70431 -2.70431 + -0.12131 -0.12131 + 6.59494 6.59494 + 0.830427 0.830427 + 3.40436 3.40436 + 6.98828 6.98828 + -2.33332 -2.33332 + 5.85244 5.85244 + -10.0398 -10.0398 + -0.242519 -0.242519 + -3.38719 -3.38719 + 2.74288 2.74288 + 3.82961 3.82961 + -6.85166 -6.85166 + -0.345431 -0.345431 + -3.03082 -3.03082 + 1.68089 1.68089 + -0.785036 -0.785036 + -2.92804 -2.92804 + 1.03727 1.03727 + 5.51647 5.51647 + -2.15538 -2.15538 + -6.20918 -6.20918 + -0.986195 -0.986195 + -4.4207 -4.4207 + -0.314791 -0.314791 + -6.64843 -6.64843 + 1.255 1.255 + 4.39107 4.39107 + 2.20706 2.20706 + -1.894 -1.894 + -3.01471 -3.01471 + -0.0623641 -0.0623641 + -5.76316 -5.76316 + -2.45987 -2.45987 + -2.09262 -2.09262 + 0.0458748 0.0458748 + 5.09539 5.09539 + -3.80431 -3.80431 + -3.90738 -3.90738 + -6.48843 -6.48843 + -2.58373 -2.58373 + -6.38764 -6.38764 + 7.38858 7.38858 + 0.492176 0.492176 + 8.79347 8.79347 + 2.04442 2.04442 + -0.216083 -0.216083 + 11.3375 11.3375 + -3.4177 -3.4177 + 3.90111 3.90111 + 4.92081 4.92081 + 4.45964 4.45964 + 11.1458 11.1458 + -2.2688 -2.2688 + -4.43463 -4.43463 + -4.22186 -4.22186 + -5.93987 -5.93987 + 3.4437 3.4437 + -5.60816 -5.60816 + -8.04401 -8.04401 + -4.95256 -4.95256 + 3.88283 3.88283 + -0.173935 -0.173935 + -2.63243 -2.63243 + -1.03812 -1.03812 + -9.14078 -9.14078 + -6.1411 -6.1411 + 3.4284 3.4284 + -9.8305 -9.8305 + 6.76115 6.76115 + -11.3646 -11.3646 + -5.7296 -5.7296 + -2.41831 -2.41831 + -5.21505 -5.21505 + 10.4347 10.4347 + 2.06721 2.06721 + 1.02265 1.02265 + -6.93537 -6.93537 + 1.28707 1.28707 + 0.939615 0.939615 + 11.262 11.262 + 1.2805 1.2805 + 4.8619 4.8619 + 3.15836 3.15836 + -5.18747 -5.18747 + -2.98078 -2.98078 + -2.0489 -2.0489 + -2.85634 -2.85634 + -4.56059 -4.56059 + -4.0715 -4.0715 + 0.469543 0.469543 + -2.05188 -2.05188 + -2.79567 -2.79567 + 3.82027 3.82027 + 2.55175 2.55175 + -0.468207 -0.468207 + -5.65994 -5.65994 + 2.13508 2.13508 + -3.17019 -3.17019 + 6.53032 6.53032 + -4.98714 -4.98714 + -1.94956 -1.94956 + -3.08465 -3.08465 + 8.11664 8.11664 + 8.86283 8.86283 + 0.84108 0.84108 + 5.22353 5.22353 + -3.45671 -3.45671 + -1.38725 -1.38725 + 1.35206 1.35206 + -10.4407 -10.4407 + -2.20051 -2.20051 + -0.228019 -0.228019 + -1.38039 -1.38039 + 11.1342 11.1342 + 5.17568 5.17568 + -4.54852 -4.54852 + -1.26392 -1.26392 + 5.69792 5.69792 + -4.90866 -4.90866 + 2.84526 2.84526 + 10.9699 10.9699 + 12.9756 12.9756 + 8.48223 8.48223 + 2.11902 2.11902 + 3.74471 3.74471 + -5.14437 -5.14437 + -14.7206 -14.7206 + 3.01028 3.01028 + -2.67988 -2.67988 + -2.88296 -2.88296 + -4.95895 -4.95895 + -1.82286 -1.82286 + 5.23419 5.23419 + -2.23867 -2.23867 + 0.610838 0.610838 + 2.09177 2.09177 + 5.74677 5.74677 + 3.6242 3.6242 + 2.0758 2.0758 + -2.85159 -2.85159 + -3.93562 -3.93562 + 3.85649 3.85649 + -5.75638 -5.75638 + -7.07444 -7.07444 + 0.907402 0.907402 + -8.92532 -8.92532 + -4.09782 -4.09782 + 1.85777 1.85777 + 5.73041 5.73041 + -2.17118 -2.17118 + -3.4713 -3.4713 + 7.95825 7.95825 + 9.10838 9.10838 + 1.80182 1.80182 + -0.54593 -0.54593 + -4.89919 -4.89919 + -2.97982 -2.97982 + 0.807424 0.807424 + -2.27 -2.27 + -13.2338 -13.2338 + -3.94367 -3.94367 + -5.72938 -5.72938 + -2.42243 -2.42243 + -3.69581 -3.69581 + -4.71307 -4.71307 + 1.38983 1.38983 + -5.37869 -5.37869 + -6.82815 -6.82815 + 2.73203 2.73203 + 13.6495 13.6495 + -6.29731 -6.29731 + -8.43712 -8.43712 + 14.1567 14.1567 + -0.978804 -0.978804 + 1.26264 1.26264 + -9.25575 -9.25575 + -8.10968 -8.10968 + -3.98015 -3.98015 + 6.60273 6.60273 + -3.98373 -3.98373 + 1.35817 1.35817 + 1.20988 1.20988 + 1.53069 1.53069 + 4.08368 4.08368 + -2.38429 -2.38429 + -4.67381 -4.67381 + -5.49726 -5.49726 + 0.657715 0.657715 + -0.00123905 -0.00123905 + 4.62712 4.62712 + -0.317445 -0.317445 + -5.08829 -5.08829 + -9.85674 -9.85674 + 5.31787 5.31787 + 1.61793 1.61793 + 3.9901 3.9901 + -1.04243 -1.04243 + -3.73679 -3.73679 + 0.670282 0.670282 + 9.03148 9.03148 + -4.77058 -4.77058 + 8.60147 8.60147 + -0.664744 -0.664744 + 1.97711 1.97711 + -5.35794 -5.35794 + -9.70033 -9.70033 + 10.7781 10.7781 + 1.96443 1.96443 + 1.84069 1.84069 + -12.0109 -12.0109 + 2.08404 2.08404 + 3.64031 3.64031 + 8.65585 8.65585 + -11.8355 -11.8355 + 9.89404 9.89404 + 0.279063 0.279063 + -0.315296 -0.315296 + 3.74263 3.74263 + 6.54645 6.54645 + 5.43941 5.43941 + 4.83252 4.83252 + 1.70716 1.70716 + -3.27497 -3.27497 + -3.07764 -3.07764 + 9.25309 9.25309 + -1.69559 -1.69559 + 10.1694 10.1694 + -3.42523 -3.42523 + 6.39435 6.39435 + 2.18084 2.18084 + 1.33177 1.33177 + -0.709393 -0.709393 + 1.44799 1.44799 + 0.881759 0.881759 + -2.35085 -2.35085 + -1.91407 -1.91407 + 0.302603 0.302603 + 1.40288 1.40288 + -2.37323 -2.37323 + -7.74084 -7.74084 + -7.73224 -7.73224 + 2.8793 2.8793 + 6.62065 6.62065 + 1.4654 1.4654 + -0.982735 -0.982735 + -0.97328 -0.97328 + -8.38882 -8.38882 + 8.74643 8.74643 + -7.86996 -7.86996 + 3.25655 3.25655 + 2.78551 2.78551 + -5.17511 -5.17511 + 4.90515 4.90515 + 0.28899 0.28899 + 3.57292 3.57292 + -5.25376 -5.25376 + -8.57274 -8.57274 + -1.18267 -1.18267 + 37.4072 37.4072 + -4.00801 -4.00801 + 4.8073 4.8073 + -4.45001 -4.45001 + 7.66024 7.66024 + -4.47725 -4.47725 + -10.2209 -10.2209 + -4.80026 -4.80026 + -0.64446 -0.64446 + -0.899171 -0.899171 + 1.09833 1.09833 + -0.988097 -0.988097 + 2.82126 2.82126 + -8.19269 -8.19269 + -2.64922 -2.64922 + -9.16004 -9.16004 + -2.39588 -2.39588 + -4.72025 -4.72025 + 2.34077 2.34077 + 3.83879 3.83879 + 1.9499 1.9499 + -0.361603 -0.361603 + 7.79929 7.79929 + 2.34774 2.34774 + -8.21052 -8.21052 + -2.02077 -2.02077 + -1.58017 -1.58017 + -0.410542 -0.410542 + -10.7206 -10.7206 + 3.26874 3.26874 + 2.80972 2.80972 + 0.0906836 0.0906836 + -1.64773 -1.64773 + 6.49353 6.49353 + -0.791109 -0.791109 + 4.71404 4.71404 + 0.0741314 0.0741314 + -0.414415 -0.414415 + 6.84572 6.84572 + -0.367457 -0.367457 + 1.17563 1.17563 + 0.51039 0.51039 + 4.40348 4.40348 + 0.978932 0.978932 + 3.79206 3.79206 + 4.57632 4.57632 + 2.77883 2.77883 + 0.490867 0.490867 + -0.151798 -0.151798 + 6.72243 6.72243 + 4.77773 4.77773 + -0.50633 -0.50633 + -8.08639 -8.08639 + 4.88619 4.88619 + -2.07669 -2.07669 + -2.24093 -2.24093 + 1.72994 1.72994 + -7.45157 -7.45157 + -12.1192 -12.1192 + 1.4328 1.4328 + -8.14432 -8.14432 + -6.25485 -6.25485 + 0.516865 0.516865 + 7.11864 7.11864 + -0.616318 -0.616318 + -0.761916 -0.761916 + -5.99496 -5.99496 + 10.4321 10.4321 + -0.516052 -0.516052 + -5.68287 -5.68287 + -4.15541 -4.15541 + 1.56619 1.56619 + -20.8292 -20.8292 + 0.788033 0.788033 + 3.34264 3.34264 + 3.70493 3.70493 + -0.0822138 -0.0822138 + 2.31304 2.31304 + -1.69352 -1.69352 + 2.10396 2.10396 + 7.2613 7.2613 + -1.81799 -1.81799 + -2.09968 -2.09968 + -3.8336 -3.8336 + -3.93478 -3.93478 + 3.3059 3.3059 + 4.19189 4.19189 + -1.93794 -1.93794 + 2.7117 2.7117 + 9.43261 9.43261 + -1.83318 -1.83318 + -1.12685 -1.12685 + 2.40725 2.40725 + 7.50947 7.50947 + 7.65688 7.65688 + -5.02792 -5.02792 + -2.55777 -2.55777 + -1.9946 -1.9946 + -0.126192 -0.126192 + -3.30905 -3.30905 + -0.209775 -0.209775 + 9.06409 9.06409 + -3.79201 -3.79201 + 8.80185 8.80185 + -1.59367 -1.59367 + -2.49213 -2.49213 + -3.5242 -3.5242 + 2.4892 2.4892 + 5.68222 5.68222 + 4.29073 4.29073 + 0.490494 0.490494 + 3.31313 3.31313 + 8.27344 8.27344 + 1.44936 1.44936 + 5.94283 5.94283 + -5.90497 -5.90497 + 0.316931 0.316931 + 1.93975 1.93975 + -1.33405 -1.33405 + -4.17957 -4.17957 + 2.45999 2.45999 + -10.0965 -10.0965 + 0.648564 0.648564 + -0.745957 -0.745957 + -6.08922 -6.08922 + -6.45851 -6.45851 + 2.70093 2.70093 + -2.59331 -2.59331 + -2.73319 -2.73319 + -6.50584 -6.50584 + 4.14167 4.14167 + 6.78757 6.78757 + 4.63335 4.63335 + 2.01754 2.01754 + 3.97717 3.97717 + 2.73775 2.73775 + 2.04299 2.04299 + 7.03044 7.03044 + -8.59414 -8.59414 + -4.19956 -4.19956 + 0.0135157 0.0135157 + -5.45393 -5.45393 + 2.75578 2.75578 + 0.730278 0.730278 + -0.410035 -0.410035 + 10.7831 10.7831 + -2.82537 -2.82537 + 1.85601 1.85601 + 1.68496 1.68496 + 2.75249 2.75249 + 9.40848 9.40848 + 1.6032 1.6032 + -3.91263 -3.91263 + 1.12247 1.12247 + -3.46516 -3.46516 + -1.48668 -1.48668 + 6.7676 6.7676 + -5.76927 -5.76927 + -2.19943 -2.19943 + -1.61329 -1.61329 + 3.35791 3.35791 + -7.80737 -7.80737 + 3.06567 3.06567 + -12.2037 -12.2037 + 12.541 12.541 + 4.42316 4.42316 + 6.48419 6.48419 + 1.17664 1.17664 + 2.97986 2.97986 + -8.63966 -8.63966 + 0.241757 0.241757 + -5.03654 -5.03654 + -1.94594 -1.94594 + 12.8093 12.8093 + -3.58644 -3.58644 + -3.35952 -3.35952 + -0.864134 -0.864134 + -12.4807 -12.4807 + -1.69909 -1.69909 + -5.67676 -5.67676 + -10.6435 -10.6435 + -3.86815 -3.86815 + 4.20674 4.20674 + -4.94992 -4.94992 + 7.63289 7.63289 + -5.5226 -5.5226 + 1.58362 1.58362 + 1.14864 1.14864 + 5.98635 5.98635 + 11.9692 11.9692 + -0.208588 -0.208588 + -0.177219 -0.177219 + 6.35143 6.35143 + -2.21028 -2.21028 + 0.693657 0.693657 + 2.66882 2.66882 + -0.494413 -0.494413 + 10.9482 10.9482 + 2.9522 2.9522 + 1.69427 1.69427 + -5.54007 -5.54007 + -1.44208 -1.44208 + -2.75377 -2.75377 + 7.62773 7.62773 + -0.0991657 -0.0991657 + 0.541024 0.541024 + 0.383422 0.383422 + -6.28538 -6.28538 + -3.63239 -3.63239 + 5.54891 5.54891 + 4.38377 4.38377 + -4.21607 -4.21607 + -1.58462 -1.58462 + 1.99568 1.99568 + 1.70177 1.70177 + 1.65142 1.65142 + 1.79811 1.79811 + -6.82605 -6.82605 + 3.65159 3.65159 + 2.60935 2.60935 + -2.91237 -2.91237 + -1.56808 -1.56808 + -3.07334 -3.07334 + 0.883426 0.883426 + -1.59697 -1.59697 + 4.44432 4.44432 + -2.72255 -2.72255 + -0.853149 -0.853149 + -0.132598 -0.132598 + -0.63629 -0.63629 + -3.69308 -3.69308 + -7.18449 -7.18449 + 1.20547 1.20547 + 14.3427 14.3427 + 5.08288 5.08288 + 0.957041 0.957041 + 0.153537 0.153537 + -7.14906 -7.14906 + -8.78572 -8.78572 + 4.05049 4.05049 + 3.22929 3.22929 + -3.34601 -3.34601 + 3.86442 3.86442 + -2.80641 -2.80641 + 6.51055 6.51055 + -4.58706 -4.58706 + -1.51146 -1.51146 + 3.88212 3.88212 + 1.89549 1.89549 + 3.50062 3.50062 + -1.43005 -1.43005 + -2.91969 -2.91969 + -6.52573 -6.52573 + -3.8843 -3.8843 + -8.34716 -8.34716 + -7.42192 -7.42192 + -3.98985 -3.98985 + 15.526 15.526 + 8.70318 8.70318 + -1.10105 -1.10105 + 2.14694 2.14694 + 7.71484 7.71484 + -0.0260442 -0.0260442 + -3.31138 -3.31138 + 1.67906 1.67906 + -0.083112 -0.083112 + -8.42905 -8.42905 + -8.82729 -8.82729 + 11.2859 11.2859 + -8.07136 -8.07136 + -3.9371 -3.9371 + -4.63176 -4.63176 + -1.23605 -1.23605 + -2.08565 -2.08565 + 1.93918 1.93918 + -12.5031 -12.5031 + -0.442281 -0.442281 + -5.50289 -5.50289 + -0.815112 -0.815112 + 0.0898735 0.0898735 + 4.69373 4.69373 + -7.22004 -7.22004 + 0.543294 0.543294 + 4.2932 4.2932 + 2.12984 2.12984 + -4.42752 -4.42752 + 3.03694 3.03694 + -3.73337 -3.73337 + -12.0483 -12.0483 + -5.99704 -5.99704 + 0.0707967 0.0707967 + -4.52239 -4.52239 + 3.65625 3.65625 + -5.61903 -5.61903 + 9.78971 9.78971 + 8.47575 8.47575 + -0.320966 -0.320966 + -7.10339 -7.10339 + 0.485669 0.485669 + 3.19439 3.19439 + -0.411976 -0.411976 + -0.782875 -0.782875 + 16.4086 16.4086 + -2.67312 -2.67312 + 0.73424 0.73424 + 8.32014 8.32014 + -1.24665 -1.24665 + 3.70031 3.70031 + -6.22155 -6.22155 + -6.34804 -6.34804 + -4.84631 -4.84631 + 7.19111 7.19111 + 2.58937 2.58937 + 2.12044 2.12044 + -0.304369 -0.304369 + -11.5161 -11.5161 + -4.75933 -4.75933 + -5.40287 -5.40287 + -14.7511 -14.7511 + -11.3269 -11.3269 + -3.40961 -3.40961 + -8.36998 -8.36998 + -7.86816 -7.86816 + 3.46638 3.46638 + 5.10745 5.10745 + 9.12589 9.12589 + 4.53119 4.53119 + -0.0952322 -0.0952322 + -1.67069 -1.67069 + 1.48937 1.48937 + 2.1548 2.1548 + -0.680895 -0.680895 + 6.00943 6.00943 + -6.23597 -6.23597 + 15.2635 15.2635 + -5.39621 -5.39621 + 2.9004 2.9004 + -7.2031 -7.2031 + 0.188095 0.188095 + -5.65511 -5.65511 + 8.80472 8.80472 + 4.77116 4.77116 + -0.320718 -0.320718 + -0.094774 -0.094774 + 4.24892 4.24892 + -0.729715 -0.729715 + 3.46906 3.46906 + -4.86913 -4.86913 + -2.05092 -2.05092 + 3.24008 3.24008 + 2.67334 2.67334 + 5.41008 5.41008 + 4.61387 4.61387 + -11.9338 -11.9338 + 2.15538 2.15538 + 3.39914 3.39914 + 2.71216 2.71216 + 6.79031 6.79031 + -0.750493 -0.750493 + -0.683416 -0.683416 + 7.23875 7.23875 + 4.67949 4.67949 + -2.16467 -2.16467 + 3.64787 3.64787 + -1.27823 -1.27823 + -1.43992 -1.43992 + 3.183 3.183 + -8.60412 -8.60412 + -5.42757 -5.42757 + -0.564214 -0.564214 + -1.17837 -1.17837 + 2.45248 2.45248 + 3.60909 3.60909 + 2.61183 2.61183 + 5.20279 5.20279 + -1.07145 -1.07145 + -0.919519 -0.919519 + 3.89898 3.89898 + 3.72175 3.72175 + -9.9673 -9.9673 + 1.50607 1.50607 + -0.456562 -0.456562 + 10.9984 10.9984 + -2.18673 -2.18673 + -7.39159 -7.39159 + -5.54389 -5.54389 + 2.6353 2.6353 + 6.87535 6.87535 + -10.4019 -10.4019 + -5.51375 -5.51375 + -3.33244 -3.33244 + 7.60358 7.60358 + -9.48529 -9.48529 + -0.514099 -0.514099 + 6.20569 6.20569 + -4.60198 -4.60198 + -1.28686 -1.28686 + -0.383981 -0.383981 + -0.173934 -0.173934 + -7.97782 -7.97782 + 5.9926 5.9926 + -3.7357 -3.7357 + -7.77841 -7.77841 + 3.09245 3.09245 + -3.70421 -3.70421 + -1.50012 -1.50012 + -3.90181 -3.90181 + 0.183002 0.183002 + -4.72374 -4.72374 + -3.36966 -3.36966 + 8.23642 8.23642 + 0.387898 0.387898 + -2.53048 -2.53048 + 4.46348 4.46348 + -0.932844 -0.932844 + -1.76804 -1.76804 + -0.390175 -0.390175 + 8.28101 8.28101 + 8.66959 8.66959 + 2.47585 2.47585 + 6.33837 6.33837 + 3.05846 3.05846 + 6.43047 6.43047 + 0.167477 0.167477 + 0.615034 0.615034 + -8.467 -8.467 + 2.15566 2.15566 + 6.59172 6.59172 + -8.30068 -8.30068 + -2.92268 -2.92268 + -1.14616 -1.14616 + 3.864 3.864 + -8.07267 -8.07267 + 0.382952 0.382952 + 4.79087 4.79087 + 7.87692 7.87692 + -1.27352 -1.27352 + -0.439992 -0.439992 + -0.361056 -0.361056 + 5.51463 5.51463 + 4.10827 4.10827 + -1.36056 -1.36056 + -10.9063 -10.9063 + -3.12566 -3.12566 + -1.52612 -1.52612 + 2.47429 2.47429 + 1.92973 1.92973 + 6.05399 6.05399 + 6.35717 6.35717 + -6.54112 -6.54112 + 0.16752 0.16752 + -0.581192 -0.581192 + -3.91981 -3.91981 + 3.29046 3.29046 + -9.85289 -9.85289 + -1.68008 -1.68008 + -0.294261 -0.294261 + -2.33446 -2.33446 + 8.72203 8.72203 + -7.53754 -7.53754 + 1.8548 1.8548 + 0.0863562 0.0863562 + 3.71224 3.71224 + -2.72156 -2.72156 + 6.92717 6.92717 + 4.22066 4.22066 + 2.9384 2.9384 + -0.436476 -0.436476 + 7.94505 7.94505 + 3.35167 3.35167 + 4.57606 4.57606 + -1.94551 -1.94551 + 7.26891 7.26891 + 5.7114 5.7114 + -4.8975 -4.8975 + 0.24802 0.24802 + 4.4272 4.4272 + 3.21714 3.21714 + -2.75997 -2.75997 + 3.0239 3.0239 + 6.00743 6.00743 + 1.95157 1.95157 + -8.23524 -8.23524 + -0.0388194 -0.0388194 + -1.59723 -1.59723 + -15.7227 -15.7227 + 5.01363 5.01363 + 2.59661 2.59661 + 0.344503 0.344503 + 7.85727 7.85727 + 0.142462 0.142462 + -3.54743 -3.54743 + -4.18558 -4.18558 + 3.96172 3.96172 + -0.376684 -0.376684 + 3.78763 3.78763 + -1.58384 -1.58384 + 15.837 15.837 + -0.887404 -0.887404 + 0.855016 0.855016 + 11.1701 11.1701 + 5.15206 5.15206 + 6.83176 6.83176 + -0.91331 -0.91331 + -10.3398 -10.3398 + 2.48231 2.48231 + -2.03572 -2.03572 + 1.09096 1.09096 + -0.162198 -0.162198 + -7.32758 -7.32758 + -6.97941 -6.97941 + 5.98831 5.98831 + -7.43703 -7.43703 + -8.97936 -8.97936 + 0.676949 0.676949 + 1.37291 1.37291 + 4.41159 4.41159 + 2.45643 2.45643 + 2.79374 2.79374 + 2.36712 2.36712 + -7.74483 -7.74483 + 0.602922 0.602922 + -2.48544 -2.48544 + 0.299035 0.299035 + 6.77695 6.77695 + 1.44763 1.44763 + 1.94637 1.94637 + -4.04181 -4.04181 + 16.3509 16.3509 + 6.4273 6.4273 + 5.41235 5.41235 + -5.91387 -5.91387 + -6.06301 -6.06301 + 3.4536 3.4536 + -3.39128 -3.39128 + 11.299 11.299 + 2.62685 2.62685 + 1.00866 1.00866 + 10.6766 10.6766 + -0.805083 -0.805083 + 3.91073 3.91073 + 3.67201 3.67201 + -9.14116 -9.14116 + 15.6406 15.6406 + 3.22084 3.22084 + -2.90513 -2.90513 + 4.58966 4.58966 + 0.0983211 0.0983211 + 2.35908 2.35908 + 0.658109 0.658109 + 2.37478 2.37478 + -6.70679 -6.70679 + 6.08307 6.08307 + -29.6624 -29.6624 + 1.55578 1.55578 + 5.31311 5.31311 + -5.40681 -5.40681 + 1.80228 1.80228 + 4.50431 4.50431 + 7.25673 7.25673 + 5.89811 5.89811 + -2.92888 -2.92888 + 7.48853 7.48853 + -1.67318 -1.67318 + 0.974302 0.974302 + -8.10178 -8.10178 + 3.29435 3.29435 + -1.64519 -1.64519 + -7.08854 -7.08854 + 6.68891 6.68891 + -5.69927 -5.69927 + -3.51768 -3.51768 + 11.2895 11.2895 + -0.828568 -0.828568 + 5.53562 5.53562 + -0.358066 -0.358066 + -5.92559 -5.92559 + 4.39224 4.39224 + -5.1225 -5.1225 + -9.51174 -9.51174 + 9.80076 9.80076 + -1.85858 -1.85858 + 6.95181 6.95181 + -1.71297 -1.71297 + -0.275297 -0.275297 + -0.860135 -0.860135 + -0.484906 -0.484906 + 5.71425 5.71425 + 2.74639 2.74639 + -8.40417 -8.40417 + -1.84935 -1.84935 + 2.94526 2.94526 + 10.708 10.708 + 0.892511 0.892511 + -1.36773 -1.36773 + -7.25911 -7.25911 + 3.91428 3.91428 + -0.776027 -0.776027 + 3.44102 3.44102 + -4.87806 -4.87806 + 3.65101 3.65101 + -3.01077 -3.01077 + 1.17918 1.17918 + 5.82266 5.82266 + 8.52564 8.52564 + 4.35296 4.35296 + -2.94897 -2.94897 + -4.19366 -4.19366 + -4.7939 -4.7939 + 3.44038 3.44038 + -7.87089 -7.87089 + -3.18931 -3.18931 + -6.65708 -6.65708 + 1.09687 1.09687 + -4.36662 -4.36662 + 2.90783 2.90783 + 4.66889 4.66889 + -1.26146 -1.26146 + -2.01469 -2.01469 + -2.44566 -2.44566 + -2.15098 -2.15098 + 3.4006 3.4006 + 0.0396139 0.0396139 + 2.29469 2.29469 + -7.62709 -7.62709 + 7.18738 7.18738 + 1.45481 1.45481 + 2.37791 2.37791 + -5.37208 -5.37208 + -0.0612415 -0.0612415 + -1.46115 -1.46115 + 4.29624 4.29624 + 3.25993 3.25993 + 2.42986 2.42986 + 6.56133 6.56133 + -2.07349 -2.07349 + 5.61643 5.61643 + 5.48251 5.48251 + -0.703666 -0.703666 + -5.09456 -5.09456 + 0.57249 0.57249 + 4.28577 4.28577 + 2.468 2.468 + -10.013 -10.013 + -3.26046 -3.26046 + -7.91038 -7.91038 + -2.03302 -2.03302 + 3.49234 3.49234 + -1.2481 -1.2481 + -1.87417 -1.87417 + -1.93016 -1.93016 + 2.14307 2.14307 + -9.0722 -9.0722 + 2.03124 2.03124 + -0.938906 -0.938906 + 0.817464 0.817464 + 2.23636 2.23636 + 1.3076 1.3076 + 4.90629 4.90629 + 2.16603 2.16603 + 5.84398 5.84398 + -6.56748 -6.56748 + 7.22968 7.22968 + 0.664381 0.664381 + 11.2001 11.2001 + -4.98902 -4.98902 + 0.841822 0.841822 + -1.35522 -1.35522 + -2.43996 -2.43996 + 5.14732 5.14732 + -7.50974 -7.50974 + 5.73113 5.73113 + -2.72015 -2.72015 + -5.04474 -5.04474 + -13.1 -13.1 + 0.0777815 0.0777815 + 7.85631 7.85631 + -0.323243 -0.323243 + -2.97974 -2.97974 + 0.925187 0.925187 + 5.77219 5.77219 + 4.39868 4.39868 + 2.22326 2.22326 + 1.79052 1.79052 + -3.37507 -3.37507 + -4.08645 -4.08645 + 5.59349 5.59349 + 11.879 11.879 + -0.8099 -0.8099 + 16.6866 16.6866 + 2.85772 2.85772 + 3.73902 3.73902 + -0.406009 -0.406009 + 7.49033 7.49033 + -1.01733 -1.01733 + 4.03678 4.03678 + 4.91574 4.91574 + 14.6191 14.6191 + -1.18215 -1.18215 + -2.79895 -2.79895 + -5.16604 -5.16604 + -2.24596 -2.24596 + 1.83945 1.83945 + 1.72673 1.72673 + -23.2963 -23.2963 + -0.623748 -0.623748 + -2.8419 -2.8419 + 6.56374 6.56374 + 10.3431 10.3431 + 5.28302 5.28302 + 3.12716 3.12716 + 8.41242 8.41242 + 0.416003 0.416003 + -2.43236 -2.43236 + -1.63284 -1.63284 + 5.3806 5.3806 + 9.39975 9.39975 + 4.44496 4.44496 + -3.01441 -3.01441 + -1.33538 -1.33538 + 2.23541 2.23541 + -4.30131 -4.30131 + -1.20324 -1.20324 + 4.79406 4.79406 + 0.692551 0.692551 + -2.20403 -2.20403 + 0.12931 0.12931 + 0.842875 0.842875 + 0.29791 0.29791 + 6.59639 6.59639 + 8.6591 8.6591 + 2.07311 2.07311 + -6.48842 -6.48842 + 2.70007 2.70007 + -0.143695 -0.143695 + 3.99651 3.99651 + 6.86089 6.86089 + -2.54281 -2.54281 + -5.085 -5.085 + 3.61747 3.61747 + 2.09466 2.09466 + 3.35667 3.35667 + 7.38405 7.38405 + 0.816999 0.816999 + -0.564258 -0.564258 + 2.46281 2.46281 + -0.081471 -0.081471 + 12.0933 12.0933 + 9.45364 9.45364 + 0.303564 0.303564 + -2.20687 -2.20687 + 1.90101 1.90101 + -2.65606 -2.65606 + -11.3589 -11.3589 + -1.68249 -1.68249 + -1.25813 -1.25813 + -0.96125 -0.96125 + -2.84666 -2.84666 + 1.18914 1.18914 + 0.211945 0.211945 + -4.8988 -4.8988 + 0.894798 0.894798 + 3.9685 3.9685 + -0.852608 -0.852608 + 3.37537 3.37537 + -0.847579 -0.847579 + -4.37006 -4.37006 + -4.12787 -4.12787 + 4.37155 4.37155 + -7.86631 -7.86631 + -3.59755 -3.59755 + -2.55397 -2.55397 + 4.25921 4.25921 + 2.21721 2.21721 + 5.72299 5.72299 + 8.32362 8.32362 + 14.4057 14.4057 + 1.49376 1.49376 + 3.108 3.108 + -1.34388 -1.34388 + 3.77816 3.77816 + 5.69761 5.69761 + 0.255491 0.255491 + 4.15979 4.15979 + -14.6016 -14.6016 + 3.1475 3.1475 + 2.86732 2.86732 + -2.7875 -2.7875 + -8.78827 -8.78827 + -1.38068 -1.38068 + -2.74156 -2.74156 + -4.82257 -4.82257 + -4.64984 -4.64984 + -0.462036 -0.462036 + 2.36274 2.36274 + 2.73927 2.73927 + -4.01583 -4.01583 + -4.20256 -4.20256 + 7.33455 7.33455 + 7.53557 7.53557 + 3.2532 3.2532 + -0.556551 -0.556551 + 4.39618 4.39618 + 2.92025 2.92025 + -49.4395 -49.4395 + 1.84066 1.84066 + -6.03682 -6.03682 + 9.70956 9.70956 + 12.18 12.18 + -0.134471 -0.134471 + 0.388477 0.388477 + -4.30526 -4.30526 + 3.98614 3.98614 + -3.20351 -3.20351 + 3.81764 3.81764 + 5.34853 5.34853 + 0.382215 0.382215 + -0.473372 -0.473372 + -4.4073 -4.4073 + -10.1129 -10.1129 + -6.82482 -6.82482 + 5.39935 5.39935 + -0.664077 -0.664077 + 7.75577 7.75577 + -5.565 -5.565 + -2.28518 -2.28518 + -3.09472 -3.09472 + 6.0196 6.0196 + -1.32035 -1.32035 + 2.5721 2.5721 + -9.0201 -9.0201 + 6.87621 6.87621 + 7.57662 7.57662 + -2.42131 -2.42131 + -7.11 -7.11 + 1.5457 1.5457 + 1.38686 1.38686 + -1.67077 -1.67077 + 5.34357 5.34357 + -5.22992 -5.22992 + -5.50112 -5.50112 + -0.820436 -0.820436 + -6.85987 -6.85987 + 4.36935 4.36935 + 8.27737 8.27737 + 7.16613 7.16613 + 7.21538 7.21538 + 0.0297893 0.0297893 + -3.30991 -3.30991 + 1.18508 1.18508 + -0.745072 -0.745072 + -1.31153 -1.31153 + -2.57184 -2.57184 + -0.187369 -0.187369 + 6.79233 6.79233 + 8.04294 8.04294 + 3.06986 3.06986 + -5.13761 -5.13761 + 0.539648 0.539648 + 5.02007 5.02007 + 2.67737 2.67737 + -6.69984 -6.69984 + 6.76321 6.76321 + 6.25102 6.25102 + 3.80545 3.80545 + -2.16059 -2.16059 + 2.81803 2.81803 + 0.447194 0.447194 + 1.84756 1.84756 + -6.42528 -6.42528 + -2.23379 -2.23379 + -2.61151 -2.61151 + -2.86143 -2.86143 + -2.94039 -2.94039 + -3.38503 -3.38503 + 0.474985 0.474985 + -9.66389 -9.66389 + 4.96293 4.96293 + -5.6718 -5.6718 + 7.06422 7.06422 + -8.36354 -8.36354 + 0.0182466 0.0182466 + 9.20883 9.20883 + 8.23981 8.23981 + -1.41968 -1.41968 + -1.36057 -1.36057 + -3.99568 -3.99568 + 2.51484 2.51484 + 5.41846 5.41846 + -10.8511 -10.8511 + -8.41267 -8.41267 + 2.04668 2.04668 + -5.61525 -5.61525 + -9.73507 -9.73507 + -0.497102 -0.497102 + 4.29467 4.29467 + -1.61424 -1.61424 + -0.818494 -0.818494 + -7.02135 -7.02135 + 13.4836 13.4836 + -4.10115 -4.10115 + -8.11914 -8.11914 + -2.79895 -2.79895 + -4.39428 -4.39428 + -0.737467 -0.737467 + 1.37013 1.37013 + 9.56244 9.56244 + 2.92491 2.92491 + -7.13393 -7.13393 + -0.179291 -0.179291 + -6.00313 -6.00313 + 7.27104 7.27104 + -1.7103 -1.7103 + -7.84843 -7.84843 + 13.7304 13.7304 + 2.40973 2.40973 + -7.07755 -7.07755 + 1.31745 1.31745 + -9.99271 -9.99271 + -15.4753 -15.4753 + 4.38711 4.38711 + -5.41127 -5.41127 + -1.06491 -1.06491 + 1.09245 1.09245 + -1.33961 -1.33961 + -4.42681 -4.42681 + -4.44164 -4.44164 + -1.80772 -1.80772 + -5.06035 -5.06035 + 0.197369 0.197369 + 7.27798 7.27798 + -6.88382 -6.88382 + 3.21319 3.21319 + 8.04111 8.04111 + -3.94107 -3.94107 + 1.79716 1.79716 + -0.2134 -0.2134 + 1.36955 1.36955 + 13.7009 13.7009 + -7.3497 -7.3497 + 1.80078 1.80078 + 4.25352 4.25352 + -2.80092 -2.80092 + -3.81295 -3.81295 + -4.92036 -4.92036 + 0.856001 0.856001 + -1.26696 -1.26696 + 2.65207 2.65207 + -1.01876 -1.01876 + 1.50837 1.50837 + -11.5335 -11.5335 + 5.80989 5.80989 + 2.45606 2.45606 + 1.64394 1.64394 + 2.73651 2.73651 + -11.1653 -11.1653 + -1.66359 -1.66359 + -0.0317267 -0.0317267 + 0.115458 0.115458 + 4.43585 4.43585 + 1.24902 1.24902 + 7.30894 7.30894 + 16.7814 16.7814 + -0.456154 -0.456154 + -3.94033 -3.94033 + -4.4947 -4.4947 + -2.52048 -2.52048 + 0.0890704 0.0890704 + -4.66338 -4.66338 + 3.88142 3.88142 + 2.35984 2.35984 + 4.84037 4.84037 + 6.95444 6.95444 + 2.74408 2.74408 + -3.23958 -3.23958 + -0.467292 -0.467292 + 6.26367 6.26367 + -1.50588 -1.50588 + 4.13389 4.13389 + -2.53819 -2.53819 + -4.4987 -4.4987 + -10.3487 -10.3487 + -14.8297 -14.8297 + -8.48112 -8.48112 + 3.95155 3.95155 + 1.2289 1.2289 + -4.38025 -4.38025 + -0.61687 -0.61687 + 10.8511 10.8511 + 1.15556 1.15556 + -2.19768 -2.19768 + -7.66931 -7.66931 + 4.72919 4.72919 + -7.6738 -7.6738 + -0.688528 -0.688528 + 4.74928 4.74928 + 4.92126 4.92126 + 0.897546 0.897546 + 3.85735 3.85735 + 0.201364 0.201364 + -5.62425 -5.62425 + -3.83117 -3.83117 + 4.05866 4.05866 + 3.10063 3.10063 + 2.5224 2.5224 + -1.51274 -1.51274 + -0.683338 -0.683338 + -3.23147 -3.23147 + -4.21268 -4.21268 + -2.21401 -2.21401 + 1.57887 1.57887 + 0.848257 0.848257 + -5.83704 -5.83704 + -7.00011 -7.00011 + 3.16884 3.16884 + -4.44161 -4.44161 + -7.62482 -7.62482 + -0.266943 -0.266943 + 0.41761 0.41761 + -7.45144 -7.45144 + -0.211132 -0.211132 + 0.276707 0.276707 + 16.7781 16.7781 + 0.689757 0.689757 + -3.04049 -3.04049 + 2.91684 2.91684 + 1.97161 1.97161 + 3.7721 3.7721 + -1.60698 -1.60698 + -4.18868 -4.18868 + 7.66491 7.66491 + -0.64664 -0.64664 + -0.660623 -0.660623 + 8.68174 8.68174 + 0.282074 0.282074 + -2.85266 -2.85266 + -1.91293 -1.91293 + 7.18736 7.18736 + -10.3875 -10.3875 + -1.91603 -1.91603 + 6.29739 6.29739 + -0.0375388 -0.0375388 + -1.60576 -1.60576 + -3.22148 -3.22148 + -4.24549 -4.24549 + 1.30822 1.30822 + 2.52307 2.52307 + 0.403345 0.403345 + -0.744478 -0.744478 + 2.41241 2.41241 + -4.58098 -4.58098 + -0.791842 -0.791842 + 3.73626 3.73626 + -1.43002 -1.43002 + 4.30716 4.30716 + 3.30255 3.30255 + -4.08011 -4.08011 + -5.07282 -5.07282 + -1.54759 -1.54759 + -2.2305 -2.2305 + 6.8791 6.8791 + 9.7396 9.7396 + -6.50395 -6.50395 + 3.57178 3.57178 + 7.08987 7.08987 + 6.2669 6.2669 + 5.87329 5.87329 + 2.36823 2.36823 + -6.16 -6.16 + 1.96238 1.96238 + 7.31651 7.31651 + -1.5257 -1.5257 + -2.89061 -2.89061 + 0.407546 0.407546 + 5.10645 5.10645 + 11.0716 11.0716 + 4.7443 4.7443 + -8.77353 -8.77353 + -0.631177 -0.631177 + -4.36973 -4.36973 + 1.48666 1.48666 + 7.7678 7.7678 + -2.65407 -2.65407 + 4.56869 4.56869 + -0.541163 -0.541163 + 2.89543 2.89543 + 5.39424 5.39424 + -3.62954 -3.62954 + 3.77547 3.77547 + -5.96886 -5.96886 + -4.38947 -4.38947 + -2.96756 -2.96756 + 2.28222 2.28222 + -1.08489 -1.08489 + 1.74726 1.74726 + -3.46088 -3.46088 + 11.9371 11.9371 + -5.02359 -5.02359 + 2.51632 2.51632 + -0.0297022 -0.0297022 + -2.60011 -2.60011 + 0.254202 0.254202 + 9.7949 9.7949 + 3.64937 3.64937 + 10.0857 10.0857 + -5.36637 -5.36637 + 4.11127 4.11127 + 8.90571 8.90571 + -5.97219 -5.97219 + -7.21379 -7.21379 + -5.01561 -5.01561 + 2.98616 2.98616 + 1.99064 1.99064 + 0.16465 0.16465 + -4.07902 -4.07902 + 4.34018 4.34018 + -2.13528 -2.13528 + 2.39903 2.39903 + 4.00804 4.00804 + -1.85741 -1.85741 + -7.73083 -7.73083 + -4.21139 -4.21139 + 4.65743 4.65743 + 0.963549 0.963549 + 0.29506 0.29506 + 6.05798 6.05798 + 12.4428 12.4428 + -0.398651 -0.398651 + -0.584559 -0.584559 + 2.75445 2.75445 + -0.207975 -0.207975 + 6.11926 6.11926 + -8.66125 -8.66125 + 3.07568 3.07568 + -3.19358 -3.19358 + -2.53024 -2.53024 + 14.1187 14.1187 + -0.412049 -0.412049 + 12.5809 12.5809 + 6.26236 6.26236 + 5.23037 5.23037 + -0.11356 -0.11356 + -6.62321 -6.62321 + -1.29651 -1.29651 + -1.48734 -1.48734 + 13.0753 13.0753 + 4.21767 4.21767 + -2.4425 -2.4425 + -0.0901323 -0.0901323 + 9.79684 9.79684 + 4.74522 4.74522 + -3.34804 -3.34804 + 7.37816 7.37816 + 2.57938 2.57938 + 1.92968 1.92968 + 3.75166 3.75166 + 5.0617 5.0617 + 8.74324 8.74324 + -0.93703 -0.93703 + -1.36031 -1.36031 + -2.5439 -2.5439 + 1.56784 1.56784 + 2.56237 2.56237 + -1.02578 -1.02578 + 6.62085 6.62085 + 7.69745 7.69745 + 6.26864 6.26864 + -4.20046 -4.20046 + -2.30926 -2.30926 + 2.74598 2.74598 + 4.11078 4.11078 + 2.8455 2.8455 + -3.45407 -3.45407 + 2.82327 2.82327 + -1.00356 -1.00356 + 8.85974 8.85974 + 6.35864 6.35864 + -1.59146 -1.59146 + -0.361996 -0.361996 + -1.25198 -1.25198 + 8.2867 8.2867 + 0.981644 0.981644 + 2.68003 2.68003 + 1.10236 1.10236 + -1.63423 -1.63423 + -2.79552 -2.79552 + -6.5718 -6.5718 + -0.257779 -0.257779 + -4.49325 -4.49325 + 5.0455 5.0455 + 14.4508 14.4508 + 3.60407 3.60407 + 3.09003 3.09003 + -8.32962 -8.32962 + -1.41178 -1.41178 + 12.5777 12.5777 + -2.01342 -2.01342 + -1.48205 -1.48205 + 0.967158 0.967158 + -0.532548 -0.532548 + -5.23274 -5.23274 + -1.49702 -1.49702 + 0.739607 0.739607 + 3.49171 3.49171 + -1.0507 -1.0507 + -7.48299 -7.48299 + 7.57395 7.57395 + -3.04813 -3.04813 + 16.322 16.322 + 7.81441 7.81441 + -3.41529 -3.41529 + 2.05401 2.05401 + 1.08232 1.08232 + 12.5735 12.5735 + 0.126572 0.126572 + -6.92158 -6.92158 + -1.4651 -1.4651 + -3.19425 -3.19425 + -1.44093 -1.44093 + -3.82056 -3.82056 + 6.72914 6.72914 + -5.46583 -5.46583 + -1.43396 -1.43396 + 7.42164 7.42164 + 1.00438 1.00438 + -0.41415 -0.41415 + -2.54987 -2.54987 + 6.88491 6.88491 + 3.84807 3.84807 + -5.62245 -5.62245 + 5.24133 5.24133 + 7.99514 7.99514 + -2.51593 -2.51593 + 8.19568 8.19568 + 0.854985 0.854985 + -6.20478 -6.20478 + -2.58235 -2.58235 + -6.51346 -6.51346 + 12.8877 12.8877 + 8.6194 8.6194 + -6.82669 -6.82669 + -4.67379 -4.67379 + 8.13137 8.13137 + 0.733511 0.733511 + 5.66079 5.66079 + -2.94337 -2.94337 + -3.29462 -3.29462 + -6.3809 -6.3809 + -1.85613 -1.85613 + 0.635069 0.635069 + 0.432626 0.432626 + -14.6426 -14.6426 + 8.05825 8.05825 + 6.50637 6.50637 + 1.44014 1.44014 + -4.60602 -4.60602 + -6.49137 -6.49137 + 6.33163 6.33163 + -1.97616 -1.97616 + 0.573379 0.573379 + -2.78039 -2.78039 + -0.140087 -0.140087 + 1.52619 1.52619 + 6.83379 6.83379 + -0.197981 -0.197981 + -3.00849 -3.00849 + -2.09725 -2.09725 + -2.06883 -2.06883 + -0.328198 -0.328198 + -0.212338 -0.212338 + 5.4425 5.4425 + 6.48574 6.48574 + 2.00073 2.00073 + -3.15642 -3.15642 + -0.0673389 -0.0673389 + -4.19911 -4.19911 + 4.5466 4.5466 + 3.73221 3.73221 + -1.01059 -1.01059 + -4.29015 -4.29015 + 4.9909 4.9909 + 3.22397 3.22397 + -1.27984 -1.27984 + 2.83358 2.83358 + 2.25695 2.25695 + 7.2879 7.2879 + -1.47955 -1.47955 + 12.7627 12.7627 + -3.72449 -3.72449 + 3.97719 3.97719 + 14.2197 14.2197 + -1.24031 -1.24031 + -7.41824 -7.41824 + 1.90207 1.90207 + 1.10939 1.10939 + -7.47202 -7.47202 + 3.85738 3.85738 + -4.12085 -4.12085 + 1.12097 1.12097 + -0.545646 -0.545646 + 3.04129 3.04129 + 1.05043 1.05043 + 0.993448 0.993448 + -5.78424 -5.78424 + -1.97199 -1.97199 + -5.74806 -5.74806 + 2.70835 2.70835 + -8.09729 -8.09729 + -6.36035 -6.36035 + -1.24361 -1.24361 + -2.44813 -2.44813 + 7.48353 7.48353 + 2.0202 2.0202 + 3.04366 3.04366 + -3.98778 -3.98778 + 4.80106 4.80106 + 0.926552 0.926552 + 3.35253 3.35253 + -4.10577 -4.10577 + -3.57853 -3.57853 + 4.03372 4.03372 + -2.38792 -2.38792 + 0.12177 0.12177 + -0.761671 -0.761671 + -4.25652 -4.25652 + 7.27933 7.27933 + 0.165182 0.165182 + 1.34367 1.34367 + -7.36923 -7.36923 + 2.38548 2.38548 + 0.117217 0.117217 + 2.02002 2.02002 + -4.60023 -4.60023 + 2.78 2.78 + -1.34604 -1.34604 + 4.7234 4.7234 + 7.37673 7.37673 + 2.07986 2.07986 + -5.72573 -5.72573 + -6.66143 -6.66143 + 2.43072 2.43072 + 1.34782 1.34782 + -0.114238 -0.114238 + 2.32103 2.32103 + 1.84042 1.84042 + 1.07005 1.07005 + 3.88182 3.88182 + -0.752264 -0.752264 + -2.43517 -2.43517 + -5.29216 -5.29216 + -0.13527 -0.13527 + 1.40188 1.40188 + -5.87815 -5.87815 + -1.90167 -1.90167 + 2.88562 2.88562 + -2.29028 -2.29028 + 2.35477 2.35477 + -3.50731 -3.50731 + 6.0621 6.0621 + 3.2011 3.2011 + 2.19115 2.19115 + -3.03557 -3.03557 + -8.49394 -8.49394 + 0.936501 0.936501 + 7.19188 7.19188 + 4.50162 4.50162 + 0.341394 0.341394 + 2.54484 2.54484 + 1.67305 1.67305 + 3.05008 3.05008 + -2.0266 -2.0266 + 7.28431 7.28431 + -7.70924 -7.70924 + 2.60851 2.60851 + 6.8054 6.8054 + 1.8878 1.8878 + 1.87624 1.87624 + -5.13611 -5.13611 + -3.23698 -3.23698 + 4.03201 4.03201 + -5.27165 -5.27165 + -4.95817 -4.95817 + -0.200461 -0.200461 + 4.27259 4.27259 + 0.449661 0.449661 + 7.49752 7.49752 + -5.47923 -5.47923 + -2.40934 -2.40934 + 25.0066 25.0066 + -3.14511 -3.14511 + -1.62587 -1.62587 + -1.67652 -1.67652 + -2.17888 -2.17888 + 2.37296 2.37296 + -4.41408 -4.41408 + 0.65204 0.65204 + 10.849 10.849 + -2.3021 -2.3021 + 2.20417 2.20417 + 10.0579 10.0579 + -4.03489 -4.03489 + 7.60982 7.60982 + -5.74951 -5.74951 + -2.97582 -2.97582 + -8.61382 -8.61382 + -1.90903 -1.90903 + -3.64556 -3.64556 + -16.2304 -16.2304 + -15.9793 -15.9793 + -4.59448 -4.59448 + -2.67688 -2.67688 + -1.67148 -1.67148 + 5.57026 5.57026 + 0.846445 0.846445 + -7.54149 -7.54149 + -3.61401 -3.61401 + 4.03723 4.03723 + 0.711821 0.711821 + 8.99009 8.99009 + -6.15866 -6.15866 + -1.36865 -1.36865 + -4.31058 -4.31058 + 6.31659 6.31659 + -6.23773 -6.23773 + 0.857388 0.857388 + 3.6152 3.6152 + -1.28774 -1.28774 + -4.92094 -4.92094 + 3.08527 3.08527 + -5.74582 -5.74582 + -4.20897 -4.20897 + -5.19406 -5.19406 + -4.06851 -4.06851 + 5.73867 5.73867 + 3.32767 3.32767 + -11.2588 -11.2588 + -7.94126 -7.94126 + 5.38746 5.38746 + -0.0253579 -0.0253579 + -1.7856 -1.7856 + -1.31209 -1.31209 + 6.85519 6.85519 + 2.71496 2.71496 + -2.58838 -2.58838 + -6.86996 -6.86996 + 1.01204 1.01204 + 3.43433 3.43433 + -0.249192 -0.249192 + 7.96322 7.96322 + 14.3414 14.3414 + 2.44774 2.44774 + 4.73731 4.73731 + -9.14288 -9.14288 + 2.70325 2.70325 + 6.48202 6.48202 + -2.58391 -2.58391 + -4.52079 -4.52079 + -0.64105 -0.64105 + -3.75531 -3.75531 + -3.93321 -3.93321 + -2.5879 -2.5879 + 2.34697 2.34697 + -3.89721 -3.89721 + -1.60712 -1.60712 + -7.49452 -7.49452 + -0.518596 -0.518596 + 0.996693 0.996693 + 2.83468 2.83468 + -6.19363 -6.19363 + -7.25683 -7.25683 + 0.391546 0.391546 + -7.52756 -7.52756 + -0.810817 -0.810817 + -2.64942 -2.64942 + -2.95081 -2.95081 + -6.34989 -6.34989 + 3.9961 3.9961 + 1.36755 1.36755 + -0.335808 -0.335808 + -11.7919 -11.7919 + 1.16904 1.16904 + 6.26031 6.26031 + -4.68064 -4.68064 + 5.55008 5.55008 + 3.65873 3.65873 + -3.95177 -3.95177 + 7.62708 7.62708 + -2.4932 -2.4932 + -0.713266 -0.713266 + 6.76214 6.76214 + -0.802523 -0.802523 + -0.327543 -0.327543 + -6.9053 -6.9053 + -2.69604 -2.69604 + 9.729 9.729 + -7.61691 -7.61691 + -0.658653 -0.658653 + 1.62531 1.62531 + 0.532107 0.532107 + 1.71729 1.71729 + -10.1795 -10.1795 + 5.54208 5.54208 + 4.02502 4.02502 + -1.47596 -1.47596 + 11.818 11.818 + 4.40414 4.40414 + 5.64827 5.64827 + 5.89386 5.89386 + -6.19187 -6.19187 + 4.77889 4.77889 + -0.261731 -0.261731 + -0.570525 -0.570525 + 3.80941 3.80941 + -3.95414 -3.95414 + 0.642971 0.642971 + -7.23493 -7.23493 + 0.744423 0.744423 + 11.5682 11.5682 + -3.17145 -3.17145 + 9.02877 9.02877 + 10.5452 10.5452 + -7.05642 -7.05642 + -6.01952 -6.01952 + -5.61355 -5.61355 + 1.28759 1.28759 + 3.44186 3.44186 + -2.52363 -2.52363 + 8.95712 8.95712 + -1.33999 -1.33999 + -3.25858 -3.25858 + 2.33509 2.33509 + 2.16314 2.16314 + 14.4002 14.4002 + -5.22345 -5.22345 + -5.6232 -5.6232 + -4.20801 -4.20801 + 0.677359 0.677359 + 1.92688 1.92688 + 2.4265 2.4265 + -3.47901 -3.47901 + -3.35004 -3.35004 + -5.32445 -5.32445 + 0.817822 0.817822 + 5.9241 5.9241 + 2.13342 2.13342 + 9.30726 9.30726 + -6.00328 -6.00328 + 5.10125 5.10125 + 16.6941 16.6941 + -1.41774 -1.41774 + 0.843709 0.843709 + 3.71326 3.71326 + -12.7315 -12.7315 + -1.58947 -1.58947 + 2.7713 2.7713 + -5.89993 -5.89993 + -10.1427 -10.1427 + -1.60823 -1.60823 + -4.98621 -4.98621 + -10.6258 -10.6258 + 0.255858 0.255858 + 5.87781 5.87781 + 0.549239 0.549239 + -0.361649 -0.361649 + 2.89543 2.89543 + -1.56252 -1.56252 + -7.04269 -7.04269 + 0.360599 0.360599 + -0.80318 -0.80318 + -8.15537 -8.15537 + 7.86106 7.86106 + 4.25906 4.25906 + 1.78474 1.78474 + 4.15764 4.15764 + -1.8884 -1.8884 + -7.16959 -7.16959 + 2.84539 2.84539 + -3.33161 -3.33161 + 4.89863 4.89863 + -3.36503 -3.36503 + -4.68013 -4.68013 + 5.18058 5.18058 + -9.69276 -9.69276 + -1.56116 -1.56116 + -3.58275 -3.58275 + -2.73766 -2.73766 + 6.64492 6.64492 + -3.78966 -3.78966 + 2.63467 2.63467 + -12.4868 -12.4868 + -3.4241 -3.4241 + 3.2898 3.2898 + 2.20265 2.20265 + -1.36672 -1.36672 + 2.71448 2.71448 + 5.87839 5.87839 + 0.160837 0.160837 + -2.64458 -2.64458 + -3.8078 -3.8078 + 5.08743 5.08743 + -14.014 -14.014 + 4.44746 4.44746 + 6.61584 6.61584 + -0.916513 -0.916513 + -8.08277 -8.08277 + -8.088 -8.088 + -5.14152 -5.14152 + -4.30739 -4.30739 + -8.76727 -8.76727 + -4.53313 -4.53313 + 11.0356 11.0356 + -2.37348 -2.37348 + -8.71711 -8.71711 + -2.22971 -2.22971 + 8.19346 8.19346 + -0.330962 -0.330962 + 1.10067 1.10067 + 1.01878 1.01878 + -10.2666 -10.2666 + 8.15909 8.15909 + 9.09316 9.09316 + -0.862864 -0.862864 + -7.54443 -7.54443 + -3.44703 -3.44703 + 5.21819 5.21819 + -2.06834 -2.06834 + 9.55442 9.55442 + -1.89649 -1.89649 + -5.57892 -5.57892 + 4.22421 4.22421 + -4.06375 -4.06375 + 3.81452 3.81452 + 3.09071 3.09071 + -7.34297 -7.34297 + -1.67899 -1.67899 + 0.58489 0.58489 + -5.33824 -5.33824 + 2.82705 2.82705 + -3.70864 -3.70864 + 4.21641 4.21641 + 3.82508 3.82508 + -4.04356 -4.04356 + 20.0249 20.0249 + -13.1531 -13.1531 + 2.98603 2.98603 + 5.54713 5.54713 + -1.39722 -1.39722 + 2.13016 2.13016 + -2.40215 -2.40215 + 0.168123 0.168123 + 2.77021 2.77021 + -2.32327 -2.32327 + -1.06731 -1.06731 + 2.53877 2.53877 + -1.94325 -1.94325 + 1.47106 1.47106 + 0.294436 0.294436 + -0.547055 -0.547055 + 0.116016 0.116016 + 1.56148 1.56148 + 3.21789 3.21789 + -2.89007 -2.89007 + -4.33765 -4.33765 + 0.566163 0.566163 + 0.402729 0.402729 + -7.80674 -7.80674 + 4.72058 4.72058 + 3.97584 3.97584 + 1.91646 1.91646 + 2.09298 2.09298 + 1.88552 1.88552 + -2.37581 -2.37581 + -18.2615 -18.2615 + 2.68651 2.68651 + 5.5 5.5 + 0.355051 0.355051 + 5.6052 5.6052 + 7.74854 7.74854 + -0.512378 -0.512378 + 1.60299 1.60299 + -5.49563 -5.49563 + -1.96455 -1.96455 + -16.3228 -16.3228 + -6.87737 -6.87737 + -4.60755 -4.60755 + -1.32116 -1.32116 + 2.87263 2.87263 + -2.09541 -2.09541 + 3.43595 3.43595 + 3.63528 3.63528 + 3.52056 3.52056 + -3.59484 -3.59484 + 1.03764 1.03764 + -7.14947 -7.14947 + -5.80634 -5.80634 + 4.71397 4.71397 + 0.720588 0.720588 + -2.24074 -2.24074 + 5.82418 5.82418 + -3.22013 -3.22013 + 3.68858 3.68858 + -1.43166 -1.43166 + 4.47978 4.47978 + -4.83356 -4.83356 + -3.96257 -3.96257 + -5.95512 -5.95512 + 0.496691 0.496691 + -7.58825 -7.58825 + -6.47331 -6.47331 + -1.14446 -1.14446 + 3.91615 3.91615 + -0.588841 -0.588841 + 6.56683 6.56683 + 3.97252 3.97252 + -4.3126 -4.3126 + -8.20913 -8.20913 + 0.310182 0.310182 + -7.3006 -7.3006 + 7.92805 7.92805 + 2.1756 2.1756 + 1.06404 1.06404 + 1.14471 1.14471 + -1.50242 -1.50242 + 0.00723557 0.00723557 + 5.76841 5.76841 + -1.96707 -1.96707 + 8.87243 8.87243 + -3.23281 -3.23281 + 12.3087 12.3087 + 3.3245 3.3245 + 3.00334 3.00334 + -5.74048 -5.74048 + 7.43939 7.43939 + -0.906001 -0.906001 + 2.24067 2.24067 + -6.23989 -6.23989 + 2.81483 2.81483 + -1.62648 -1.62648 + -7.26368 -7.26368 + 1.69171 1.69171 + -11.2631 -11.2631 + -2.32992 -2.32992 + -6.07361 -6.07361 + -7.56822 -7.56822 + -7.56737 -7.56737 + 5.97037 5.97037 + 6.74398 6.74398 + -2.24599 -2.24599 + 2.95213 2.95213 + -12.7864 -12.7864 + 0.680035 0.680035 + -1.39988 -1.39988 + -4.74028 -4.74028 + 3.01887 3.01887 + 1.89636 1.89636 + 4.46014 4.46014 + -4.38308 -4.38308 + 11.7633 11.7633 + -3.54671 -3.54671 + -3.47584 -3.47584 + 3.80037 3.80037 + 7.77849 7.77849 + -7.00006 -7.00006 + -4.87665 -4.87665 + -4.54736 -4.54736 + -7.81752 -7.81752 + -0.0654465 -0.0654465 + -3.70587 -3.70587 + -2.24231 -2.24231 + 5.58005 5.58005 + -3.09415 -3.09415 + -5.55063 -5.55063 + -4.19666 -4.19666 + -6.83328 -6.83328 + -6.9216 -6.9216 + -3.72782 -3.72782 + -2.18574 -2.18574 + 1.28076 1.28076 + -3.40691 -3.40691 + 0.486964 0.486964 + -2.11025 -2.11025 + -1.42349 -1.42349 + 6.06854 6.06854 + -1.37534 -1.37534 + 9.47832 9.47832 + -0.567045 -0.567045 + -6.98328 -6.98328 + 6.73139 6.73139 + -1.56812 -1.56812 + 0.141683 0.141683 + 1.78697 1.78697 + -2.03874 -2.03874 + 1.28356 1.28356 + 6.9912 6.9912 + -3.8858 -3.8858 + -1.38808 -1.38808 + -2.16632 -2.16632 + 3.57955 3.57955 + 2.73506 2.73506 + -3.03108 -3.03108 + -3.44677 -3.44677 + 1.37111 1.37111 + -10.0008 -10.0008 + -3.61651 -3.61651 + 1.97313 1.97313 + 2.11298 2.11298 + 0.174957 0.174957 + -0.131546 -0.131546 + 7.58484 7.58484 + 4.27907 4.27907 + 0.855439 0.855439 + 4.44153 4.44153 + -1.04577 -1.04577 + -7.49625 -7.49625 + 2.1572 2.1572 + 13.0815 13.0815 + 4.57025 4.57025 + 0.704658 0.704658 + 3.25079 3.25079 + -0.682139 -0.682139 + -4.17209 -4.17209 + -1.38547 -1.38547 + 5.52688 5.52688 + -4.90717 -4.90717 + 2.56402 2.56402 + -1.37164 -1.37164 + -6.05044 -6.05044 + 8.3158 8.3158 + -0.640461 -0.640461 + -2.40145 -2.40145 + -1.02959 -1.02959 + -6.75028 -6.75028 + 4.20206 4.20206 + 0.615412 0.615412 + -0.389435 -0.389435 + -5.07439 -5.07439 + -5.34136 -5.34136 + -1.88522 -1.88522 + -4.82628 -4.82628 + 0.54435 0.54435 + -3.28948 -3.28948 + 5.0051 5.0051 + -8.5501 -8.5501 + 7.31448 7.31448 + 0.145651 0.145651 + 3.28586 3.28586 + -1.8624 -1.8624 + -8.9235 -8.9235 + 3.15894 3.15894 + -9.9459 -9.9459 + 0.517233 0.517233 + -4.59899 -4.59899 + 0.641116 0.641116 + 10.3809 10.3809 + 2.39935 2.39935 + -0.378496 -0.378496 + 0.680329 0.680329 + 2.35584 2.35584 + -2.24714 -2.24714 + -4.8742 -4.8742 + -3.96429 -3.96429 + 1.29263 1.29263 + 0.618875 0.618875 + -0.611961 -0.611961 + 1.06612 1.06612 + -3.39289 -3.39289 + -0.226022 -0.226022 + 4.24418 4.24418 + 0.884239 0.884239 + 8.25747 8.25747 + -3.23019 -3.23019 + -9.99374 -9.99374 + 8.54414 8.54414 + -6.06374 -6.06374 + -4.92601 -4.92601 + 7.22101 7.22101 + 11.5756 11.5756 + 13.436 13.436 + 4.13522 4.13522 + 9.67412 9.67412 + -3.13805 -3.13805 + 7.50856 7.50856 + -7.98069 -7.98069 + 4.92059 4.92059 + -6.72969 -6.72969 + -4.48762 -4.48762 + -3.60328 -3.60328 + -1.75053 -1.75053 + 1.5638 1.5638 + 4.74213 4.74213 + 5.16046 5.16046 + -1.9857 -1.9857 + -6.34885 -6.34885 + -3.58963 -3.58963 + 4.96795 4.96795 + 1.44405 1.44405 + -2.74682 -2.74682 + -0.545296 -0.545296 + -10.7507 -10.7507 + -0.117477 -0.117477 + -0.436907 -0.436907 + -1.11656 -1.11656 + 1.64789 1.64789 + -4.08799 -4.08799 + -1.04262 -1.04262 + 6.06007 6.06007 + -6.68208 -6.68208 + 6.81976 6.81976 + -6.89836 -6.89836 + -0.555115 -0.555115 + -2.85307 -2.85307 + -7.76567 -7.76567 + -5.65104 -5.65104 + 8.93521 8.93521 + -5.0663 -5.0663 + 2.52214 2.52214 + 0.382824 0.382824 + -0.398468 -0.398468 + 5.05183 5.05183 + 4.134 4.134 + 1.42909 1.42909 + 2.99357 2.99357 + 10.7821 10.7821 + -4.54764 -4.54764 + -0.0440308 -0.0440308 + 0.647161 0.647161 + 3.27569 3.27569 + -32.9478 -32.9478 + 6.92399 6.92399 + -3.05953 -3.05953 + -2.29742 -2.29742 + -0.41863 -0.41863 + 2.99125 2.99125 + 3.40805 3.40805 + -1.36651 -1.36651 + -3.25561 -3.25561 + 5.11504 5.11504 + -0.532291 -0.532291 + 9.93341 9.93341 + -2.2806 -2.2806 + 10.9617 10.9617 + -2.53642 -2.53642 + 0.995763 0.995763 + -1.28898 -1.28898 + -2.99921 -2.99921 + -2.46773 -2.46773 + -11.0849 -11.0849 + -11.64 -11.64 + -3.73617 -3.73617 + 2.74223 2.74223 + -0.976817 -0.976817 + -0.384814 -0.384814 + -3.38815 -3.38815 + 2.27591 2.27591 + -5.25732 -5.25732 + -1.65764 -1.65764 + -5.8501 -5.8501 + -4.85863 -4.85863 + 2.78987 2.78987 + 5.3324 5.3324 + -9.16758 -9.16758 + 7.90047 7.90047 + 5.68696 5.68696 + 7.2668 7.2668 + -0.857072 -0.857072 + 0.0834347 0.0834347 + 1.11833 1.11833 + 0.88212 0.88212 + -4.40785 -4.40785 + 5.25846 5.25846 + 7.46283 7.46283 + 6.26981 6.26981 + -10.8935 -10.8935 + -0.226332 -0.226332 + -1.64568 -1.64568 + -0.389003 -0.389003 + -0.854872 -0.854872 + -3.38063 -3.38063 + -4.74874 -4.74874 + -1.81717 -1.81717 + -6.03338 -6.03338 + 9.41153 9.41153 + -2.75636 -2.75636 + -4.03638 -4.03638 + -2.82527 -2.82527 + 0.641039 0.641039 + -3.08939 -3.08939 + -1.04523 -1.04523 + -4.17379 -4.17379 + 0.453503 0.453503 + 5.64541 5.64541 + 2.72225 2.72225 + -1.67354 -1.67354 + -6.68729 -6.68729 + -1.20785 -1.20785 + 3.51562 3.51562 + 2.38257 2.38257 + 2.75735 2.75735 + -4.62925 -4.62925 + 7.98247 7.98247 + 6.254 6.254 + 3.85448 3.85448 + -4.40298 -4.40298 + -8.28751 -8.28751 + -7.28055 -7.28055 + 7.31675 7.31675 + 3.53957 3.53957 + 2.94378 2.94378 + 1.41268 1.41268 + 5.2878 5.2878 + -0.807317 -0.807317 + -13.141 -13.141 + 5.71505 5.71505 + -3.86739 -3.86739 + 0.922435 0.922435 + -4.52167 -4.52167 + 0.82741 0.82741 + 4.1254 4.1254 + -3.64229 -3.64229 + -4.34879 -4.34879 + -5.69361 -5.69361 + 10.0503 10.0503 + -6.20878 -6.20878 + -5.70531 -5.70531 + -0.265037 -0.265037 + 4.91217 4.91217 + -9.85839 -9.85839 + 9.14639 9.14639 + 0.78426 0.78426 + -6.03581 -6.03581 + -1.225 -1.225 + -1.82514 -1.82514 + -4.38257 -4.38257 + -4.14898 -4.14898 + 1.30056 1.30056 + -4.04361 -4.04361 + -10.7862 -10.7862 + -1.71033 -1.71033 + -5.3235 -5.3235 + -5.05158 -5.05158 + 2.03088 2.03088 + -4.639 -4.639 + -8.90379 -8.90379 + -1.46286 -1.46286 + 4.78737 4.78737 + 2.84292 2.84292 + -4.60125 -4.60125 + -0.454598 -0.454598 + -3.54703 -3.54703 + -3.15574 -3.15574 + -5.66794 -5.66794 + -0.499733 -0.499733 + 4.80394 4.80394 + 7.0018 7.0018 + -12.2494 -12.2494 + -0.705371 -0.705371 + 0.0740021 0.0740021 + -2.66987 -2.66987 + 2.48263 2.48263 + -9.06332 -9.06332 + -1.01261 -1.01261 + 3.84118 3.84118 + 4.21216 4.21216 + -1.18673 -1.18673 + -11.0005 -11.0005 + -9.71638 -9.71638 + 1.76212 1.76212 + -2.83766 -2.83766 + -9.13768 -9.13768 + -1.05015 -1.05015 + 2.53008 2.53008 + 0.379504 0.379504 + 5.28803 5.28803 + -6.17221 -6.17221 + 5.75619 5.75619 + 2.3737 2.3737 + -9.0974 -9.0974 + -7.85433 -7.85433 + -10.9094 -10.9094 + 1.20756 1.20756 + 2.61486 2.61486 + 1.23359 1.23359 + 43.6151 43.6151 + -1.72859 -1.72859 + -0.965831 -0.965831 + -0.482239 -0.482239 + -1.82159 -1.82159 + 1.661 1.661 + 1.93636 1.93636 + -11.9999 -11.9999 + 0.104367 0.104367 + -1.70555 -1.70555 + -9.81074 -9.81074 + 12.7941 12.7941 + -3.36221 -3.36221 + -6.06523 -6.06523 + 0.47411 0.47411 + -6.64475 -6.64475 + -0.763006 -0.763006 + -3.9763 -3.9763 + -2.86732 -2.86732 + -20.6937 -20.6937 + 1.84418 1.84418 + 5.65243 5.65243 + 10.7255 10.7255 + -1.21293 -1.21293 + 3.15057 3.15057 + 8.96094 8.96094 + -0.205015 -0.205015 + 8.44579 8.44579 + 2.01362 2.01362 + 2.36648 2.36648 + 11.6752 11.6752 + 2.19072 2.19072 + -13.9182 -13.9182 + 3.3257 3.3257 + -6.60627 -6.60627 + 1.62083 1.62083 + -2.00847 -2.00847 + 11.6978 11.6978 + 5.93254 5.93254 + 4.93134 4.93134 + -2.50847 -2.50847 + -5.92846 -5.92846 + 1.16717 1.16717 + 6.9673 6.9673 + -1.21182 -1.21182 + 7.25413 7.25413 + -4.24031 -4.24031 + -3.12368 -3.12368 + 1.73734 1.73734 + -2.6551 -2.6551 + 5.01063 5.01063 + 10.9923 10.9923 + 3.08502 3.08502 + -1.67866 -1.67866 + 10.7003 10.7003 + -0.982895 -0.982895 + 1.97681 1.97681 + -1.29045 -1.29045 + 1.64227 1.64227 + 3.21157 3.21157 + -4.63376 -4.63376 + 4.47725 4.47725 + 7.77208 7.77208 + 0.332548 0.332548 + 2.82084 2.82084 + 0.958649 0.958649 + 1.21302 1.21302 + -3.16936 -3.16936 + 0.0672417 0.0672417 + 0.563038 0.563038 + -1.87542 -1.87542 + -3.01753 -3.01753 + 2.73107 2.73107 + -3.68276 -3.68276 + 4.64376 4.64376 + -12.4341 -12.4341 + 4.43429 4.43429 + 5.72878 5.72878 + 2.39332 2.39332 + 1.91106 1.91106 + 2.50458 2.50458 + 0.942479 0.942479 + -0.489758 -0.489758 + 0.311101 0.311101 + -2.74953 -2.74953 + 4.95959 4.95959 + 1.26862 1.26862 + 10.3622 10.3622 + 3.61213 3.61213 + -2.19285 -2.19285 + 1.28587 1.28587 + -1.85274 -1.85274 + -1.62541 -1.62541 + 2.00382 2.00382 + -5.8959 -5.8959 + -0.918042 -0.918042 + 6.43711 6.43711 + 0.419441 0.419441 + -2.61133 -2.61133 + -0.0277654 -0.0277654 + 2.77443 2.77443 + 3.83764 3.83764 + -1.44486 -1.44486 + -0.611288 -0.611288 + -4.30436 -4.30436 + 5.29466 5.29466 + 1.56058 1.56058 + 1.88962 1.88962 + 0.761408 0.761408 + 1.76505 1.76505 + 1.18453 1.18453 + 1.71559 1.71559 + -3.14851 -3.14851 + 2.73145 2.73145 + -1.23904 -1.23904 + 0.00672958 0.00672958 + 3.40979 3.40979 + -1.77498 -1.77498 + -7.12266 -7.12266 + -9.24697 -9.24697 + -4.12038 -4.12038 + -2.77817 -2.77817 + 8.23453 8.23453 + -1.29818 -1.29818 + -7.02203 -7.02203 + -5.8994 -5.8994 + 8.20499 8.20499 + 0.356509 0.356509 + -0.515947 -0.515947 + -6.23904 -6.23904 + 5.59801 5.59801 + -4.44281 -4.44281 + -2.28591 -2.28591 + -3.31819 -3.31819 + 2.39253 2.39253 + 3.18355 3.18355 + -2.73303 -2.73303 + -0.0346074 -0.0346074 + -10.2692 -10.2692 + 6.74308 6.74308 + 5.72055 5.72055 + -4.49033 -4.49033 + 1.99176 1.99176 + 6.10782 6.10782 + 2.65759 2.65759 + 1.97884 1.97884 + 0.927606 0.927606 + 1.25006 1.25006 + 9.3695 9.3695 + -2.75726 -2.75726 + -0.580415 -0.580415 + 2.92463 2.92463 + -4.49535 -4.49535 + -1.61397 -1.61397 + 3.26733 3.26733 + -3.61505 -3.61505 + -2.46453 -2.46453 + 2.42436 2.42436 + 5.68683 5.68683 + 6.07494 6.07494 + 4.35205 4.35205 + -5.29467 -5.29467 + -3.90039 -3.90039 + -1.70776 -1.70776 + -6.3172 -6.3172 + 4.03858 4.03858 + -2.58786 -2.58786 + -1.1514 -1.1514 + -0.632569 -0.632569 + -0.343314 -0.343314 + -12.2115 -12.2115 + 0.405742 0.405742 + -6.46017 -6.46017 + -2.30808 -2.30808 + 1.1336 1.1336 + 1.47556 1.47556 + 1.98494 1.98494 + 2.24865 2.24865 + -1.65786 -1.65786 + -4.62769 -4.62769 + 4.43717 4.43717 + 8.75249 8.75249 + 4.29167 4.29167 + -3.96876 -3.96876 + -3.52244 -3.52244 + 0.161164 0.161164 + -4.13202 -4.13202 + 1.42269 1.42269 + -3.05155 -3.05155 + 1.81371 1.81371 + -1.03765 -1.03765 + 0.696656 0.696656 + 2.95359 2.95359 + -4.74837 -4.74837 + -9.03481 -9.03481 + 4.8852 4.8852 + 9.47173 9.47173 + 11.3037 11.3037 + -3.88084 -3.88084 + -5.99356 -5.99356 + 7.81639 7.81639 + -6.51949 -6.51949 + 7.801 7.801 + -0.795429 -0.795429 + -0.801046 -0.801046 + 2.70658 2.70658 + 5.51012 5.51012 + 1.8181 1.8181 + -0.452854 -0.452854 + -10.1558 -10.1558 + 1.95877 1.95877 + -3.88197 -3.88197 + 1.72033 1.72033 + -1.8939 -1.8939 + -1.64082 -1.64082 + -0.409815 -0.409815 + 9.98658 9.98658 + -0.115277 -0.115277 + 1.49827 1.49827 + 1.6696 1.6696 + 2.29297 2.29297 + -2.14941 -2.14941 + 2.43318 2.43318 + 3.59845 3.59845 + -4.58877 -4.58877 + -9.25371 -9.25371 + 2.03609 2.03609 + 5.5921 5.5921 + -0.532859 -0.532859 + 4.34937 4.34937 + 1.57036 1.57036 + 2.30747 2.30747 + 7.5055 7.5055 + 3.41771 3.41771 + 0.589402 0.589402 + 1.55834 1.55834 + 5.12407 5.12407 + -1.41727 -1.41727 + 1.03223 1.03223 + -2.06257 -2.06257 + 3.11532 3.11532 + 1.90042 1.90042 + 8.66814 8.66814 + 5.36716 5.36716 + 2.38085 2.38085 + 5.72834 5.72834 + -6.5998 -6.5998 + 0.852569 0.852569 + -7.5648 -7.5648 + 2.98063 2.98063 + 7.81573 7.81573 + 1.82276 1.82276 + -1.81083 -1.81083 + 5.48043 5.48043 + -1.85315 -1.85315 + -1.62277 -1.62277 + -10.4951 -10.4951 + 5.34799 5.34799 + -1.77515 -1.77515 + 5.88005 5.88005 + 0.0799242 0.0799242 + 1.23264 1.23264 + -11.835 -11.835 + 3.56828 3.56828 + 7.53741 7.53741 + -5.24051 -5.24051 + -0.206917 -0.206917 + 4.36865 4.36865 + -4.10348 -4.10348 + 0.857712 0.857712 + -5.09677 -5.09677 + 7.37208 7.37208 + -3.14614 -3.14614 + 12.061 12.061 + 4.80096 4.80096 + 2.82421 2.82421 + -4.97446 -4.97446 + -11.0289 -11.0289 + -8.33282 -8.33282 + 0.69922 0.69922 + 5.08771 5.08771 + 2.65174 2.65174 + -3.30182 -3.30182 + 5.21741 5.21741 + 8.85373 8.85373 + 8.36416 8.36416 + 2.54295 2.54295 + -1.61657 -1.61657 + 1.12017 1.12017 + -7.33205 -7.33205 + 3.82582 3.82582 + -0.858026 -0.858026 + 1.40304 1.40304 + 1.35079 1.35079 + 4.19532 4.19532 + -1.77923 -1.77923 + -10.5119 -10.5119 + 10.8061 10.8061 + -3.49603 -3.49603 + 3.12404 3.12404 + -3.93328 -3.93328 + -6.73356 -6.73356 + 1.80532 1.80532 + -0.368024 -0.368024 + -3.47875 -3.47875 + -4.22893 -4.22893 + 2.52519 2.52519 + -3.54943 -3.54943 + -2.39869 -2.39869 + 4.22126 4.22126 + -0.253856 -0.253856 + 7.51866 7.51866 + -4.54093 -4.54093 + 3.44497 3.44497 + 4.77417 4.77417 + 4.49646 4.49646 + -5.78678 -5.78678 + 0.745013 0.745013 + 1.69763 1.69763 + -2.64759 -2.64759 + 1.66108 1.66108 + -4.68276 -4.68276 + 5.31823 5.31823 + 3.52288 3.52288 + 4.9695 4.9695 + 12.2016 12.2016 + 2.46849 2.46849 + -7.60038 -7.60038 + 8.21628 8.21628 + 5.99856 5.99856 + -6.80947 -6.80947 + 7.22522 7.22522 + -2.00065 -2.00065 + -8.24049 -8.24049 + -0.0804049 -0.0804049 + -2.06638 -2.06638 + -2.82884 -2.82884 + -4.25891 -4.25891 + -5.20258 -5.20258 + -3.19396 -3.19396 + -5.14527 -5.14527 + -4.28244 -4.28244 + 4.70805 4.70805 + -3.08065 -3.08065 + -4.86906 -4.86906 + -29.0266 -29.0266 + -1.22941 -1.22941 + -1.30928 -1.30928 + -6.35234 -6.35234 + 1.87904 1.87904 + 8.37797 8.37797 + -5.8821 -5.8821 + 3.10138 3.10138 + -3.27553 -3.27553 + -0.208451 -0.208451 + -2.28999 -2.28999 + 12.2896 12.2896 + -1.27394 -1.27394 + -3.41924 -3.41924 + -0.289592 -0.289592 + 1.79867 1.79867 + 1.98504 1.98504 + 1.55159 1.55159 + 1.10858 1.10858 + 0.352842 0.352842 + -0.309044 -0.309044 + -0.165336 -0.165336 + 1.15822 1.15822 + -1.39342 -1.39342 + -0.162562 -0.162562 + -8.06055 -8.06055 + -5.02776 -5.02776 + -8.66927 -8.66927 + 1.14576 1.14576 + -1.52122 -1.52122 + -1.29436 -1.29436 + 3.26421 3.26421 + 7.55561 7.55561 + 7.7265 7.7265 + -0.48821 -0.48821 + 12.439 12.439 + 7.0264 7.0264 + -11.9855 -11.9855 + -3.74151 -3.74151 + -0.200302 -0.200302 + 5.39515 5.39515 + -4.3468 -4.3468 + 9.25599 9.25599 + 3.37455 3.37455 + -6.15424 -6.15424 + -6.6271 -6.6271 + 0.000272481 0.000272481 + -5.48117 -5.48117 + -0.493191 -0.493191 + -3.46473 -3.46473 + 2.33812 2.33812 + 0.885965 0.885965 + 4.74926 4.74926 + 1.51959 1.51959 + 2.50956 2.50956 + -0.728024 -0.728024 + 1.0381 1.0381 + 5.48121 5.48121 + -1.68033 -1.68033 + -5.05915 -5.05915 + -0.646233 -0.646233 + 0.614062 0.614062 + 4.54219 4.54219 + -1.63006 -1.63006 + -3.10589 -3.10589 + -3.12801 -3.12801 + -5.98177 -5.98177 + -3.59188 -3.59188 + 1.7066 1.7066 + -7.43935 -7.43935 + 10.6141 10.6141 + 12.6478 12.6478 + -1.7222 -1.7222 + -2.1519 -2.1519 + -7.16573 -7.16573 + 0.887314 0.887314 + -8.59735 -8.59735 + -1.3609 -1.3609 + 4.47651 4.47651 + 0.900892 0.900892 + 7.81857 7.81857 + 6.19857 6.19857 + 2.12844 2.12844 + 3.08551 3.08551 + 4.15866 4.15866 + 2.09657 2.09657 + -2.27786 -2.27786 + 1.33571 1.33571 + 4.46899 4.46899 + 4.46674 4.46674 + 3.20736 3.20736 + 5.68287 5.68287 + 10.1058 10.1058 + 5.1894 5.1894 + 3.5452 3.5452 + 10.06 10.06 + 7.02935 7.02935 + -1.06066 -1.06066 + 10.32 10.32 + -0.860463 -0.860463 + 5.95992 5.95992 + -6.30137 -6.30137 + -5.01947 -5.01947 + 5.75187 5.75187 + -1.10079 -1.10079 + -1.91783 -1.91783 + 0.815744 0.815744 + -0.958663 -0.958663 + -3.28825 -3.28825 + -6.37854 -6.37854 + 6.91577 6.91577 + 2.54565 2.54565 + 1.39487 1.39487 + 1.59679 1.59679 + 4.72347 4.72347 + 2.49221 2.49221 + 1.29896 1.29896 + -4.08232 -4.08232 + -0.648436 -0.648436 + -6.43531 -6.43531 + -0.556197 -0.556197 + -1.40304 -1.40304 + 0.699818 0.699818 + -5.29777 -5.29777 + -3.44335 -3.44335 + 7.35309 7.35309 + 8.846 8.846 + 8.39833 8.39833 + -2.71436 -2.71436 + 3.37063 3.37063 + -3.18723 -3.18723 + 1.32256 1.32256 + -3.09485 -3.09485 + 8.78146 8.78146 + -1.30004 -1.30004 + 3.03526 3.03526 + -1.4592 -1.4592 + 3.90288 3.90288 + -13.5124 -13.5124 + 1.35105 1.35105 + 3.37337 3.37337 + 2.5171 2.5171 + -4.22085 -4.22085 + 13.1858 13.1858 + -6.02839 -6.02839 + 5.75692 5.75692 + 2.46171 2.46171 + -0.950315 -0.950315 + -3.63255 -3.63255 + 1.88 1.88 + 5.48758 5.48758 + 4.96786 4.96786 + -6.17199 -6.17199 + -0.284244 -0.284244 + -1.80256 -1.80256 + 3.03221 3.03221 + -8.90171 -8.90171 + -8.66084 -8.66084 + -9.06366 -9.06366 + -3.02007 -3.02007 + -8.2276 -8.2276 + 8.10032 8.10032 + -4.11364 -4.11364 + -3.39291 -3.39291 + 3.64208 3.64208 + -0.739833 -0.739833 + -2.84156 -2.84156 + -0.843081 -0.843081 + -0.249744 -0.249744 + 7.05075 7.05075 + 0.369632 0.369632 + -1.90893 -1.90893 + 9.79465 9.79465 + 3.52356 3.52356 + 4.14091 4.14091 + 1.66568 1.66568 + -10.7162 -10.7162 + -7.64522 -7.64522 + 1.54688 1.54688 + 7.84479 7.84479 + 0.466458 0.466458 + 4.03315 4.03315 + 0.472926 0.472926 + 1.73319 1.73319 + 1.79317 1.79317 + 1.46234 1.46234 + -8.45267 -8.45267 + 7.30327 7.30327 + 3.08869 3.08869 + 5.27442 5.27442 + 2.92876 2.92876 + -1.6673 -1.6673 + 14.4442 14.4442 + 13.4055 13.4055 + -1.47522 -1.47522 + -3.57821 -3.57821 + 9.00659 9.00659 + -9.6723 -9.6723 + 2.8818 2.8818 + -2.61898 -2.61898 + 1.17927 1.17927 + -3.15135 -3.15135 + -0.976968 -0.976968 + 1.45062 1.45062 + 4.66687 4.66687 + 4.94346 4.94346 + -2.20375 -2.20375 + 2.93643 2.93643 + 7.51365 7.51365 + 6.50034 6.50034 + 1.74088 1.74088 + -4.43403 -4.43403 + 0.796894 0.796894 + -1.23803 -1.23803 + 5.33941 5.33941 + 4.90517 4.90517 + 0.569053 0.569053 + -0.609673 -0.609673 + 5.091 5.091 + 1.76184 1.76184 + -3.81174 -3.81174 + -5.39095 -5.39095 + -3.09718 -3.09718 + -1.87868 -1.87868 + -4.85278 -4.85278 + -1.05327 -1.05327 + -1.11892 -1.11892 + -3.52006 -3.52006 + -2.8466 -2.8466 + 3.03494 3.03494 + 3.7605 3.7605 + -1.8123 -1.8123 + -5.10186 -5.10186 + 2.85973 2.85973 + -3.6241 -3.6241 + 1.78302 1.78302 + -12.3108 -12.3108 + 0.378043 0.378043 + -1.70182 -1.70182 + -0.91773 -0.91773 + -5.37355 -5.37355 +```` + +### Using postprocessing function +Add normalization as postprocessing function to normalize embeddings on reception (for easy cosine similarity later) + +````julia +using LinearAlgebra +schema = PT.OllamaManagedSchema() + +msg = aiembed(schema, + ["embed me", "and me too"], + LinearAlgebra.normalize; + model = "openhermes2.5-mistral") +```` + +```` +DataMessage(Matrix{Float64} of size (4096, 2)) +```` + +Cosine similarity is then a simple multiplication + +````julia +msg.content' * msg.content[:, 1] +```` + +```` +2-element Vector{Float64}: + 0.9999999999999946 + 0.34130017815042357 +```` + +--- + +*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* + diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md new file mode 100644 index 000000000..5d06928ea --- /dev/null +++ b/docs/src/frequently_asked_questions.md @@ -0,0 +1,132 @@ +# Frequently Asked Questions + +## Why OpenAI + +OpenAI's models are at the forefront of AI research and provide robust, state-of-the-art capabilities for many tasks. + +There will be situations not or cannot use it (eg, privacy, cost, etc.). In that case, you can use local models (eg, Ollama) or other APIs (eg, Anthropic). + +Note: To get started with [Ollama.ai](https://ollama.ai/), see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. + +## Data Privacy and OpenAI + +At the time of writing, OpenAI does NOT use the API calls for training their models. + +> **API** +> +> OpenAI does not use data submitted to and generated by our API to train OpenAI models or improve OpenAI’s service offering. In order to support the continuous improvement of our models, you can fill out this form to opt-in to share your data with us. -- [How your data is used to improve our models](https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance) + +You can always double-check the latest information on the [OpenAI's How we use your data](https://platform.openai.com/docs/models/how-we-use-your-data) page. + +Resources: +- [OpenAI's How we use your data](https://platform.openai.com/docs/models/how-we-use-your-data) +- [Data usage for consumer services FAQ](https://help.openai.com/en/articles/7039943-data-usage-for-consumer-services-faq) +- [How your data is used to improve our models](https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance) + + +## Creating OpenAI API Key + +You can get your API key from OpenAI by signing up for an account and accessing the API section of the OpenAI website. + +1. Create an account with [OpenAI](https://platform.openai.com/signup) +2. Go to [API Key page](https://platform.openai.com/account/api-keys) +3. Click on “Create new secret key” + !!! Do not share it with anyone and do NOT save it to any files that get synced online. + +Resources: +- [OpenAI Documentation](https://platform.openai.com/docs/quickstart?context=python) +- [Visual tutorial](https://www.maisieai.com/help/how-to-get-an-openai-api-key-for-chatgpt) + +Pro tip: Always set the spending limits! + +## Setting OpenAI Spending Limits + +OpenAI allows you to set spending limits directly on your account dashboard to prevent unexpected costs. + +1. Go to [OpenAI Billing](https://platform.openai.com/account/billing) +2. Set Soft Limit (you’ll receive a notification) and Hard Limit (API will stop working not to spend more money) + +A good start might be a soft limit of c.$5 and a hard limit of c.$10 - you can always increase it later in the month. + +Resources: +- [OpenAI Forum](https://community.openai.com/t/how-to-set-a-price-limit/13086) + +### How much does it cost? Is it worth paying for? + +If you use a local model (eg, with Ollama), it's free. If you use any commercial APIs (eg, OpenAI), you will likely pay per "token" (a sub-word unit). + +For example, a simple request with a simple question and 1 sentence response in return (”Is statement XYZ a positive comment”) will cost you ~$0.0001 (ie, one hundredth of a cent) + +**Is it worth paying for?** + +GenAI is a way to buy time! You can pay cents to save tens of minutes every day. + +Continuing the example above, imagine you have a table with 200 comments. Now, you can parse each one of them with an LLM for the features/checks you need. +Assuming the price per call was $0.0001, you'd pay 2 cents for the job and save 30-60 minutes of your time! + + +Resources: +- [OpenAI Pricing per 1000 tokens](https://openai.com/pricing) + +## Configuring the Environment Variable for API Key + +To use the OpenAI API with PromptingTools.jl, set your API key as an environment variable: + +```julia +ENV["OPENAI_API_KEY"] = "your-api-key" +``` + +As a one-off, you can: +- set it in the terminal before launching Julia: `export OPENAI_API_KEY = ` +- set it in your `setup.jl` (make sure not to commit it to GitHub!) + +Make sure to start Julia from the same terminal window where you set the variable. +Easy check in Julia, run `ENV["OPENAI_API_KEY"]` and you should see your key! + +A better way: +- On a Mac, add the configuration line to your terminal's configuration file (eg, `~/.zshrc`). It will get automatically loaded every time you launch the terminal +- On Windows, set it as a system variable in "Environment Variables" settings (see the Resources) + +Resources: +- [OpenAI Guide](https://platform.openai.com/docs/quickstart?context=python) + +Note: In the future, we hope to add `Preferences.jl`-based workflow to set the API key and other preferences. + +## Understanding the API Keyword Arguments in `aigenerate` (`api_kwargs`) + +See [OpenAI API reference](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) for more information. + +## Instant Access from Anywhere + +For easy access from anywhere, add PromptingTools into your `startup.jl` (can be found in `~/.julia/config/startup.jl`). + +Add the following snippet: +``` +using PromptingTools +const PT = PromptingTools # to access unexported functions and types +``` + +Now, you can just use `ai"Help me do X to achieve Y"` from any REPL session! + +## Open Source Alternatives + +The ethos of PromptingTools.jl is to allow you to use whatever model you want, which includes Open Source LLMs. The most popular and easiest to setup is [Ollama.ai](https://ollama.ai/) - see below for more information. + +## Setup Guide for Ollama + +Ollama runs a background service hosting LLMs that you can access via a simple API. It's especially useful when you're working with some sensitive data that should not be sent anywhere. + +Installation is very easy, just download the latest version [here](https://ollama.ai/download). + +Once you've installed it, just launch the app and you're ready to go! + +To check if it's running, go to your browser and open `127.0.0.1:11434`. You should see the message "Ollama is running". +Alternatively, you can run `ollama serve` in your terminal and you'll get a message that it's already running. + +There are many models available in [Ollama Library](https://ollama.ai/library), including Llama2, CodeLlama, SQLCoder, or my personal favorite `openhermes2.5-mistral`. + +Download new models with `ollama pull ` (eg, `ollama pull openhermes2.5-mistral`). + +Show currently available models with `ollama list`. + +See [Ollama.ai](https://ollama.ai/) for more information. \ No newline at end of file diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md new file mode 100644 index 000000000..3ab182a3d --- /dev/null +++ b/docs/src/getting_started.md @@ -0,0 +1,91 @@ +# Getting Started + +## Prerequisites + +**OpenAI API key saved in the environment variable `OPENAI_API_KEY`** + +You will need to register with OpenAI and generate an API key: + +1. Create an account with [OpenAI](https://platform.openai.com/signup) +2. Go to [API Key page](https://platform.openai.com/account/api-keys) +3. Click on “Create new secret key” + !!! Do not share it with anyone and do NOT save it to any files that get synced online. + +Resources: +- [OpenAI Documentation](https://platform.openai.com/docs/quickstart?context=python) +- [Visual tutorial](https://www.maisieai.com/help/how-to-get-an-openai-api-key-for-chatgpt) + +You will need to set this key as an environment variable before using PromptingTools.jl: + +For a quick start, simply set it via `ENV["OPENAI_API_KEY"] = "your-api-key"` +Alternatively, you can: +- set it in the terminal before launching Julia: `export OPENAI_API_KEY = ` +- set it in your `setup.jl` (make sure not to commit it to GitHub!) + +Make sure to start Julia from the same terminal window where you set the variable. +Easy check in Julia, run `ENV["OPENAI_API_KEY"]` and you should see your key! + +For other options or more robust solutions, see the FAQ section. + +Resources: +- [OpenAI Guide](https://platform.openai.com/docs/quickstart?context=python) + +## Installation + +The PromptingTools package has not yet been registered. But it can be installed using the following commands: + +```julia +using Pkg +Pkg.add("https://github.com/svilupp/PromptingTools.jl") +``` + +Throughout the rest of this tutorial, we will assume that you have installed the +PromptingTools package and have already typed `using PromptingTools` to bring all of the +relevant variables into your current namespace. + +## Quick Start with `@ai_str` + +The easiest start is the `@ai_str` macro. Simply type `ai"your prompt"` and you will get a response from the default model (GPT-3.5 Turbo). + +```julia +ai"What is the capital of France?" +``` + +```plaintext +[ Info: Tokens: 31 @ Cost: $0.0 in 1.5 seconds --> Be in control of your spending! +AIMessage("The capital of France is Paris.") +``` + +Returned object is a light wrapper with generated message in field `:content` (eg, `ans.content`) for additional downstream processing. + +You can easily inject any variables with string interpolation: +```julia +country = "Spain" +ai"What is the capital of \$(country)?" +``` + +```plaintext +[ Info: Tokens: 32 @ Cost: $0.0001 in 0.5 seconds +AIMessage("The capital of Spain is Madrid.") +``` + +Pro tip: Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4` (use `gpt4t` for the new GPT-4 Turbo model). Great for those extra hard questions! + +## Using `aigenerate` with placeholders + +For more complex prompt templates, you can use handlebars-style templating and provide variables as keyword arguments: + +```julia +msg = aigenerate("What is the capital of {{country}}? Is the population larger than {{population}}?", country="Spain", population="1M") +``` + +```plaintext +[ Info: Tokens: 74 @ Cost: $0.0001 in 1.3 seconds +AIMessage("The capital of Spain is Madrid. And yes, the population of Madrid is larger than 1 million. As of 2020, the estimated population of Madrid is around 3.3 million people.") +``` + +Pro tip: Use `asyncmap` to run multiple AI-powered tasks concurrently. + +Pro tip: If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` + +For more practical examples, see the [Various Examples](@ref) section. \ No newline at end of file diff --git a/docs/src/index.md b/docs/src/index.md index 47c47c966..d6f988025 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -6,9 +6,22 @@ CurrentModule = PromptingTools Documentation for [PromptingTools](https://github.com/svilupp/PromptingTools.jl). -```@index -``` +Streamline your life using PromptingTools.jl, the Julia package that simplifies interacting with large language models. -```@autodocs -Modules = [PromptingTools] -``` +PromptingTools.jl is not meant for building large-scale systems. It's meant to be the go-to tool in your global environment that will save you 20 minutes every day! + +## Why PromptingTools.jl? + +Prompt engineering is neither fast nor easy. Moreover, different models and their fine-tunes might require different prompt formats and tricks, or perhaps the information you work with requires special models to be used. PromptingTools.jl is meant to unify the prompts for different backends and make the common tasks (like templated prompts) as simple as possible. + +Some features: +- **`aigenerate` Function**: Simplify prompt templates with handlebars (eg, `{{variable}}`) and keyword arguments +- **`@ai_str` String Macro**: Save keystrokes with a string macro for simple prompts +- **Easy to Remember**: All exported functions start with `ai...` for better discoverability +- **Light Wraper Types**: Benefit from Julia's multiple dispatch by having AI outputs wrapped in specific types +- **Minimal Dependencies**: Enjoy an easy addition to your global environment with very light dependencies +- **No Context Switching**: Access cutting-edge LLMs with no context switching and minimum extra keystrokes directly in your REPL + +## First Steps + +To get started, see the [Getting Started](@ref) section. diff --git a/docs/src/reference.md b/docs/src/reference.md new file mode 100644 index 000000000..fdd9b7835 --- /dev/null +++ b/docs/src/reference.md @@ -0,0 +1,8 @@ +# Reference + +```@index +``` + +```@autodocs +Modules = [PromptingTools] +``` diff --git a/examples/working_with_aitemplates.jl b/examples/working_with_aitemplates.jl index 520f3533b..49f5cf830 100644 --- a/examples/working_with_aitemplates.jl +++ b/examples/working_with_aitemplates.jl @@ -1,5 +1,8 @@ -# This file contains examples of how to work with AITemplate(s). +# # Using AITemplates +# This file contains examples of how to work with AITemplate(s). +# +# First, let's import the package and define a helper link for calling un-exported functions: using PromptingTools const PT = PromptingTools @@ -12,43 +15,30 @@ const PT = PromptingTools PT.load_templates!(); # You can (create them) and use them for any ai* function instead of a prompt: -# Let's use a template called :JuliaExpertAsk +# Let's use a template called `:JuliaExpertAsk` # alternatively, you can use `AITemplate(:JuliaExpertAsk)` for cleaner dispatch msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") -# ... some response from GPT3.5 -# + # You can see that it had a placeholder for the actual question (`ask`) that we provided as a keyword argument. # We did not have to write any system prompt for personas, tone, etc. -- it was all provided by the template! # # How to know which templates are available? You can search for them with `aitemplates()`: # You can search by Symbol (only for partial name match), String (partial match on name or description), or Regex (more fields) tmps = aitemplates("JuliaExpertAsk") -# Outputs a list of available templates that match the search -- there is just one in this case: -# -# 1-element Vector{AITemplateMetadata}: -# PromptingTools.AITemplateMetadata -# name: Symbol JuliaExpertAsk -# description: String "For asking questions about Julia language. Placeholders: `ask`" -# version: String "1" -# wordcount: Int64 237 -# variables: Array{Symbol}((1,)) -# system_preview: String "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your commun" -# user_preview: String "# Question\n\n{{ask}}" -# source: String "" + +# You can see that it outputs a list of available templates that match the search - there is just one in this case. # -# You see not just the description, but also a preview of the actual prompts, placeholders available, and the length (to gauge how much it would cost). +# Moreover, it shows not just the description, but also a preview of the actual prompts, placeholders available, and the length (to gauge how much it would cost). # # If you use VSCode, you can display them in a nice scrollable table with `vscodedisplay`: -using DataFrames -DataFrame(tmp) |> vscodedisplay -# +# ```plaintext +# using DataFrames +# DataFrame(tmp) |> vscodedisplay +# ``` # # You can also just `render` the template to see the underlying mesages: msgs = PT.render(AITemplate(:JuliaExpertAsk)) -# -# 2-element Vector{PromptingTools.AbstractChatMessage}: -# PromptingTools.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") -# PromptingTools.UserMessage("# Question\n\n{{ask}}") + # # Now, you know exactly what's in the template! # @@ -61,7 +51,7 @@ tpl = [PT.SystemMessage("You are a world-class Julia language programmer with th filename = joinpath(pkgdir(PromptingTools), "templates", "persona-task", - "JuliaDataExpertAsk.json") + "JuliaDataExpertAsk_123.json") PT.save_template(filename, tpl; description = "For asking data analysis questions in Julia language. Placeholders: `ask`") diff --git a/examples/working_with_ollama.jl b/examples/working_with_ollama.jl index b4e524fd2..d1aad3176 100644 --- a/examples/working_with_ollama.jl +++ b/examples/working_with_ollama.jl @@ -1,3 +1,9 @@ +# # Local models with Ollama.ai + +# This file contains examples of how to work with [Ollama.ai](https://ollama.ai/) models. +# It assumes that you've already installated and launched the Ollama server. For more details or troubleshooting advice, see the [Frequently Asked Questions](@ref) section. +# +# First, let's import the package and define a helper link for calling un-exported functions: using PromptingTools const PT = PromptingTools @@ -6,38 +12,35 @@ schema = PT.OllamaManagedSchema() # You can choose models from https://ollama.ai/library - I prefer `openhermes2.5-mistral` model = "openhermes2.5-mistral" -# # Text Generation with aigenerate +# ## Text Generation with aigenerate -# ## Simple message +# ### Simple message msg = aigenerate(schema, "Say hi!"; model) -# ## Standard string interpolation +# ### Standard string interpolation a = 1 msg = aigenerate(schema, "What is `$a+$a`?"; model) name = "John" msg = aigenerate(schema, "Say hi to {{name}}."; name, model) -# ## Advanced Prompts +# ### Advanced Prompts conversation = [ PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), PT.UserMessage("I have feelings for my iPhone. What should I do?")] msg = aigenerate(schema, conversation; model) -# # Embeddings with aiembed +# ## Embeddings with aiembed -# ## Simple embedding for one document -msg = aiembed(schema, "Embed me"; model) -msg.content +# ### Simple embedding for one document +msg = aiembed(schema, "Embed me"; model) # access msg.content # One document and we materialize the data into a Vector with copy (`postprocess` function argument) msg = aiembed(schema, "Embed me", copy; model) -msg.content -# ## Multiple documents embedding +# ### Multiple documents embedding # Multiple documents - embedded sequentially, you can get faster speed with async msg = aiembed(schema, ["Embed me", "Embed me"]; model) -msg.content # You can use Threads.@spawn or asyncmap, whichever you prefer, to paralellize the model calls docs = ["Embed me", "Embed me"] @@ -46,7 +49,7 @@ tasks = asyncmap(docs) do doc end embedding = mapreduce(x -> x.content, hcat, tasks) -# ## Using postprocessing function +# ### Using postprocessing function # Add normalization as postprocessing function to normalize embeddings on reception (for easy cosine similarity later) using LinearAlgebra schema = PT.OllamaManagedSchema() @@ -57,4 +60,4 @@ msg = aiembed(schema, model = "openhermes2.5-mistral") # Cosine similarity is then a simple multiplication -msg.content' * msg.content[:, 1] # [1.0, 0.34] \ No newline at end of file +msg.content' * msg.content[:, 1] \ No newline at end of file From 3918eb96a5ac11ab34aae418eb78649dfdcdf626 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 25 Nov 2023 15:45:10 +0000 Subject: [PATCH 018/251] add templates for transcripts and surveys --- src/utils.jl | 71 +++++++++++++++++++ .../AnalystChaptersInTranscript.json | 22 ++++++ .../AnalystThemesInResponses.json | 23 ++++++ test/utils.jl | 33 +++++++++ 4 files changed, 149 insertions(+) create mode 100644 templates/persona-task/AnalystChaptersInTranscript.json create mode 100644 templates/persona-task/AnalystThemesInResponses.json diff --git a/src/utils.jl b/src/utils.jl index 1365fa166..af7b68378 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,3 +1,74 @@ +### USEFUL BUT NOT EXPORTED FUNCTIONS +""" + split_by_length(text::String; separator::String=" ", max_length::Int=35000) -> Vector{String} + +Split a given string `text` into chunks of a specified maximum length `max_length`. +This is particularly useful for splitting larger documents or texts into smaller segments, suitable for models or systems with smaller context windows. + +# Arguments +- `text::String`: The text to be split. +- `separator::String=" "`: The separator used to split the text into minichunks. Defaults to a space character. +- `max_length::Int=35000`: The maximum length of each chunk. Defaults to 35,000 characters, which should fit within 16K context window. + +# Returns +`Vector{String}`: A vector of strings, each representing a chunk of the original text that is smaller than or equal to `max_length`. + +# Notes + +- The function ensures that each chunk is as close to `max_length` as possible without exceeding it. +- If the `text` is empty, the function returns an empty array. +- The `separator` is re-added to the text chunks after splitting, preserving the original structure of the text as closely as possible. + +# Examples + +Splitting text with the default separator (" "): +```julia +text = "Hello world. How are you?" +chunks = splitbysize(text; max_length=13) +length(chunks) # Output: 2 +``` + +Using a custom separator and custom `max_length` +```julia +text = "Hello,World," ^ 2900 # length 34900 chars +split_by_length(text; separator=",", max_length=10000) # for 4K context window +length(chunks[1]) # Output: 4 +``` +""" +function split_by_length(text::String; separator::String = " ", max_length::Int = 35000) + minichunks = split(text, separator) + sep_length = length(separator) + chunks = String[] + current_chunk = IOBuffer() + current_length = 0 + for i in eachindex(minichunks) + sep_length_ = i < length(minichunks) ? sep_length : 0 + # Check if the current chunk is full + if current_length + length(minichunks[i]) + sep_length_ > max_length + # Save chunk, excluding the current mini chunk + save_chunk = String(take!(current_chunk)) + if length(save_chunk) > 0 + push!(chunks, save_chunk) + end + current_length = 0 + end + write(current_chunk, minichunks[i]) + current_length += length(minichunks[i]) + if i < length(minichunks) + write(current_chunk, separator) + current_length += sep_length + end + end + + # Add the last chunk if it's not empty + final_chunk = String(take!(current_chunk)) + if length(final_chunk) > 0 + push!(chunks, final_chunk) + end + + return chunks +end +### INTERNAL FUNCTIONS - DO NOT USE DIRECTLY # helper to extract handlebar variables (eg, `{{var}}`) from a prompt string function _extract_handlebar_variables(s::AbstractString) Symbol[Symbol(m[1]) for m in eachmatch(r"\{\{([^\}]+)\}\}", s)] diff --git a/templates/persona-task/AnalystChaptersInTranscript.json b/templates/persona-task/AnalystChaptersInTranscript.json new file mode 100644 index 000000000..903b538ca --- /dev/null +++ b/templates/persona-task/AnalystChaptersInTranscript.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "Template for summarizing transcripts of videos and meetings into chapters with key insights. If you don't need the instructions, set `instructions=\"None.\"`. Placeholders: {{transcript}}, {{instructions}}", + "version": "1", + "source": "Customized version of [jxnl's Youtube Chapters prompt](https://github.com/jxnl/youtubechapters-backend/blob/main/summary_app/md_summarize.py)", + "_type": "metadatamessage" + }, + { + "content": "Act as a super-human AI analyst trained to precisely summarize transcripts of videos and meetings with incredible precision and quality. \nSummarize the transcript in a clear and concise manner that makes use of timestamps, when available, to help others study the transcript. Split the notes into Chapters, which should be meaningful and not too short.\n\nTo format your markdown file, follow this structure:\n```\n# Chapter 1: [Descriptive Title] [Timestamp as HH:MM:SS]\n\n- \n\n## Section 1.1: [Descriptive Title] [Timestamp as HH:MM:SS]\n\n\n- \n\nRepeat the above structure as necessary, and use subheadings to organize your notes.\n```\n\nFormatting Tips:\n* Do not make the chapters too short, ensure that each section has a few brief bullet points. \n* Bullet points should be concise and to the point, so people can scan them quickly.\n* Use [] to denote timestamps\n* Use subheadings and bullet points to organize your notes and make them easier to read and understand. When relevant, include timestamps to link to the corresponding part of the video.\n* Use bullet points to describe important steps and insights, being as comprehensive as possible.\n* Use quotes to highlight important points and insights.\n\nSummary Tips:\n* Do not mention anything if its only playing music and if nothing happens don't include it in the notes.\n* Use only content from the transcript. Do not add any additional information.\n* Make a new line after each # or ## and before each bullet point\n* Titles should be informative or even a question that the video answers\n* Titles should not be conclusions since you may only be getting a small part of the video\n\nKeep it CONCISE!!\nIf Special Instructions are provided by the user, they take precedence over any previous instructions and you MUST follow they precisely.\n", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Transcript\n\n{{transcript}}\n\n\n\n# Special Instructions\n\n{{instructions}}", + "variables": [ + "transcript", + "instructions" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/AnalystThemesInResponses.json b/templates/persona-task/AnalystThemesInResponses.json new file mode 100644 index 000000000..900fd9436 --- /dev/null +++ b/templates/persona-task/AnalystThemesInResponses.json @@ -0,0 +1,23 @@ +[ + { + "content": "Template Metadata", + "description": "Template for summarizing survey verbatim responses into 3-5 themes with an example for each theme. If you don't need the instructions, set `instructions=\"None.\"`. Placeholders: {{question}}, {{responses}}, {{instructions}}", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "\"Act a world-class behavioural researcher, who specializes on survey analysis. Categorize the provided survey responses into several themes. \nThe responses should be analyzed, and each theme identified should be labeled clearly. Examples from the responses should be given to illustrate each theme. The output should be formatted as specified, with a clear indication of the theme and corresponding verbatim examples.\n\n# Sub-tasks\n\n1. Read the provided survey responses carefully, especially in the context of the question. \n2. Identify 3-5 distinct themes present in the responses related to the survey question. It should be the most important themes that must be raised to the CEO/leadership. \n3. For each theme, choose at least one verbatim example from the responses that best represents it. This example should be a direct quote from the responses. This example should belong to only one theme and must not be applicable to any other themes.\n4. Format the output as specified.\n\n# Formatting\n\nTo format your markdown file, follow this structure (omit the triple backticks):\n ```\n # Theme 1: [Theme Description]\n - Best illustrated by: \"...\"\n\n # Theme 2: [Theme Description]\n - Best illustrated by: \"...\"\n ...\n ```\n\nKeep it CONCISE!!\nIf Special Instructions are provided by the user, they take precedence over any previous instructions and you MUST follow they precisely.\n", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Survey Question\n\n{{question}}\n\n\n# Verbatim Responses\n\n{{responses}}\n\n\n# Special Instructions\n\n{{instructions}}\n", + "variables": [ + "question", + "responses", + "instructions" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/test/utils.jl b/test/utils.jl index 138343257..e3a05cecd 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,6 +1,39 @@ +using PromptingTools: split_by_length using PromptingTools: _extract_handlebar_variables, _report_stats using PromptingTools: _string_to_vector, _encode_local_image +@testset "split_by_length" begin + text = "Hello world. How are you?" + chunks = split_by_length(text, max_length = 100) + @test length(chunks) == 1 + @test chunks[1] == text + chunks = split_by_length(text, max_length = 25) + @test length(chunks) == 1 + @test chunks[1] == text + @test maximum(length.(chunks)) <= 25 + chunks = split_by_length(text, max_length = 10) + @test length(chunks) == 4 + @test maximum(length.(chunks)) <= 10 + chunks = split_by_length(text, max_length = 11) + @test length(chunks) == 3 + @test maximum(length.(chunks)) <= 11 + @test join(chunks, "") == text + + # Test with empty text + chunks = split_by_length("") + @test isempty(chunks) + + # Test custom separator + text = "Hello,World,"^50 + chunks = split_by_length(text, separator = ",", max_length = length(text)) + @test length(chunks) == 1 + @test chunks[1] == text + chunks = split_by_length(text, separator = ",", max_length = 20) + @test length(chunks) == 34 + @test maximum(length.(chunks)) <= 20 + @test join(chunks, "") == text +end + @testset "extract_handlebar_variables" begin # Extracts handlebar variables enclosed in double curly braces input_string = "Hello {{name}}, how are you?" From fcb68d35e3c5330c35d5c8f368f48a26bc94472e Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:39:39 +0000 Subject: [PATCH 019/251] add decisions template, fix tests --- CHANGELOG.md | 3 +- Project.toml | 2 ++ src/PromptingTools.jl | 7 +++-- src/utils.jl | 31 +++++++++++++++++++ .../AnalystDecisionsInTranscript.json | 22 +++++++++++++ templates/persona-task/DrafterEmailBrief.json | 21 +++++++++++++ test/templates.jl | 4 ++- test/utils.jl | 13 +++++++- 8 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 templates/persona-task/AnalystDecisionsInTranscript.json create mode 100644 templates/persona-task/DrafterEmailBrief.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 4439adea6..9d40e6bef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,4 +9,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). - Add `aiextract` function to extract structured information from text quickly and easily. See `?aiextract` for more information. - Add `aiscan` for image scanning (ie, image comprehension tasks). You can transcribe screenshots or reason over images as if they were text. Images can be provided either as a local file (`image_path`) or as an url (`image_url`). See `?aiscan` for more information. -- Add support for [Ollama.ai](https://ollama.ai/)'s local models. Only `aigenerate` and `aiembed` functions are supported at the moment. \ No newline at end of file +- Add support for [Ollama.ai](https://ollama.ai/)'s local models. Only `aigenerate` and `aiembed` functions are supported at the moment. +- Add a few non-coding templates, eg, verbatim analysis (see `aitemplates("survey")`) and meeting summarization (see `aitemplates("meeting")`), and supporting utilities (non-exported): `split_by_length` and `replace_words` to make it easy to work with smaller open source models. \ No newline at end of file diff --git a/Project.toml b/Project.toml index 76050193d..bdc95d5ea 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ version = "0.2.0-DEV" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" @@ -15,6 +16,7 @@ Aqua = "0.7" Base64 = "<0.0.1, 1" HTTP = "1" JSON3 = "1" +Logging = "<0.0.1, 1" OpenAI = "0.8.7" PrecompileTools = "1" Test = "<0.0.1, 1" diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 466a4e1cb..82e12c7b9 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -1,6 +1,7 @@ module PromptingTools using Base64: base64encode +using Logging using OpenAI using JSON3 using JSON3: StructTypes @@ -61,7 +62,9 @@ function __init__() load_templates!() end -# Enable precompilation to reduce start time -@compile_workload include("precompilation.jl") +# Enable precompilation to reduce start time, disabled logging +with_logger(NullLogger()) do + @compile_workload include("precompilation.jl") +end end # module PromptingTools diff --git a/src/utils.jl b/src/utils.jl index af7b68378..53bacee6b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,4 +1,35 @@ ### USEFUL BUT NOT EXPORTED FUNCTIONS + +""" + replace_words(text::AbstractString, words::Vector{<:AbstractString}; replacement::AbstractString="ABC") + +Replace all occurrences of words in `words` with `replacement` in `text`. Useful to quickly remove specific names or entities from a text. + +# Arguments +- `text::AbstractString`: The text to be processed. +- `words::Vector{<:AbstractString}`: A vector of words to be replaced. +- `replacement::AbstractString="ABC"`: The replacement string to be used. Defaults to "ABC". + +# Example +```julia +text = "Disney is a great company" +replace_words(text, ["Disney", "Snow White", "Mickey Mouse"]) +# Output: "ABC is a great company" +``` +""" +replace_words(text::AbstractString, words::Vector{<:AbstractString}; replacement::AbstractString = "ABC") = replace_words(text, + Regex("\\b$(join(words, "\\b|\\b"))\\b", "i"), + replacement) +function replace_words(text::AbstractString, pattern::Regex, replacement::AbstractString) + replace(text, pattern => replacement) +end +# dispatch for single word +function replace_words(text::AbstractString, + word::AbstractString; + replacement::AbstractString = "ABC") + replace_words(text, [word]; replacement) +end + """ split_by_length(text::String; separator::String=" ", max_length::Int=35000) -> Vector{String} diff --git a/templates/persona-task/AnalystDecisionsInTranscript.json b/templates/persona-task/AnalystDecisionsInTranscript.json new file mode 100644 index 000000000..711a8936f --- /dev/null +++ b/templates/persona-task/AnalystDecisionsInTranscript.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "Template for summarizing transcripts of videos and meetings into decisions made and agreed next steps. If you don't need the instructions, set `instructions=\"None.\"`. Placeholders: {{transcript}}, {{instructions}}", + "version": "1", + "source": "Evolved from [jxnl's Youtube Chapters prompt](https://github.com/jxnl/youtubechapters-backend/blob/main/summary_app/md_summarize.py)", + "_type": "metadatamessage" + }, + { + "content": "Act as a super-human AI analyst trained to meticulously analyze transcripts of videos and meetings. Your role is to identify and summarize key decisions and next steps, enhancing clarity and utility for those studying the transcript. \nUse timestamps to pinpoint when these decisions and steps are discussed. Organize your notes into distinct sections, each dedicated to a significant decision or action plan.\n\nFormat your markdown file using this structure:\n```\n# Key Decision 1: [Descriptive Title] [Timestamp as HH:MM:SS]\n- \n\n## Next Steps for Decision 1\n- \n\nRepeat this structure for each key decision and its corresponding next steps.\n\n# Other Next Steps\n- \n```\n\nFormatting Tips:\n* Ensure each section is substantial, providing a clear and concise summary of each key decision and its next steps.\n* Use bullet points to make the summary easy to scan and understand.\n* All next steps should be actionable and clearly defined. All next steps must be relevant to the decision they are associated with. Any general next steps, should be included in section `Other Next Steps`\n* Include timestamps in brackets to refer to the specific parts of the video where these discussions occur.\n* Titles should be informative, reflecting the essence of the decision.\n\nSummary Tips:\n* Exclude sections where only music plays or no significant content is present.\n* Base your summary strictly on the transcript content without adding extra information.\n* Maintain a clear structure: place a new line after each # or ##, and before each bullet point.\n* Titles should pose a question answered by the decision or describe the nature of the next steps.\n\nKeep the summary concise and focused on key decisions and next steps. \nIf the user provides special instructions, prioritize these over the general guidelines.", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Transcript\n\n{{transcript}}\n\n\n\n# Special Instructions\n\n{{instructions}}", + "variables": [ + "transcript", + "instructions" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/DrafterEmailBrief.json b/templates/persona-task/DrafterEmailBrief.json new file mode 100644 index 000000000..ce74d89cf --- /dev/null +++ b/templates/persona-task/DrafterEmailBrief.json @@ -0,0 +1,21 @@ +[ + { + "content": "Template Metadata", + "description": "Template for quick email drafts. Provide a brief in 5-7 words as headlines, eg, `Follow up email. Sections: Agreements, Next steps` Placeholders: {{brief}}", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "Act as a world-class office communications expert, skilled in creating efficient, clear, and friendly internal email communications.\n Craft a concise email subject and email draft from the provided User Brief. \n\n Use the following format for the body of the email:\n ```\n Section Name \n - Bullet point 1\n - Bullet point 2\n\n \n ```\n\n # Guidelines\n - Focus on clear and efficient communication, suitable for internal business correspondence\n - Where information is missing, use your best judgement to fill in the gaps\n - It should be informal and friendly, eg, start with \"Hi\"\n - Ensure the tone is professional yet casual, suitable for internal communication\n - Write as plain text, with no markdown syntax\n - Format into Sections. Each section should have 3-5 bullet points\n - Close the email on a positive note, encouraging communication and collaboration\n - It should be brief and concise with 150 words or less\n \n\n Follow the above guidelines, unless the user explicitly asks for something different. In that case, follow the user's instructions precisely.\n", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# User Brief\n\n{{brief}}\n\n", + "variables": [ + "brief" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/test/templates.jl b/test/templates.jl index 5a3a1b2c6..335fa0071 100644 --- a/test/templates.jl +++ b/test/templates.jl @@ -32,8 +32,10 @@ end @testset "Templates - search" begin # search all - tmps = aitemplates("") + tmps = aitemplates(""; limit = typemax(Int)) @test tmps == PT.TEMPLATE_METADATA + @info length(tmps) + @info length(PT.TEMPLATE_METADATA) # Exact search for JudgeIsItTrue tmps = aitemplates(:JudgeIsItTrue) @test length(tmps) == 1 diff --git a/test/utils.jl b/test/utils.jl index e3a05cecd..10f268b62 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,7 +1,18 @@ -using PromptingTools: split_by_length +using PromptingTools: split_by_length, replace_words using PromptingTools: _extract_handlebar_variables, _report_stats using PromptingTools: _string_to_vector, _encode_local_image +@testset "replace_words" begin + words = ["Disney", "Snow White", "Mickey Mouse"] + @test replace_words("Disney is a great company", + ["Disney", "Snow White", "Mickey Mouse"]) == "ABC is a great company" + @test replace_words("Snow White and Mickey Mouse are great", + ["Disney", "Snow White", "Mickey Mouse"]) == "ABC and ABC are great" + @test replace_words("LSTM is a great model", "LSTM") == "ABC is a great model" + @test replace_words("LSTM is a great model", "LSTM"; replacement = "XYZ") == + "XYZ is a great model" +end + @testset "split_by_length" begin text = "Hello world. How are you?" chunks = split_by_length(text, max_length = 100) From 8b10ddfbe7ef11a591b1ceb2b3c3d89d05df6729 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:46:50 +0000 Subject: [PATCH 020/251] tag020 --- CHANGELOG.md | 8 ++++++++ Project.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d40e6bef..1219d3438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added + +### Fixed + +## [0.2.0] + ### Added + - Add support for prompt templates with `AITemplate` struct. Search for suitable templates with `aitemplates("query string")` and then simply use them with `aigenerate(AITemplate(:TemplateABC); variableX = "some value") -> AIMessage` or use a dispatch on the template name as a `Symbol`, eg, `aigenerate(:TemplateABC; variableX = "some value") -> AIMessage`. Templates are saved as JSON files in the folder `templates/`. If you add new templates, you can reload them with `load_templates!()` (notice the exclamation mark to override the existing `TEMPLATE_STORE`). - Add `aiextract` function to extract structured information from text quickly and easily. See `?aiextract` for more information. - Add `aiscan` for image scanning (ie, image comprehension tasks). You can transcribe screenshots or reason over images as if they were text. Images can be provided either as a local file (`image_path`) or as an url (`image_url`). See `?aiscan` for more information. diff --git a/Project.toml b/Project.toml index bdc95d5ea..767e62f5f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.2.0-DEV" +version = "0.2.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From 4628843743af691e5618d3d315114bbb31cd2267 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 26 Nov 2023 11:04:22 +0000 Subject: [PATCH 021/251] fail gracefully without api key --- src/PromptingTools.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 82e12c7b9..6825b140c 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -12,7 +12,10 @@ using PrecompileTools const MODEL_CHAT = "gpt-3.5-turbo" const MODEL_EMBEDDING = "text-embedding-ada-002" const API_KEY = get(ENV, "OPENAI_API_KEY", "") -@assert isempty(API_KEY)==false "Please set OPENAI_API_KEY environment variable!" +# Note: Disable this warning by setting OPENAI_API_KEY to anything +isempty(API_KEY) && + @warn "OPENAI_API_KEY environment variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.API_KEY=`!" + # Cost per 1K tokens as of 7th November 2023 const MODEL_COSTS = Dict("gpt-3.5-turbo" => (0.0015, 0.002), "gpt-3.5-turbo-1106" => (0.001, 0.002), From 205f009d9a2eda2367423fb86fb369b6dd22d57d Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Thu, 30 Nov 2023 16:15:38 +0000 Subject: [PATCH 022/251] Use [!TIP] markdown for pro tips --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 40d4c9421..7758e1baa 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ ai"What is the capital of \$(country)?" # AIMessage("The capital of Spain is Madrid.") ``` -Pro tip: Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4` (use `gpt4t` for the new GPT-4 Turbo model). Great for those extra hard questions! +> [!TIP] +> Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4` (use `gpt4t` for the new GPT-4 Turbo model). Great for those extra hard questions! For more complex prompt templates, you can use handlebars-style templating and provide variables as keyword arguments: @@ -53,9 +54,11 @@ msg = aigenerate("What is the capital of {{country}}? Is the population larger t # AIMessage("The capital of Spain is Madrid. And yes, the population of Madrid is larger than 1 million. As of 2020, the estimated population of Madrid is around 3.3 million people.") ``` -Pro tip: Use `asyncmap` to run multiple AI-powered tasks concurrently. +> [!TIP] +> Use `asyncmap` to run multiple AI-powered tasks concurrently. -Pro tip: If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` +> [!TIP] +> If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` For more practical examples, see the `examples/` folder and the [Advanced Examples](#advanced-examples) section below. @@ -225,7 +228,8 @@ prompts = [aigenerate("Translate 'Hello, World!' to {{language}}"; language) for responses = asyncmap(aigenerate, prompts) ``` -Pro tip: You can limit the number of concurrent tasks with the keyword `asyncmap(...; ntasks=10)`. +> [!TIP] +> You can limit the number of concurrent tasks with the keyword `asyncmap(...; ntasks=10)`. ### Model Aliases @@ -491,7 +495,8 @@ Resources: - [OpenAI Documentation](https://platform.openai.com/docs/quickstart?context=python) - [Visual tutorial](https://www.maisieai.com/help/how-to-get-an-openai-api-key-for-chatgpt) -Pro tip: Always set the spending limits! +> [!TIP] +> Always set the spending limits! ### Setting OpenAI Spending Limits @@ -603,4 +608,4 @@ Please note that while PromptingTools.jl aims to provide a smooth experience, it --- -Thank you for choosing PromptingTools.jl to empower your applications with AI! \ No newline at end of file +Thank you for choosing PromptingTools.jl to empower your applications with AI! From 03596887a88f8c9d8cadb89ad596052e0e8aa504 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 30 Nov 2023 21:21:28 +0100 Subject: [PATCH 023/251] up version minor --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 767e62f5f..3a79267cc 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.2.0" +version = "0.3.0-DEV" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From 5b3867f1ad5ba9d5864df2c9289734e0c4b0dda2 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 1 Dec 2023 21:07:54 +0100 Subject: [PATCH 024/251] add codecov token --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a3b9b3e1b..ac5bb55e2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -39,6 +39,7 @@ jobs: - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v3 with: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} files: lcov.info docs: name: Documentation From 97cc11e57b9245b37c8d0da602799117d2452f81 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 3 Dec 2023 11:08:05 +0000 Subject: [PATCH 025/251] update registration + ollama health check --- README.md | 4 ++-- docs/src/examples/working_with_ollama.md | 2 +- docs/src/getting_started.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7758e1baa..41d3769bd 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ For a quick start, simply set it via `ENV["OPENAI_API_KEY"] = "your-api-key"` Install PromptingTools: ```julia using Pkg -Pkg.add("https://github.com/svilupp/PromptingTools.jl") +Pkg.add("PromptingTools.jl") ``` And we're ready to go! @@ -33,7 +33,7 @@ ai"What is the capital of France?" # AIMessage("The capital of France is Paris.") ``` -Returned object is a light wrapper with generated message in field `:content` (eg, `ans.content`) for additional downstream processing. +The returned object is a light wrapper with a generated message in the field `:content` (eg, `ans.content`) for additional downstream processing. You can easily inject any variables with string interpolation: ```julia diff --git a/docs/src/examples/working_with_ollama.md b/docs/src/examples/working_with_ollama.md index b4aeea77a..b7454bc44 100644 --- a/docs/src/examples/working_with_ollama.md +++ b/docs/src/examples/working_with_ollama.md @@ -5,7 +5,7 @@ EditURL = "../../../examples/working_with_ollama.jl" # Local models with Ollama.ai This file contains examples of how to work with [Ollama.ai](https://ollama.ai/) models. -It assumes that you've already installated and launched the Ollama server. For more details or troubleshooting advice, see the [Frequently Asked Questions](@ref) section. +It assumes that you've already installed and launched the Ollama server. Quick check: open the following website in your browser `http://127.0.0.1:11434/` and you should see the message "Ollama is running". For more details or troubleshooting advice, see the [Frequently Asked Questions](@ref) section. First, let's import the package and define a helper link for calling un-exported functions: diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index 3ab182a3d..27245ac8f 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -32,11 +32,11 @@ Resources: ## Installation -The PromptingTools package has not yet been registered. But it can be installed using the following commands: +PromptingTools can be installed using the following commands: ```julia using Pkg -Pkg.add("https://github.com/svilupp/PromptingTools.jl") +Pkg.add("PromptingTools.jl") ``` Throughout the rest of this tutorial, we will assume that you have installed the From 5f1e6bc6fa373ab61a6262f2e1c11c8c1db2b6f7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 3 Dec 2023 11:19:20 +0000 Subject: [PATCH 026/251] update docs + type --- CHANGELOG.md | 1 + docs/src/examples/working_with_ollama.md | 30 ++++++++++++++++++++++++ src/llm_interface.jl | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1219d3438..1f67f8094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Fixed +- Changed type of global `PROMPT_SCHEMA::AbstractPromptSchema` for an easier switch to local models as a default option ## [0.2.0] diff --git a/docs/src/examples/working_with_ollama.md b/docs/src/examples/working_with_ollama.md index b4aeea77a..39be7036c 100644 --- a/docs/src/examples/working_with_ollama.md +++ b/docs/src/examples/working_with_ollama.md @@ -38,6 +38,36 @@ model = "openhermes2.5-mistral" "openhermes2.5-mistral" ```` +## Setting Ollama as a default LLM + +We need to change the global variables for PROMPT_SCHEMA and default models + +```julia +using PromptingTools +const PT = PromptingTools + + +PT.PROMPT_SCHEMA = PT.OllamaManagedSchema() +PT.MODEL_CHAT = "openhermes2.5-mistral" +# You could do the same for PT.MODEL_EMBEDDING +``` + +We can also add a nicer alias for the above Mistral model + +```julia +PT.MODEL_ALIASES["mistral"]= "openhermes2.5-mistral" +# potentially also yi 34bn if you want a bigger more powerful model +PT.MODEL_ALIASES["yi"]= "yi:34b-chat" +``` + +Now, we can use the `@ai_str` macro with Ollama models: +```julia +ai"Say hi to me!" # defaults to mistral because we set MODEL_CHAT above +ai"Say hi to me in Chinese!"yi # defaults to yi 34Bn model +``` + +Note: Another quite popular model is `zephyr:7b-beta` + ## Text Generation with aigenerate ### Simple message diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 3921dcdee..0dc4952f9 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -79,7 +79,7 @@ struct OllamaManagedSchema <: AbstractOllamaManagedSchema end end ## Dispatch into default schema -const PROMPT_SCHEMA = OpenAISchema() +const PROMPT_SCHEMA::AbstractPromptSchema = OpenAISchema() aigenerate(prompt; kwargs...) = aigenerate(PROMPT_SCHEMA, prompt; kwargs...) function aiembed(doc_or_docs, args...; kwargs...) From 5c56f69261e164472ac2f84590c86c73a860dcac Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:51:52 +0000 Subject: [PATCH 027/251] add coding utilities --- CHANGELOG.md | 1 + src/PromptingTools.jl | 5 + src/code_generation.jl | 349 ++++++++++++++++++++++++++++++++++++++++ src/extraction.jl | 2 - test/code_generation.jl | 245 ++++++++++++++++++++++++++++ test/runtests.jl | 15 ++ 6 files changed, 615 insertions(+), 2 deletions(-) create mode 100644 src/code_generation.jl create mode 100644 test/code_generation.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f67f8094..c7d768cfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Introduced a set of utilities for working with generate Julia code (Eg, extract code-fenced Julia code with `PromptingTools.extract_code_blocks` ) or simply apply `AICode` to the AI messages. `AICode` tries to extract, parse and eval Julia code, if it fails both stdout and errors are captured. It is useful for generating Julia code and, in the future, creating self-healing code agents ### Fixed - Changed type of global `PROMPT_SCHEMA::AbstractPromptSchema` for an easier switch to local models as a default option diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 6825b140c..b791094e7 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -52,6 +52,11 @@ const TEMPLATE_METADATA = Vector{AITemplateMetadata}() ## Utilities to support structured extraction include("extraction.jl") +## Utilities to support code generation +export AICode +# Not export extract_code_blocks, extract_function_name +include("code_generation.jl") + ## Individual interfaces include("llm_openai.jl") include("llm_ollama_managed.jl") diff --git a/src/code_generation.jl b/src/code_generation.jl new file mode 100644 index 000000000..3f2e81196 --- /dev/null +++ b/src/code_generation.jl @@ -0,0 +1,349 @@ +# These are utilities to support code generation +# +# Types defined (not exported!): +# - AbstractCodeBlock +# - AICode +# +# Functions defined (not exported!): +# - detect_pkg_operation, extract_julia_imports, detect_missing_packages +# - extract_code_blocks +# - eval! +# +# +# +## # Types + +abstract type AbstractCodeBlock end + +""" + AICode(code::AbstractString; safe_eval::Bool=false, prefix::AbstractString="", suffix::AbstractString="") + +A mutable structure representing a code block (received from the AI model) with automatic parsing, execution, and output/error capturing capabilities. + +Upon instantiation with a string, the `AICode` object automatically runs a code parser and executor (via `PromptingTools.eval!()`), capturing any standard output (`stdout`) or errors. +This structure is useful for programmatically handling and evaluating Julia code snippets. + +See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` + +# Workflow +- Until `cb::AICode` has been evaluated, `cb.success` is set to `nothing` (and so are all other fields). +- The text in `cb.code` is parsed (saved to `cb.expression`). +- The parsed expression is evaluated. +- Outputs of the evaluated expression are captured in `cb.output`. +- Any `stdout` outputs (e.g., from `println`) are captured in `cb.stdout`. +- If an error occurs during evaluation, it is saved in `cb.error`. +- After successful evaluation without errors, `cb.success` is set to `true`. + Otherwise, it is set to `false` and you can inspect the `cb.error` to understand why. + +# Properties +- `code::AbstractString`: The raw string of the code to be parsed and executed. +- `expression`: The parsed Julia expression (set after parsing `code`). +- `stdout`: Captured standard output from the execution of the code. +- `output`: The result of evaluating the code block. +- `success::Union{Nothing, Bool}`: Indicates whether the code block executed successfully (`true`), unsuccessfully (`false`), or has yet to be evaluated (`nothing`). +- `error::Union{Nothing, Exception}`: Any exception raised during the execution of the code block. + +# Keyword Arguments +- `safe_eval::Bool`: If set to `true`, the code block checks for package operations (e.g., installing new packages) and missing imports, and then evaluates the code inside a bespoke scratch module. This is to ensure that the evaluation does not alter any user-defined variables or the global state. Defaults to `false`. +- `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. + Useful to add some additional code definition or necessary imports. Defaults to an empty string. +- `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. + Useful to check that tests pass or that an example executes. Defaults to an empty string. + +# Methods +- `Base.isvalid(cb::AICode)`: Check if the code block has executed successfully. Returns `true` if `cb.success == true`. + +# Examples + +```julia +code = AICode("println(\"Hello, World!\")") # Auto-parses and evaluates the code, capturing output and errors. +isvalid(code) # Output: true +code.stdout # Output: "Hello, World!\n" +``` + +We try to evaluate "safely" by default (eg, inside a custom module, to avoid changing user variables). + You can avoid that with `save_eval=false`: + +```julia +code = AICode("new_variable = 1"; safe_eval=false) +isvalid(code) # Output: true +new_variable # Output: 1 +``` + +You can also call AICode directly on an AIMessage, which will extract the Julia code blocks, concatenate them and evaluate them: + +```julia +msg = aigenerate("In Julia, how do you create a vector of 10 random numbers?") +code = AICode(msg) +# Output: AICode(Success: True, Parsed: True, Evaluated: True, Error Caught: N/A, StdOut: True, Code: 2 Lines) + +# show the code +code.code |> println +# Output: +# numbers = rand(10) +# numbers = rand(1:100, 10) + +# or copy it to the clipboard +code.code |> clipboard + +# or execute it in the current module (=Main) +eval(code.expression) +``` +""" +@kwdef mutable struct AICode <: AbstractCodeBlock + code::AbstractString + expression = nothing + stdout = nothing + output = nothing + success::Union{Nothing, Bool} = nothing + error::Union{Nothing, Exception} = nothing +end +# Eager evaluation if instantiated with a string +function (CB::Type{T})(md::AbstractString; + safe_eval::Bool = true, + prefix::AbstractString = "", + suffix::AbstractString = "") where {T <: AbstractCodeBlock} + cb = CB(; code = md) + eval!(cb; safe_eval, prefix, suffix) +end +Base.isvalid(cb::AbstractCodeBlock) = cb.success == true +function Base.copy(cb::AbstractCodeBlock) + AICode(cb.code, cb.expression, cb.stdout, cb.output, cb.success, cb.error) +end +function Base.show(io::IO, cb::AICode) + success_str = cb.success === nothing ? "N/A" : titlecase(string(cb.success)) + expression_str = cb.expression === nothing ? "N/A" : "True" + stdout_str = cb.stdout === nothing ? "N/A" : "True" + output_str = cb.output === nothing ? "N/A" : "True" + error_str = cb.error === nothing ? "N/A" : "True" + count_lines = count(==('\n'), collect(cb.code)) + 1 # there is always at least one line + + print(io, + "AICode(Success: $success_str, Parsed: $expression_str, Evaluated: $output_str, Error Caught: $error_str, StdOut: $stdout_str, Code: $count_lines Lines)") +end + +## Overload for AIMessage - simply extracts the code blocks and concatenates them +function AICode(msg::AIMessage; kwargs...) + code = extract_code_blocks(msg.content) |> Base.Fix2(join, "\n") + return AICode(code; kwargs...) +end + +## # Functions + +# Utility to detect if Pkg.* is called in a string (for `safe` code evaluation) +function detect_pkg_operation(input::AbstractString) + m = match(r"\bPkg.[a-z]", input) + return !isnothing(m) +end +# Utility to detect dependencies in a string (for `safe` code evaluation / understand when we don't have a necessary package) +function extract_julia_imports(input::AbstractString) + package_names = Symbol[] + for line in split(input, "\n") + if occursin(r"(^using |^import )"m, line) + subparts = replace(replace(line, "using" => ""), "import" => "") + ## TODO: add split on . + subparts = map(x -> contains(x, ':') ? split(x, ':')[1] : x, + split(subparts, ",")) + subparts = replace(join(subparts, ' '), ',' => ' ') + packages = filter(!isempty, split(subparts, " ")) .|> Symbol + append!(package_names, packages) + end + end + return package_names +end + +# Utility to pinpoint unavailable dependencies +function detect_missing_packages(imports_required::AbstractVector{<:Symbol}) + available_packages = Base.loaded_modules |> values .|> Symbol + missing_packages = filter(pkg -> !in(pkg, available_packages), imports_required) + if length(missing_packages) > 0 + return true, missing_packages + else + return false, Symbol[] + end +end + +""" + extract_code_blocks(markdown_content::String) -> Vector{String} + +Extract Julia code blocks from a markdown string. + +This function searches through the provided markdown content, identifies blocks of code specifically marked as Julia code +(using the ```julia ... ``` code fence patterns), and extracts the code within these blocks. +The extracted code blocks are returned as a vector of strings, with each string representing one block of Julia code. + +Note: Only the content within the code fences is extracted, and the code fences themselves are not included in the output. + +# Arguments +- `markdown_content::String`: A string containing the markdown content from which Julia code blocks are to be extracted. + +# Returns +- `Vector{String}`: A vector containing strings of extracted Julia code blocks. If no Julia code blocks are found, an empty vector is returned. + +# Examples + +Example with a single Julia code block +```julia +markdown_single = \""" +```julia +println("Hello, World!") +``` +\""" +extract_code_blocks(markdown_single) +# Output: [\"Hello, World!\"] +``` + +```julia +# Example with multiple Julia code blocks +markdown_multiple = \""" +```julia +x = 5 +``` +Some text in between +```julia +y = x + 2 +``` +\""" +extract_code_blocks(markdown_multiple) +# Output: ["x = 5", "y = x + 2"] +``` +""" +function extract_code_blocks(markdown_content::AbstractString) + # Define the pattern for Julia code blocks + pattern = r"```julia\n(.*?)\n```"s + + # Find all matches and extract the code + matches = eachmatch(pattern, markdown_content) + + # Extract and clean the code blocks + code_blocks = String[m.captures[1] for m in matches] + + return code_blocks +end + +""" + extract_function_name(code_block::String) -> Union{String, Nothing} + +Extract the name of a function from a given Julia code block. The function searches for two patterns: +- The explicit function declaration pattern: `function name(...) ... end` +- The concise function declaration pattern: `name(...) = ...` + +If a function name is found, it is returned as a string. If no function name is found, the function returns `nothing`. + +# Arguments +- `code_block::String`: A string containing Julia code. + +# Returns +- `Union{String, Nothing}`: The extracted function name or `nothing` if no name is found. + +# Example +```julia +code = \""" +function myFunction(arg1, arg2) + # Function body +end +\""" +extract_function_name(code) +# Output: "myFunction" +``` +""" +function extract_function_name(code_block::AbstractString) + # Regular expression for the explicit function declaration + pattern_explicit = r"function\s+(\w+)\(" + # Regular expression for the concise function declaration + pattern_concise = r"^(\w+)\(.*\)\s*=" + + # Searching for the explicit function declaration + match_explicit = match(pattern_explicit, code_block) + if match_explicit !== nothing + return match_explicit.captures[1] + end + + # Searching for the concise function declaration + match_concise = match(pattern_concise, code_block) + if match_concise !== nothing + return match_concise.captures[1] + end + + # Return nothing if no function name is found + return nothing +end + +""" + eval!(cb::AICode; safe_eval::Bool=true, prefix::AbstractString="", suffix::AbstractString="") + +Evaluates a code block `cb` in-place. It runs automatically when AICode is instantiated with a String. + +Check the outcome of evaluation with `Base.isvalid(cb)`. If `==true`, provide code block has executed successfully. + +Steps: +- If `cb::AICode` has not been evaluated, `cb.success = nothing`. + After the evaluation it will be either `true` or `false` depending on the outcome +- Parse the text in `cb.code` +- Evaluate the parsed expression +- Capture outputs of the evaluated in `cb.output` +- Capture any stdout outputs (eg, test failures) in `cb.stdout` +- If any error exception is raised, it is saved in `cb.error` +- Finally, if all steps were successful, success is set to `cb.success = true` + +# Keyword Arguments +- `safe_eval::Bool`: If `true`, we first check for any Pkg operations (eg, installing new packages) and missing imports, + then the code will be evaluated inside a bespoke scratch module (not to change any user variables) +- `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. + Useful to add some additional code definition or necessary imports. Defaults to an empty string. +- `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. + Useful to check that tests pass or that an example executes. Defaults to an empty string. +""" +function eval!(cb::AbstractCodeBlock; + safe_eval::Bool = true, + prefix::AbstractString = "", + suffix::AbstractString = "") + (; code) = cb + code_extra = string(prefix, "\n", code, "\n", suffix) + ## Safety checks on `code` only + if safe_eval + detect_pkg_operation(code) && + throw(error("Error: Use of package manager (`Pkg.*`) detected! Please verify the safety of the code or disable the safety check (`safe_eval=false`)")) + detected, missing_packages = detect_missing_packages(extract_julia_imports(code)) + detected && + throw(error("Error: Failed package import. Missing packages: $(join(string.(missing_packages),", ")). Please add them or disable the safety check (`safe_eval=false`)")) + end + ## Parse into an expression + try + ex = Meta.parseall(code_extra) + cb.expression = ex + catch e + cb.error = e + cb.success = false + return cb + end + + ## Eval + safe_module = gensym("SafeCustomModule") + # Prepare to catch any stdout + pipe = Pipe() + redirect_stdout(pipe) do + try + # eval in Main module to have access to std libs, but inside a custom module for safety + if safe_eval + cb.output = @eval(Main, module $safe_module + using Test # just in case unit tests are provided + $(cb.expression) + end) + else + # Evaluate the code directly into Main + cb.output = @eval(Main, begin + using Test # just in case unit tests are provided + $(cb.expression) + end) + end + cb.success = true + catch e + cb.error = e + cb.success = false + end + end + close(Base.pipe_writer(pipe)) + cb.stdout = read(pipe, String) + return cb +end \ No newline at end of file diff --git a/src/extraction.jl b/src/extraction.jl index b96a290a6..206ff711d 100644 --- a/src/extraction.jl +++ b/src/extraction.jl @@ -1,7 +1,5 @@ # These are utilities to support structured data extraction tasks through the OpenAI function calling interface (wrapped by `aiextract`) # -# TODOs: -# - add support for enums to_json_type(s::Type{<:AbstractString}) = "string" to_json_type(n::Type{<:Real}) = "number" to_json_type(n::Type{<:Integer}) = "integer" diff --git a/test/code_generation.jl b/test/code_generation.jl new file mode 100644 index 000000000..4a86142a3 --- /dev/null +++ b/test/code_generation.jl @@ -0,0 +1,245 @@ +using PromptingTools: extract_julia_imports +using PromptingTools: detect_pkg_operation, detect_missing_packages, extract_function_name +using PromptingTools: extract_code_blocks, eval! + +@testset "extract_imports tests" begin + @test extract_julia_imports("using Test, LinearAlgebra") == + Symbol.(["Test", "LinearAlgebra"]) + @test extract_julia_imports("import Test\nimport ABC,DEF\nusing GEM: func") == + Symbol.(["Test", "ABC", "DEF", "GEM"]) + @test extract_julia_imports("import PackageA.PackageB: funcA\nimport PackageC") == + Symbol.(["PackageA.PackageB", "PackageC"]) +end + +@testset "detect_missing_packages" begin + @test detect_missing_packages(Symbol[]) == (false, Symbol[]) + @test detect_missing_packages(Symbol.(["Test"])) == (false, Symbol[]) + @test detect_missing_packages(Symbol.(["Test", "Base", "Main"])) == (false, Symbol[]) + @test detect_missing_packages(Symbol.(["Test", + "Base", + "Main", + "SpecialPackage12345678", "SpecialPackage123456789"])) == (true, [:SpecialPackage12345678, :SpecialPackage123456789]) +end + +@testset "detect_pkg_operation" begin + @test detect_pkg_operation("Pkg.activate(\".\")") == true + @test detect_pkg_operation("Pkg.add(\"SomePkg\")") == true + @test detect_pkg_operation("blabla Pkg.activate(\".\")") == true + @test detect_pkg_operation("hello world;") == false + @test detect_pkg_operation("import Pkg;") == false +end + +@testset "extract_code_blocks" begin + # Single Julia Code Block + markdown_content = """ + # Example + ```julia + println("Hello, World!") + ``` + """ + @test extract_code_blocks(markdown_content) == ["println(\"Hello, World!\")"] + + # Multiple Julia Code Blocks + markdown_content = """ + ```julia + println("First Block") + ``` + Some text here. + ```julia + println("Second Block") + ``` + """ + @test extract_code_blocks(markdown_content) == + ["println(\"First Block\")", "println(\"Second Block\")"] + + # No Julia Code Blocks + markdown_content = """ + This is a text without Julia code blocks. + """ + @test isempty(extract_code_blocks(markdown_content)) + + # Mixed Language Code Blocks + markdown_content = """ + ```python + print("This is Python") + ``` + ```julia + println("This is Julia") + ``` + """ + @test extract_code_blocks(markdown_content) == ["println(\"This is Julia\")"] + + # Nested Code Blocks" + markdown_content = """ + ``` + ```julia + println("Nested Block") + ``` + ``` + """ + @test extract_code_blocks(markdown_content) == ["println(\"Nested Block\")"] +end + +@testset "extract_function_name" begin + # Test 1: Test an explicit function declaration + @test extract_function_name("function testFunction1()\nend") == "testFunction1" + + # Test 2: Test a concise function declaration + @test extract_function_name("testFunction2() = 42") == "testFunction2" + + # Test 3: Test a code block with no function + @test extract_function_name("let a = 10\nb = 20\nend") === nothing + + # Test 4: Test a code block with a multiline function and comments + @test extract_function_name(""" + # Comment line + function testFunction3(arg1, arg2) + # Function body + return arg1 + arg2 + end + """) == "testFunction3" + + # Test 5: Test a code block with multiple functions, should return the first function's name + @test extract_function_name(""" + function firstFunction() + end + + function secondFunction() + end + """) == "firstFunction" +end + +@testset "eval!" begin + # Test that it captures stdout and output + let cb = AICode(; code = """ + println("Hello") + a=1 + """) + eval!(cb) + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test isvalid(cb) + @test cb.stdout == "Hello\n" + @test cb.output.a == 1 + end + # Test that it captures parsing errors + let cb = AICode(; code = """ + a=1 + + mla;sda b=2 + """) + eval!(cb) + @test cb.success == false + @test !isvalid(cb) + @test cb.error isa Base.Meta.ParseError + end + # Test that it captures execution errors + let cb = AICode(; code = """ + a=1 + b # b not defined yet + b=2 + """) + eval!(cb) + @test cb.success == false + @test cb.error == UndefVarError(:b) + @test !isnothing(cb.expression) # parsed + end +end +## Addition, needs to be outside of @testset +# Test that it captures test failures, we need to move it to the main file as it as it doesn't work inside a testset +# let cb = AICode(; code = """ +# @test 1==2 +# """) +# eval!(cb) +# @test cb.success == false +# @info cb.error cb.output +# @test cb.error isa Test.FallbackTestSetException +# @test !isnothing(cb.expression) # parsed +# @test occursin("Test Failed", cb.stdout) # capture details of the test failure +# @test isnothing(cb.output) # because it failed +# end + +@testset "eval! kwargs" begin + ## Safe Eval == true mode + # package that is not available + cb = AICode(; code = "using ExoticPackage123") + @test_throws Exception eval!(cb) + @test_throws "ExoticPackage123" eval!(cb) + # Pkg operations + cb = AICode(; code = "Pkg.activate(\".\")") + @test_throws Exception eval!(cb) + # Evaluate inside a gensym'd module + cb = AICode(; code = "a=1") |> eval! + @test occursin("SafeCustomModule", string(cb.output)) + + ## Safe Eval == false mode + # package that is not available + cb = AICode(; code = "using ExoticPackage123") + eval!(cb; safe_eval = false) + @test !isvalid(cb) + @test cb.error isa ArgumentError # now it's caught by REPL that we don't have the package + # Pkg operations + cb = AICode(; code = "import Pkg; Pkg.status()") + eval!(cb; safe_eval = false) + # This works but in test mode, Julia claims it doesn't have Pkg package... + # @test isvalid(cb) + # Evaluate in Main directly + cb = AICode(; code = "a123=123") + eval!(cb; safe_eval = false) + @test cb.output == 123 + @test a123 == 123 + + # Test prefix and suffix + cb = AICode(; code = "") + eval!(cb; prefix = "a=1", suffix = "b=2") + @test cb.output.a == 1 + @test cb.output.b == 2 +end + +@testset "AICode constructors" begin + # Initiate from provided text + let cb = AICode(""" + println("Hello") + a=1 + """) + # eval! is automatic + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test cb.stdout == "Hello\n" + @test cb.output.a == 1 + end + + # From AI Message + let msg = AIMessage(""" +```julia +println(\"hello\") +``` +Some text +```julia +println(\"world\") +b=2 +``` +""") + cb = AICode(msg) + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test cb.stdout == "hello\nworld\n" + @test cb.output.b == 2 + end +end + +## Create eval object +evaluation = Dict("name" => definition["name"], "parsed" => !isnothing(cb.expression), + "executed" => isvalid(cb), + "unit_tests_passed" => test_count, "examples_executed" => example_count, + "tokens" => msg.tokens, + "elapsed_seconds" => msg.elapsed, "cost" => get_query_cost(msg, model), + "model" => model, + "timestamp" => timestamp, "prompt_strategy" => prompt_strategy) + +eval = (; name = definition["name"], parsed = !isnothing(cb.expression), + executed = isvalid(cb), + unit_tests_passed = test_count, examples_executed = example_count, tokens = msg.tokens, + elapsed_seconds = msg.elapsed, cost = get_query_cost(msg, model), model = model, + timestamp = timestamp, prompt_strategy = prompt_strategy) \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index cfb4e2482..2ba589667 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,4 +14,19 @@ end include("extraction.jl") include("llm_openai.jl") include("templates.jl") + include("code_generation.jl") end + +# Part of code_generation.jl / @testset "eval!" begin +# Test that it captures test failures, we need to move it to the main file as it as it doesn't work inside a testset +let cb = AICode(; code = """ + @test 1==2 + """) + eval!(cb) + @test cb.success == false + @info cb.error cb.output + @test cb.error isa Test.FallbackTestSetException + @test !isnothing(cb.expression) # parsed + @test occursin("Test Failed", cb.stdout) # capture details of the test failure + @test isnothing(cb.output) # because it failed +end \ No newline at end of file From 65b847698d80b741e7f6895dd3ac4deadddc9c47 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:53:29 +0000 Subject: [PATCH 028/251] up --- test/code_generation.jl | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/code_generation.jl b/test/code_generation.jl index 4a86142a3..7a86b22d4 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -228,18 +228,3 @@ b=2 @test cb.output.b == 2 end end - -## Create eval object -evaluation = Dict("name" => definition["name"], "parsed" => !isnothing(cb.expression), - "executed" => isvalid(cb), - "unit_tests_passed" => test_count, "examples_executed" => example_count, - "tokens" => msg.tokens, - "elapsed_seconds" => msg.elapsed, "cost" => get_query_cost(msg, model), - "model" => model, - "timestamp" => timestamp, "prompt_strategy" => prompt_strategy) - -eval = (; name = definition["name"], parsed = !isnothing(cb.expression), - executed = isvalid(cb), - unit_tests_passed = test_count, examples_executed = example_count, tokens = msg.tokens, - elapsed_seconds = msg.elapsed, cost = get_query_cost(msg, model), model = model, - timestamp = timestamp, prompt_strategy = prompt_strategy) \ No newline at end of file From 7b7307fd93a3ec9a710932c99c329b0d8546594c Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:51:28 +0000 Subject: [PATCH 029/251] fix test --- test/code_generation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/code_generation.jl b/test/code_generation.jl index 7a86b22d4..ff725cf44 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -131,7 +131,7 @@ end eval!(cb) @test cb.success == false @test !isvalid(cb) - @test cb.error isa Base.Meta.ParseError + @test cb.error isa Exception # can be Base.Meta.ParseError or ErrorException depending on Julia version end # Test that it captures execution errors let cb = AICode(; code = """ From 0554a5bf059b7c764dbef37d3d8132b3739e63e7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:45:36 +0000 Subject: [PATCH 030/251] new conversation interface --- src/PromptingTools.jl | 1 + src/llm_interface.jl | 5 + src/llm_openai.jl | 250 +++++++++++++++++++++++++----------------- src/llm_shared.jl | 87 +++++++++++++++ src/messages.jl | 4 + 5 files changed, 248 insertions(+), 99 deletions(-) create mode 100644 src/llm_shared.jl diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index b791094e7..d89528b52 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -58,6 +58,7 @@ export AICode include("code_generation.jl") ## Individual interfaces +include("llm_shared.jl") include("llm_openai.jl") include("llm_ollama_managed.jl") diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 0dc4952f9..3f841800d 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -13,10 +13,15 @@ function aiembed end function aiclassify end function aiextract end function aiscan end +# Re-usable blocks are defined in src/llm_shared.jl ## Prompt Schema "Defines different prompting styles based on the model training and fine-tuning." abstract type AbstractPromptSchema end + +"Schema that keeps messages (<:AbstractMessage) and does not transform for any specific model. It used by the first pass of the prompt rendering system (see `?render`)." +struct NoSchema <: AbstractPromptSchema end + abstract type AbstractOpenAISchema <: AbstractPromptSchema end """ diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 30b956627..4a8185629 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -17,32 +17,26 @@ function render(schema::AbstractOpenAISchema, kwargs...) ## @assert image_detail in ["auto", "high", "low"] "Image detail must be one of: auto, high, low" - ## + ## First pass: keep the message types but make the replacements provided in `kwargs` + messages_replaced = render(NoSchema(), messages; kwargs...) + + ## Second pass: convert to the OpenAI schema conversation = Dict{String, Any}[] - # TODO: concat multiple system messages together (2nd pass) - has_system_msg = false # replace any handlebar variables in the messages - for msg in messages - if msg isa SystemMessage - replacements = ["{{$(key)}}" => value - for (key, value) in pairs(kwargs) if key in msg.variables] - # move it to the front - pushfirst!(conversation, - Dict("role" => "system", - "content" => replace(msg.content, replacements...))) - has_system_msg = true - elseif msg isa UserMessage - replacements = ["{{$(key)}}" => value - for (key, value) in pairs(kwargs) if key in msg.variables] - push!(conversation, - Dict("role" => "user", "content" => replace(msg.content, replacements...))) - elseif msg isa UserMessageWithImages - replacements = ["{{$(key)}}" => value - for (key, value) in pairs(kwargs) if key in msg.variables] + for msg in messages_replaced + role = if msg isa SystemMessage + "system" + elseif msg isa UserMessage || msg isa UserMessageWithImages + "user" + elseif msg isa AIMessage + "assistant" + end + ## Special case for images + if msg isa UserMessageWithImages # Build message content content = Dict{String, Any}[Dict("type" => "text", - "text" => replace(msg.content, replacements...))] + "text" => msg.content)] # Add images for img in msg.image_url push!(content, @@ -50,26 +44,22 @@ function render(schema::AbstractOpenAISchema, "image_url" => Dict("url" => img, "detail" => image_detail))) end - push!(conversation, Dict("role" => "user", "content" => content)) - elseif msg isa AIMessage - push!(conversation, - Dict("role" => "assistant", "content" => msg.content)) + else + content = msg.content end - # Note: Ignores any DataMessage or other types + push!(conversation, Dict("role" => role, "content" => content)) end - ## Add default system prompt if not provided - !has_system_msg && pushfirst!(conversation, - Dict("role" => "system", "content" => "Act as a helpful AI assistant")) return conversation end ## User-Facing API """ - aigenerate([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, - model::String = MODEL_CHAT, - http_kwargs::NamedTuple = (; - retry_non_idempotent = true, + aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; + verbose::Bool = true, + api_key::String = API_KEY, + model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), kwargs...) @@ -82,15 +72,22 @@ Generate an AI response based on a given prompt using the OpenAI API. - `verbose`: A boolean indicating whether to print additional information. - `api_key`: A string representing the API key for accessing the OpenAI API. - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `http_kwargs`: A named tuple of HTTP keyword arguments. - `api_kwargs`: A named tuple of API keyword arguments. - `kwargs`: Prompt variables to be used to fill the prompt/template # Returns + +If `return_all=false` (default): - `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. Use `msg.content` to access the extracted string. -See also: `ai_str`, `aai_str`, `aiembed`, `aiclassify`, `aiextract`, `aiscan` +If `return_all=true`: +- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`AIMessage`). + +See also: `ai_str`, `aai_str`, `aiembed`, `aiclassify`, `aiextract`, `aiscan`, `aitemplates` # Example @@ -129,7 +126,7 @@ msg=aigenerate(conversation) function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, api_key::String = API_KEY, - model::String = MODEL_CHAT, + model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), @@ -139,20 +136,28 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) conversation = render(prompt_schema, prompt; kwargs...) - time = @elapsed r = create_chat(prompt_schema, api_key, - model_id, - conversation; - http_kwargs, - api_kwargs...) - msg = AIMessage(; content = r.response[:choices][begin][:message][:content] |> strip, - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], - r.response[:usage][:completion_tokens]), - elapsed = time) - ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) - return msg + if !dry_run + time = @elapsed r = create_chat(prompt_schema, api_key, + model_id, + conversation; + http_kwargs, + api_kwargs...) + msg = AIMessage(; + content = r.response[:choices][begin][:message][:content] |> strip, + status = Int(r.status), + tokens = (r.response[:usage][:prompt_tokens], + r.response[:usage][:completion_tokens]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + else + msg = nothing + end + ## Select what to return + output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) + + return output end # Extend OpenAI create_chat to allow for testing/debugging function OpenAI.create_chat(schema::AbstractOpenAISchema, @@ -176,7 +181,7 @@ end postprocess::F = identity; verbose::Bool = true, api_key::String = API_KEY, - model::String = MODEL_EMBEDDING, + model::String = MODEL_EMBEDDING, return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), @@ -192,12 +197,18 @@ The `aiembed` function generates embeddings for the given input using a specifie - `verbose::Bool`: A flag indicating whether to print verbose information. Defaults to `true`. - `api_key::String`: The API key to use for the OpenAI API. Defaults to `API_KEY`. - `model::String`: The model to use for generating embeddings. Defaults to `MODEL_EMBEDDING`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to `(retry_non_idempotent = true, retries = 5, readtimeout = 120)`. - `api_kwargs::NamedTuple`: Additional keyword arguments for the OpenAI API. Defaults to an empty `NamedTuple`. - `kwargs...`: Additional keyword arguments. ## Returns -- `msg`: A `DataMessage` object containing the embeddings, status, token count, and elapsed time. +If `return_all=false` (default): +- `msg`: A `DataMessage` object containing the embeddings, status, token count, and elapsed time. Use `msg.content` to access the embeddings. + +If `return_all=true`: +- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`DataMessage`). # Example @@ -228,6 +239,7 @@ function aiembed(prompt_schema::AbstractOpenAISchema, postprocess::F = identity; verbose::Bool = true, api_key::String = API_KEY, model::String = MODEL_EMBEDDING, + return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), @@ -236,20 +248,28 @@ function aiembed(prompt_schema::AbstractOpenAISchema, global MODEL_ALIASES, MODEL_COSTS ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) - time = @elapsed r = create_embeddings(prompt_schema, api_key, - doc_or_docs, - model_id; - http_kwargs, - api_kwargs...) - msg = DataMessage(; - content = mapreduce(x -> postprocess(x[:embedding]), hcat, r.response[:data]), - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], 0), - elapsed = time) - ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) - return msg + if !dry_run + time = @elapsed r = create_embeddings(prompt_schema, api_key, + doc_or_docs, + model_id; + http_kwargs, + api_kwargs...) + msg = DataMessage(; + content = mapreduce(x -> postprocess(x[:embedding]), hcat, r.response[:data]), + status = Int(r.status), + tokens = (r.response[:usage][:prompt_tokens], 0), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + else + msg = nothing + end + + ## Select what to return + output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) + + return output end # Extend OpenAI create_embeddings to allow for testing function OpenAI.create_embeddings(schema::AbstractOpenAISchema, @@ -332,6 +352,7 @@ end return_type::Type, verbose::Bool = true, model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (; retry_non_idempotent = true, retries = 5, @@ -353,14 +374,21 @@ It's effectively a light wrapper around `aigenerate` call, which requires additi - `verbose`: A boolean indicating whether to print additional information. - `api_key`: A string representing the API key for accessing the OpenAI API. - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `http_kwargs`: A named tuple of HTTP keyword arguments. - `api_kwargs`: A named tuple of API keyword arguments. - `kwargs`: Prompt variables to be used to fill the prompt/template # Returns +If `return_all=false` (default): - `msg`: An `DataMessage` object representing the extracted data, including the content, status, tokens, and elapsed time. Use `msg.content` to access the extracted data. +If `return_all=true`: +- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`DataMessage`). + + See also: `function_call_signature`, `MaybeExtract`, `aigenerate` # Example @@ -430,6 +458,7 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T verbose::Bool = true, api_key::String = API_KEY, model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), @@ -444,29 +473,36 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) conversation = render(prompt_schema, prompt; kwargs...) - time = @elapsed r = create_chat(prompt_schema, api_key, - model_id, - conversation; - http_kwargs, - api_kwargs...) - # "Safe" parsing of the response - it still fails if JSON is invalid - content = try - r.response[:choices][begin][:message][:function_call][:arguments] |> - x -> JSON3.read(x, return_type) - catch e - @warn "There was an error parsing the response: $e. Using the raw response instead." - r.response[:choices][begin][:message][:function_call][:arguments] |> - JSON3.read |> copy + + if !dry_run + time = @elapsed r = create_chat(prompt_schema, api_key, + model_id, + conversation; + http_kwargs, + api_kwargs...) + # "Safe" parsing of the response - it still fails if JSON is invalid + content = try + r.response[:choices][begin][:message][:function_call][:arguments] |> + x -> JSON3.read(x, return_type) + catch e + @warn "There was an error parsing the response: $e. Using the raw response instead." + r.response[:choices][begin][:message][:function_call][:arguments] |> + JSON3.read |> copy + end + msg = DataMessage(; content, + status = Int(r.status), + tokens = (r.response[:usage][:prompt_tokens], + r.response[:usage][:completion_tokens]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + else + msg = nothing end - msg = DataMessage(; content, - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], - r.response[:usage][:completion_tokens]), - elapsed = time) - ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + ## Select what to return + output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) - return msg + return output end """ @@ -477,6 +513,7 @@ aiscan([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; attach_to_latest::Bool = true, verbose::Bool = true, model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (; retry_non_idempotent = true, retries = 5, @@ -501,15 +538,21 @@ It's effectively a light wrapper around `aigenerate` call, which uses additional - `verbose`: A boolean indicating whether to print additional information. - `api_key`: A string representing the API key for accessing the OpenAI API. - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `http_kwargs`: A named tuple of HTTP keyword arguments. - `api_kwargs`: A named tuple of API keyword arguments. - `kwargs`: Prompt variables to be used to fill the prompt/template # Returns +If `return_all=false` (default): - `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. Use `msg.content` to access the extracted string. -See also: `ai_str`, `aai_str`, `aigenerate`, `aiembed`, `aiclassify`, `aiextract` +If `return_all=true`: +- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`AIMessage`). + +See also: `ai_str`, `aai_str`, `aigenerate`, `aiembed`, `aiclassify`, `aiextract`, `aitemplates` # Notes @@ -559,6 +602,7 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE verbose::Bool = true, api_key::String = API_KEY, model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = (; max_tokens = 2500), @@ -571,19 +615,27 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE msgs = attach_images_to_user_message(prompt; image_url, image_path, attach_to_latest) ## Build the conversation, pass what image detail is required (if provided) conversation = render(prompt_schema, msgs; image_detail, kwargs...) - ## Model call - time = @elapsed r = create_chat(prompt_schema, api_key, - model_id, - conversation; - http_kwargs, - api_kwargs...) - msg = AIMessage(; content = r.response[:choices][begin][:message][:content] |> strip, - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], - r.response[:usage][:completion_tokens]), - elapsed = time) - ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + if !dry_run + ## Model call + time = @elapsed r = create_chat(prompt_schema, api_key, + model_id, + conversation; + http_kwargs, + api_kwargs...) + msg = AIMessage(; + content = r.response[:choices][begin][:message][:content] |> strip, + status = Int(r.status), + tokens = (r.response[:usage][:prompt_tokens], + r.response[:usage][:completion_tokens]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + else + msg = nothing + end - return msg + ## Select what to return // input `msgs` to preserve the image attachments + output = finalize_outputs(msgs, conversation, msg; return_all, dry_run, kwargs...) + + return output end \ No newline at end of file diff --git a/src/llm_shared.jl b/src/llm_shared.jl new file mode 100644 index 000000000..6013bd546 --- /dev/null +++ b/src/llm_shared.jl @@ -0,0 +1,87 @@ +# Reusable functionality across different schemas +""" + render(schema::NoSchema, + messages::Vector{<:AbstractMessage}; + replacement_kwargs...) + +Renders a conversation history from a vector of messages with all replacement variables specified in `replacement_kwargs`. + +It is the first pass of the prompt rendering system, and is used by all other schemas. + +# Notes +- All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. +- If a SystemMessage is missing, we inject a default one at the beginning of the conversation. +""" +function render(schema::NoSchema, + messages::Vector{<:AbstractMessage}; + replacement_kwargs...) + ## + conversation = AbstractMessage[] + has_system_msg = false + # TODO: concat multiple system messages together (2nd pass) + + # replace any handlebar variables in the messages + for msg in messages + if msg isa Union{SystemMessage, UserMessage, UserMessageWithImages} + replacements = ["{{$(key)}}" => value + for (key, value) in pairs(replacement_kwargs) + if key in msg.variables] + # Rebuild the message with the replaced content + MSGTYPE = typeof(msg) + new_msg = MSGTYPE(; + # unpack the type to replace only the content field + [(field, getfield(msg, field)) for field in fieldnames(typeof(msg))]..., + content = replace(msg.content, replacements...)) + if msg isa SystemMessage + has_system_msg = true + # move to the front + pushfirst!(conversation, new_msg) + else + push!(conversation, new_msg) + end + elseif msg isa AIMessage + # no replacements + push!(conversation, msg) + else + # Note: Ignores any DataMessage or other types for the prompt/conversation history + @warn "Unexpected message type: $(typeof(msg)). Skipping." + end + end + ## Add default system prompt if not provided + !has_system_msg && pushfirst!(conversation, + SystemMessage("Act as a helpful AI assistant")) + + return conversation +end + +""" + finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conversation::AbstractVector, + msg::AbstractMessage; + return_all::Bool = false, + dry_run::Bool = false, + kwargs...) + +Finalizes the outputs of the ai* functions by either returning the conversation history or the last message. + +# Keyword arguments +- `return_all::Bool=false`: If true, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If true, does not send the messages to the model, but only renders the prompt with the given schema and replacement variables. + Useful for debugging when you want to check the specific schema rendering. +- `kwargs...`: Variables to replace in the prompt template. +""" +function finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conversation::AbstractVector, + msg::AbstractMessage; + return_all::Bool = false, + dry_run::Bool = false, + kwargs...) + if return_all + if !dry_run + # If not a dry_run, re-create the messages sent to the model before schema application + conversation = render(NoSchema(), prompt; kwargs...) + push!(conversation, msg) + end + return conversation + else + return msg + end +end \ No newline at end of file diff --git a/src/messages.jl b/src/messages.jl index e9d206d56..fc1640ec3 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -64,6 +64,10 @@ Base.var"=="(m1::AbstractMessage, m2::AbstractMessage) = false function Base.var"=="(m1::T, m2::T) where {T <: AbstractMessage} all([getproperty(m1, f) == getproperty(m2, f) for f in fieldnames(T)]) end +Base.length(t::AbstractMessage) = nfields(t) +function Base.iterate(t::AbstractMessage, iter = 1) + iter > nfields(t) ? nothing : (getfield(t, iter), iter + 1) +end ## Vision Models -- Constructor and Conversion "Construct `UserMessageWithImages` with 1 or more images. Images can be either URLs or local paths." From bf365a4fcdb515ff693a7a73711b82b2c3d69a6d Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:11:50 +0000 Subject: [PATCH 031/251] up --- src/llm_ollama_managed.jl | 31 +++++++++++++++++++---------- src/llm_openai.jl | 42 +++++++++++++-------------------------- test/runtests.jl | 1 - 3 files changed, 35 insertions(+), 39 deletions(-) diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 5a0c46df5..56ae60153 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -104,6 +104,7 @@ end """ aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) @@ -115,6 +116,8 @@ Generate an AI response based on a given prompt using the OpenAI API. - `verbose`: A boolean indicating whether to print additional information. - `api_key`: Provided for interface consistency. Not needed for locally hosted Ollama. - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to empty `NamedTuple`. - `api_kwargs::NamedTuple`: Additional keyword arguments for the Ollama API. Defaults to an empty `NamedTuple`. - `kwargs`: Prompt variables to be used to fill the prompt/template @@ -175,6 +178,7 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ verbose::Bool = true, api_key::String = API_KEY, model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## @@ -182,17 +186,24 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) conversation = render(prompt_schema, prompt; kwargs...) - time = @elapsed resp = ollama_api(prompt_schema, conversation.prompt; - conversation.system, endpoint = "generate", model, http_kwargs, api_kwargs...) - msg = AIMessage(; content = resp.response[:response] |> strip, - status = Int(resp.status), - tokens = (resp.response[:prompt_eval_count], - resp.response[:eval_count]), - elapsed = time) - ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) - return msg + if !dry_run + time = @elapsed resp = ollama_api(prompt_schema, conversation.prompt; + conversation.system, endpoint = "generate", model, http_kwargs, api_kwargs...) + msg = AIMessage(; content = resp.response[:response] |> strip, + status = Int(resp.status), + tokens = (resp.response[:prompt_eval_count], + resp.response[:eval_count]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + else + msg = nothing + end + + ## Select what to return + output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) + return output end """ diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 4a8185629..51e1bbdbd 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -181,7 +181,7 @@ end postprocess::F = identity; verbose::Bool = true, api_key::String = API_KEY, - model::String = MODEL_EMBEDDING, return_all::Bool = false, dry_run::Bool = false, + model::String = MODEL_EMBEDDING, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), @@ -197,19 +197,13 @@ The `aiembed` function generates embeddings for the given input using a specifie - `verbose::Bool`: A flag indicating whether to print verbose information. Defaults to `true`. - `api_key::String`: The API key to use for the OpenAI API. Defaults to `API_KEY`. - `model::String`: The model to use for generating embeddings. Defaults to `MODEL_EMBEDDING`. -- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). -- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to `(retry_non_idempotent = true, retries = 5, readtimeout = 120)`. - `api_kwargs::NamedTuple`: Additional keyword arguments for the OpenAI API. Defaults to an empty `NamedTuple`. - `kwargs...`: Additional keyword arguments. ## Returns -If `return_all=false` (default): - `msg`: A `DataMessage` object containing the embeddings, status, token count, and elapsed time. Use `msg.content` to access the embeddings. -If `return_all=true`: -- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`DataMessage`). - # Example ```julia @@ -239,7 +233,6 @@ function aiembed(prompt_schema::AbstractOpenAISchema, postprocess::F = identity; verbose::Bool = true, api_key::String = API_KEY, model::String = MODEL_EMBEDDING, - return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), @@ -249,27 +242,20 @@ function aiembed(prompt_schema::AbstractOpenAISchema, ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) - if !dry_run - time = @elapsed r = create_embeddings(prompt_schema, api_key, - doc_or_docs, - model_id; - http_kwargs, - api_kwargs...) - msg = DataMessage(; - content = mapreduce(x -> postprocess(x[:embedding]), hcat, r.response[:data]), - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], 0), - elapsed = time) - ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) - else - msg = nothing - end - - ## Select what to return - output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) + time = @elapsed r = create_embeddings(prompt_schema, api_key, + doc_or_docs, + model_id; + http_kwargs, + api_kwargs...) + msg = DataMessage(; + content = mapreduce(x -> postprocess(x[:embedding]), hcat, r.response[:data]), + status = Int(r.status), + tokens = (r.response[:usage][:prompt_tokens], 0), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id, MODEL_COSTS) - return output + return msg end # Extend OpenAI create_embeddings to allow for testing function OpenAI.create_embeddings(schema::AbstractOpenAISchema, diff --git a/test/runtests.jl b/test/runtests.jl index 2ba589667..a3cd6fbe3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,7 +24,6 @@ let cb = AICode(; code = """ """) eval!(cb) @test cb.success == false - @info cb.error cb.output @test cb.error isa Test.FallbackTestSetException @test !isnothing(cb.expression) # parsed @test occursin("Test Failed", cb.stdout) # capture details of the test failure From b592c9e71747581556ca2a4b438a33f75b0df3bd Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:26:53 +0000 Subject: [PATCH 032/251] add tests for shared functionality --- src/PromptingTools.jl | 16 +++ src/llm_ollama_managed.jl | 38 ++++-- src/llm_openai.jl | 55 ++++++-- src/llm_shared.jl | 38 ++++-- src/messages.jl | 24 ++++ src/serialization.jl | 40 ++++++ src/templates.jl | 30 +---- src/utils.jl | 2 +- test/llm_ollama_managed.jl | 5 + test/llm_openai.jl | 14 -- test/llm_shared.jl | 270 +++++++++++++++++++++++++++++++++++++ test/messages.jl | 7 + test/runtests.jl | 2 + test/serialization.jl | 36 +++++ test/templates.jl | 21 +-- 15 files changed, 497 insertions(+), 101 deletions(-) create mode 100644 src/serialization.jl create mode 100644 test/llm_shared.jl create mode 100644 test/serialization.jl diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index d89528b52..2efd813e8 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -32,6 +32,19 @@ const MODEL_ALIASES = Dict("gpt3" => "gpt-3.5-turbo", # the below default is defined in llm_interace.jl ! # const PROMPT_SCHEMA = OpenAISchema() +"The following keywords are reserved for internal use in the `ai*` functions and cannot be used as placeholders in the Messages" +const RESERVED_KWARGS = [ + :http_kwargs, + :api_kwargs, + :conversation, + :return_all, + :dry_run, + :image_url, + :image_path, + :image_detail, + :model, +] + include("utils.jl") export aigenerate, aiembed, aiclassify, aiextract, aiscan @@ -49,6 +62,9 @@ include("templates.jl") const TEMPLATE_STORE = Dict{Symbol, Any}() const TEMPLATE_METADATA = Vector{AITemplateMetadata}() +# export save_conversation, load_conversation, save_template, load_template +include("serialization.jl") + ## Utilities to support structured extraction include("extraction.jl") diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 56ae60153..4aa9ae2da 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -5,32 +5,37 @@ """ render(schema::AbstractOllamaManagedSchema, messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], kwargs...) Builds a history of the conversation to provide the prompt to the API. All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. Note: Due to its "managed" nature, at most 2 messages can be provided (`system` and `prompt` inputs in the API). + +# Keyword Arguments +- `conversation`: Not allowed for this schema. Provided only for compatibility. """ function render(schema::AbstractOllamaManagedSchema, messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], kwargs...) ## @assert length(messages)<=2 "Managed schema only supports 2 messages (eg, a system and user)" @assert count(isusermessage, messages)<=1 "Managed schema only supports at most 1 User message" @assert count(issystemmessage, messages)<=1 "Managed schema only supports at most 1 System message" + @assert length(conversation)==0 "OllamaManagedSchema does not allow providing past conversation history. Use the `prompt` argument or `ChatMLSchema`." ## API expects: system=SystemMessage, prompt=UserMessage system, prompt = nothing, nothing + ## First pass: keep the message types but make the replacements provided in `kwargs` + messages_replaced = render(NoSchema(), messages; conversation, kwargs...) + # replace any handlebar variables in the messages - for msg in messages + for msg in messages_replaced if msg isa SystemMessage - replacements = ["{{$(key)}}" => value - for (key, value) in pairs(kwargs) if key in msg.variables] - system = replace(msg.content, replacements...) + system = msg.content elseif msg isa UserMessage - replacements = ["{{$(key)}}" => value - for (key, value) in pairs(kwargs) if key in msg.variables] - prompt = replace(msg.content, replacements...) + prompt = msg.content elseif msg isa UserMessageWithImages error("Managed schema does not support UserMessageWithImages. Please use OpenAISchema instead.") elseif msg isa AIMessage @@ -40,8 +45,6 @@ function render(schema::AbstractOllamaManagedSchema, end ## Sense check @assert !isnothing(prompt) "Managed schema requires at least 1 User message, ie, no `prompt` provided!" - ## Add default system prompt if not provided - isnothing(system) && (system = "Act as a helpful AI assistant") return (; system, prompt) end @@ -105,6 +108,7 @@ end aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) @@ -118,6 +122,7 @@ Generate an AI response based on a given prompt using the OpenAI API. - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. - `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). - `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). +- `conversation::AbstractVector{<:AbstractMessage}=[]`: Not allowed for this schema. Provided only for compatibility. - `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to empty `NamedTuple`. - `api_kwargs::NamedTuple`: Additional keyword arguments for the Ollama API. Defaults to an empty `NamedTuple`. - `kwargs`: Prompt variables to be used to fill the prompt/template @@ -179,17 +184,18 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ api_key::String = API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## global MODEL_ALIASES, MODEL_COSTS ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) - conversation = render(prompt_schema, prompt; kwargs...) + conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) if !dry_run - time = @elapsed resp = ollama_api(prompt_schema, conversation.prompt; - conversation.system, endpoint = "generate", model, http_kwargs, api_kwargs...) + time = @elapsed resp = ollama_api(prompt_schema, conv_rendered.prompt; + conv_rendered.system, endpoint = "generate", model, http_kwargs, api_kwargs...) msg = AIMessage(; content = resp.response[:response] |> strip, status = Int(resp.status), tokens = (resp.response[:prompt_eval_count], @@ -202,7 +208,13 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ end ## Select what to return - output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) + output = finalize_outputs(prompt, + conv_rendered, + msg; + conversation, + return_all, + dry_run, + kwargs...) return output end diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 51e1bbdbd..7a3aaa2d2 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -3,22 +3,25 @@ render(schema::AbstractOpenAISchema, messages::Vector{<:AbstractMessage}; image_detail::AbstractString = "auto", + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], kwargs...) Builds a history of the conversation to provide the prompt to the API. All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. -# Arguments +# Keyword Arguments - `image_detail`: Only for `UserMessageWithImages`. It represents the level of detail to include for images. Can be `"auto"`, `"high"`, or `"low"`. +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. """ function render(schema::AbstractOpenAISchema, messages::Vector{<:AbstractMessage}; image_detail::AbstractString = "auto", + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], kwargs...) ## @assert image_detail in ["auto", "high", "low"] "Image detail must be one of: auto, high, low" ## First pass: keep the message types but make the replacements provided in `kwargs` - messages_replaced = render(NoSchema(), messages; kwargs...) + messages_replaced = render(NoSchema(), messages; conversation, kwargs...) ## Second pass: convert to the OpenAI schema conversation = Dict{String, Any}[] @@ -74,6 +77,7 @@ Generate an AI response based on a given prompt using the OpenAI API. - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. - `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). - `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. - `http_kwargs`: A named tuple of HTTP keyword arguments. - `api_kwargs`: A named tuple of API keyword arguments. - `kwargs`: Prompt variables to be used to fill the prompt/template @@ -127,6 +131,7 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ verbose::Bool = true, api_key::String = API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), @@ -135,12 +140,12 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ global MODEL_ALIASES, MODEL_COSTS ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) - conversation = render(prompt_schema, prompt; kwargs...) + conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) if !dry_run time = @elapsed r = create_chat(prompt_schema, api_key, model_id, - conversation; + conv_rendered; http_kwargs, api_kwargs...) msg = AIMessage(; @@ -155,7 +160,13 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ msg = nothing end ## Select what to return - output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) + output = finalize_outputs(prompt, + conv_rendered, + msg; + conversation, + return_all, + dry_run, + kwargs...) return output end @@ -339,6 +350,7 @@ end verbose::Bool = true, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = (; retry_non_idempotent = true, retries = 5, @@ -362,6 +374,7 @@ It's effectively a light wrapper around `aigenerate` call, which requires additi - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. - `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). - `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. - `http_kwargs`: A named tuple of HTTP keyword arguments. - `api_kwargs`: A named tuple of API keyword arguments. - `kwargs`: Prompt variables to be used to fill the prompt/template @@ -372,7 +385,7 @@ If `return_all=false` (default): Use `msg.content` to access the extracted data. If `return_all=true`: -- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`DataMessage`). +- `conversation`: A vector of `AbstractMessage` objects representing the full conversation history, including the response from the AI model (`DataMessage`). See also: `function_call_signature`, `MaybeExtract`, `aigenerate` @@ -445,6 +458,7 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T api_key::String = API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), @@ -458,12 +472,12 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T api_kwargs = merge(api_kwargs, (; functions, function_call)) ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) - conversation = render(prompt_schema, prompt; kwargs...) + conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) if !dry_run time = @elapsed r = create_chat(prompt_schema, api_key, model_id, - conversation; + conv_rendered; http_kwargs, api_kwargs...) # "Safe" parsing of the response - it still fails if JSON is invalid @@ -486,7 +500,13 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T msg = nothing end ## Select what to return - output = finalize_outputs(prompt, conversation, msg; return_all, dry_run, kwargs...) + output = finalize_outputs(prompt, + conv_rendered, + msg; + conversation, + return_all, + dry_run, + kwargs...) return output end @@ -500,6 +520,7 @@ aiscan([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = (; retry_non_idempotent = true, retries = 5, @@ -526,6 +547,7 @@ It's effectively a light wrapper around `aigenerate` call, which uses additional - `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. - `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). - `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. - `http_kwargs`: A named tuple of HTTP keyword arguments. - `api_kwargs`: A named tuple of API keyword arguments. - `kwargs`: Prompt variables to be used to fill the prompt/template @@ -536,7 +558,7 @@ If `return_all=false` (default): Use `msg.content` to access the extracted string. If `return_all=true`: -- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`AIMessage`). +- `conversation`: A vector of `AbstractMessage` objects representing the full conversation history, including the response from the AI model (`AIMessage`). See also: `ai_str`, `aai_str`, `aigenerate`, `aiembed`, `aiclassify`, `aiextract`, `aitemplates` @@ -589,6 +611,7 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE api_key::String = API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, readtimeout = 120), api_kwargs::NamedTuple = (; max_tokens = 2500), @@ -600,12 +623,12 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE ## Vision-specific functionality msgs = attach_images_to_user_message(prompt; image_url, image_path, attach_to_latest) ## Build the conversation, pass what image detail is required (if provided) - conversation = render(prompt_schema, msgs; image_detail, kwargs...) + conv_rendered = render(prompt_schema, msgs; conversation, image_detail, kwargs...) if !dry_run ## Model call time = @elapsed r = create_chat(prompt_schema, api_key, model_id, - conversation; + conv_rendered; http_kwargs, api_kwargs...) msg = AIMessage(; @@ -621,7 +644,13 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE end ## Select what to return // input `msgs` to preserve the image attachments - output = finalize_outputs(msgs, conversation, msg; return_all, dry_run, kwargs...) + output = finalize_outputs(msgs, + conv_rendered, + msg; + conversation, + return_all, + dry_run, + kwargs...) return output end \ No newline at end of file diff --git a/src/llm_shared.jl b/src/llm_shared.jl index 6013bd546..7dcaa72df 100644 --- a/src/llm_shared.jl +++ b/src/llm_shared.jl @@ -2,22 +2,29 @@ """ render(schema::NoSchema, messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], replacement_kwargs...) Renders a conversation history from a vector of messages with all replacement variables specified in `replacement_kwargs`. It is the first pass of the prompt rendering system, and is used by all other schemas. +# Keyword Arguments +- `image_detail`: Only for `UserMessageWithImages`. It represents the level of detail to include for images. Can be `"auto"`, `"high"`, or `"low"`. +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. + # Notes - All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. - If a SystemMessage is missing, we inject a default one at the beginning of the conversation. +- Only one SystemMessage is allowed (ie, cannot mix two conversations different system prompts). """ function render(schema::NoSchema, messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], replacement_kwargs...) - ## - conversation = AbstractMessage[] - has_system_msg = false + ## copy the conversation to avoid mutating the original + conversation = copy(conversation) + count_system_msg = count(issystemmessage, conversation) # TODO: concat multiple system messages together (2nd pass) # replace any handlebar variables in the messages @@ -33,7 +40,7 @@ function render(schema::NoSchema, [(field, getfield(msg, field)) for field in fieldnames(typeof(msg))]..., content = replace(msg.content, replacements...)) if msg isa SystemMessage - has_system_msg = true + count_system_msg += 1 # move to the front pushfirst!(conversation, new_msg) else @@ -47,18 +54,21 @@ function render(schema::NoSchema, @warn "Unexpected message type: $(typeof(msg)). Skipping." end end + ## Multiple system prompts are not allowed + (count_system_msg > 1) && throw(ArgumentError("Only one system message is allowed.")) ## Add default system prompt if not provided - !has_system_msg && pushfirst!(conversation, + (count_system_msg == 0) && pushfirst!(conversation, SystemMessage("Act as a helpful AI assistant")) return conversation end """ - finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conversation::AbstractVector, + finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, msg::AbstractMessage; return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], kwargs...) Finalizes the outputs of the ai* functions by either returning the conversation history or the last message. @@ -67,21 +77,27 @@ Finalizes the outputs of the ai* functions by either returning the conversation - `return_all::Bool=false`: If true, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). - `dry_run::Bool=false`: If true, does not send the messages to the model, but only renders the prompt with the given schema and replacement variables. Useful for debugging when you want to check the specific schema rendering. +- `conversation::AbstractVector{<:AbstractMessage}=[]`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. - `kwargs...`: Variables to replace in the prompt template. """ -function finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conversation::AbstractVector, +function finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, msg::AbstractMessage; return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], kwargs...) if return_all if !dry_run # If not a dry_run, re-create the messages sent to the model before schema application - conversation = render(NoSchema(), prompt; kwargs...) - push!(conversation, msg) + # This is a duplication of work, as we already have the rendered messages in conv_rendered, + # but we prioritize the user's experience over performance here (ie, render(OpenAISchema,msgs) does everything under the hood) + output = render(NoSchema(), prompt; conversation, kwargs...) + push!(output, msg) + else + output = conv_rendered end - return conversation + return output else return msg end -end \ No newline at end of file +end diff --git a/src/messages.jl b/src/messages.jl index fc1640ec3..2ba5beb2b 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -25,17 +25,41 @@ Base.@kwdef struct SystemMessage{T <: AbstractString} <: AbstractChatMessage content::T variables::Vector{Symbol} = _extract_handlebar_variables(content) _type::Symbol = :systemmessage + SystemMessage{T}(c, v, t) where {T <: AbstractString} = new(c, v, t) +end +function SystemMessage(content::T, + variables::Vector{Symbol}, + type::Symbol) where {T <: AbstractString} + not_allowed_kwargs = intersect(variables, RESERVED_KWARGS) + @assert length(not_allowed_kwargs)==0 "Error: Some placeholders are invalid, as they are reserved for `ai*` functions. Change: $(join(not_allowed_kwargs,","))" + return SystemMessage{T}(content, variables, type) end Base.@kwdef struct UserMessage{T <: AbstractString} <: AbstractChatMessage content::T variables::Vector{Symbol} = _extract_handlebar_variables(content) _type::Symbol = :usermessage + UserMessage{T}(c, v, t) where {T <: AbstractString} = new(c, v, t) +end +function UserMessage(content::T, + variables::Vector{Symbol}, + type::Symbol) where {T <: AbstractString} + not_allowed_kwargs = intersect(variables, RESERVED_KWARGS) + @assert length(not_allowed_kwargs)==0 "Error: Some placeholders are invalid, as they are reserved for `ai*` functions. Change: $(join(not_allowed_kwargs,","))" + return UserMessage{T}(content, variables, type) end Base.@kwdef struct UserMessageWithImages{T <: AbstractString} <: AbstractChatMessage content::T image_url::Vector{<:AbstractString} # no default! fail when not provided variables::Vector{Symbol} = _extract_handlebar_variables(content) _type::Symbol = :usermessagewithimages + UserMessageWithImages{T}(c, i, v, t) where {T <: AbstractString} = new(c, i, v, t) +end +function UserMessageWithImages(content::T, image_url::Vector{<:AbstractString}, + variables::Vector{Symbol}, + type::Symbol) where {T <: AbstractString} + not_allowed_kwargs = intersect(variables, RESERVED_KWARGS) + @assert length(not_allowed_kwargs)==0 "Error: Some placeholders are invalid, as they are reserved for `ai*` functions. Change: $(join(not_allowed_kwargs,","))" + return UserMessageWithImages{T}(content, image_url, variables, type) end Base.@kwdef struct AIMessage{T <: Union{AbstractString, Nothing}} <: AbstractChatMessage content::T = nothing diff --git a/src/serialization.jl b/src/serialization.jl new file mode 100644 index 000000000..b4ca0879a --- /dev/null +++ b/src/serialization.jl @@ -0,0 +1,40 @@ +## Loading / Saving +"Saves provided messaging template (`messages`) to `io_or_file`. Automatically adds metadata based on provided keyword arguments." +function save_template(io_or_file::Union{IO, AbstractString}, + messages::AbstractVector{<:AbstractChatMessage}; + content::AbstractString = "Template Metadata", + description::AbstractString = "", + version::AbstractString = "1", + source::AbstractString = "") + + # create metadata + metadata_msg = MetadataMessage(; content, description, version, source) + + # save template to IO or file + JSON3.write(io_or_file, [metadata_msg, messages...]) +end +"Loads messaging template from `io_or_file` and returns tuple of template messages and metadata." +function load_template(io_or_file::Union{IO, AbstractString}) + messages = JSON3.read(io_or_file, Vector{AbstractChatMessage}) + template, metadata = AbstractChatMessage[], MetadataMessage[] + for i in eachindex(messages) + msg = messages[i] + if msg isa MetadataMessage + push!(metadata, msg) + else + push!(template, msg) + end + end + return template, metadata +end + +## Variants without metadata: +"Saves provided conversation (`messages`) to `io_or_file`. If you need to add some metadata, see `save_template`." +function save_conversation(io_or_file::Union{IO, AbstractString}, + messages::AbstractVector{<:AbstractChatMessage}) + JSON3.write(io_or_file, messages) +end +"Loads a conversation (`messages`) from `io_or_file`" +function load_conversation(io_or_file::Union{IO, AbstractString}) + messages = JSON3.read(io_or_file, Vector{AbstractChatMessage}) +end diff --git a/src/templates.jl b/src/templates.jl index fdb90d945..86ee9c3fc 100644 --- a/src/templates.jl +++ b/src/templates.jl @@ -113,35 +113,7 @@ function render(template::AITemplate; kwargs...) render(PROMPT_SCHEMA, template; kwargs...) end -## Loading / Saving -"Saves provided messaging template (`messages`) to `io_or_file`. Automatically adds metadata based on provided keyword arguments." -function save_template(io_or_file::Union{IO, AbstractString}, - messages::AbstractVector{<:AbstractChatMessage}; - content::AbstractString = "Template Metadata", - description::AbstractString = "", - version::AbstractString = "1", - source::AbstractString = "") - - # create metadata - metadata_msg = MetadataMessage(; content, description, version, source) - - # save template to IO or file - JSON3.write(io_or_file, [metadata_msg, messages...]) -end -"Loads messaging template from `io_or_file` and returns tuple of template messages and metadata." -function load_template(io_or_file::Union{IO, AbstractString}) - messages = JSON3.read(io_or_file, Vector{AbstractChatMessage}) - template, metadata = AbstractChatMessage[], MetadataMessage[] - for i in eachindex(messages) - msg = messages[i] - if msg isa MetadataMessage - push!(metadata, msg) - else - push!(template, msg) - end - end - return template, metadata -end +## Loading/saving -- see src/serialization.jl """ remove_templates!() diff --git a/src/utils.jl b/src/utils.jl index 53bacee6b..8d0f70b9d 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -134,4 +134,4 @@ _encode_local_image(::Nothing) = String[] # Used for image_url in aiscan to provided consistent output type _string_to_vector(s::AbstractString) = [s] -_string_to_vector(v::Vector{<:AbstractString}) = v +_string_to_vector(v::Vector{<:AbstractString}) = v \ No newline at end of file diff --git a/test/llm_ollama_managed.jl b/test/llm_ollama_managed.jl index d3611db3d..98e090961 100644 --- a/test/llm_ollama_managed.jl +++ b/test/llm_ollama_managed.jl @@ -44,6 +44,11 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage @test_throws AssertionError aigenerate(schema, [UserMessage("abc"), UserMessage("abc")]) + # error if conversation is provided + @test_throws AssertionError aigenerate(schema, + UserMessage("abc"); + conversation = [SystemMessage("abc")]) + # Double check templating messages = [ SystemMessage("Act as a helpful AI assistant"), diff --git a/test/llm_openai.jl b/test/llm_openai.jl index e01632862..05a11447a 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -106,20 +106,6 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage conversation = render(schema, messages) @test conversation == expected_output - # Given a schema and a vector of messages with multiple system messages, it should concatenate them together in the conversation dictionary. - messages = [ - SystemMessage("System message 1"), - SystemMessage("System message 2"), - UserMessage("User message"), - ] - conversation = render(schema, messages) - expected_output = [ - Dict("role" => "system", "content" => "System message 1\nSystem message 2"), - Dict("role" => "user", "content" => "User message"), - ] - # Broken: Does not concatenate system messages yet - @test_broken conversation == expected_output - # Test UserMessageWithImages messages = [ SystemMessage("System message 1"), diff --git a/test/llm_shared.jl b/test/llm_shared.jl new file mode 100644 index 000000000..f3ca4799f --- /dev/null +++ b/test/llm_shared.jl @@ -0,0 +1,270 @@ +using PromptingTools: render, NoSchema +using PromptingTools: AIMessage, SystemMessage, AbstractMessage, AbstractChatMessage +using PromptingTools: UserMessage, UserMessageWithImages +using PromptingTools: finalize_outputs + +@testset "render-NoSchema" begin + schema = NoSchema() + # Given a schema and a vector of messages with handlebar variables, it should replace the variables with the correct values in the conversation dictionary. + messages = [ + SystemMessage("Act as a helpful AI assistant"), + UserMessage("Hello, my name is {{name}}"), + ] + expected_output = [ + SystemMessage("Act as a helpful AI assistant"), + UserMessage(; + content = "Hello, my name is John", + variables = [:name], + _type = :usermessage), + ] + conversation = render(schema, + messages; + conversation = AbstractChatMessage[], + name = "John") + @test conversation == expected_output + + # AI message does NOT replace variables + messages = [ + SystemMessage("Act as a helpful AI assistant"), + AIMessage("Hello, my name is {{name}}"), + ] + expected_output = [ + SystemMessage("Act as a helpful AI assistant"), + AIMessage("Hello, my name is {{name}}"), + ] + conversation = render(schema, messages; name = "John") + # AIMessage does not replace handlebar variables + @test conversation == expected_output + + # Given a schema and a vector of messages with no system messages, it should add a default system prompt to the conversation dictionary. + messages = [ + UserMessage("User message"), + ] + conversation = render(schema, messages) + expected_output = [ + SystemMessage("Act as a helpful AI assistant"), + UserMessage("User message"), + ] + @test conversation == expected_output + + # Given a schema and a vector of messages and a conversation history, it should append the messages to the conversation + conversation = [ + SystemMessage("System message 1"), + UserMessage("Hello"), + AIMessage("Hi there"), + ] + messages = [ + UserMessage("How are you?"), + AIMessage("I'm doing well, thank you!"), + ] + expected_output = [ + SystemMessage("System message 1"), + UserMessage("Hello"), + AIMessage("Hi there"), + UserMessage("How are you?"), + AIMessage("I'm doing well, thank you!"), + ] + conversation = render(schema, messages; conversation) + @test conversation == expected_output + + # Replacement placeholders should be replaced only in the messages, not in the conversation history + conversation = [ + SystemMessage("System message 1"), + UserMessage("Hello {{name}}"), + AIMessage("Hi there"), + ] + messages = [ + UserMessage("How are you, {{name}}?"), + AIMessage("I'm doing well, thank you!"), + ] + expected_output = [ + SystemMessage("System message 1"), + UserMessage("Hello {{name}}"), + AIMessage("Hi there"), + UserMessage("How are you, John?", [:name], :usermessage), + AIMessage("I'm doing well, thank you!"), + ] + conversation = render(schema, messages; conversation, name = "John") + @test conversation == expected_output + + # Given a schema and a vector of messages with a system message, it should move the system message to the front of the conversation dictionary. + messages = [ + UserMessage("Hello"), + AIMessage("Hi there"), + SystemMessage("This is a system message"), + ] + expected_output = [ + SystemMessage("This is a system message"), + UserMessage("Hello"), + AIMessage("Hi there"), + ] + conversation = render(schema, messages) + @test conversation == expected_output + + # Given an empty vector of messages, it should return an empty conversation dictionary just with the system prompt + messages = AbstractMessage[] + expected_output = [ + SystemMessage("Act as a helpful AI assistant"), + ] + conversation = render(schema, messages) + @test conversation == expected_output + + # Given a schema and a vector of messages with a system message containing handlebar variables not present in kwargs, it should replace the variables with empty strings in the conversation dictionary. + messages = [ + SystemMessage("Hello, {{name}}!"), + UserMessage("How are you?"), + ] + expected_output = [ + SystemMessage("Hello, !", [:name], :systemmessage), + UserMessage("How are you?"), + ] + conversation = render(schema, messages) + # Broken because we do not remove any unused handlebar variables + @test_broken conversation == expected_output + + # Given a schema and a vector of messages with an unknown message type, it should skip the message and continue building the conversation dictionary. + messages = [ + SystemMessage("Act as a helpful AI assistant"), + UserMessage("Hello"), + DataMessage(; content = ones(3, 3)), + AIMessage("Hi there"), + ] + expected_output = [ + SystemMessage("Act as a helpful AI assistant"), + UserMessage("Hello"), + AIMessage("Hi there"), + ] + conversation = render(schema, messages) + @test conversation == expected_output + + # Given more than 1 system message, it should throw an error. + messages = [ + SystemMessage("System message 1"), + SystemMessage("System message 2"), + UserMessage("User message"), + ] + @test_throws ArgumentError render(schema, messages) + + # Given a schema and a vector of messages with multiple system messages, it should concatenate them together in the conversation dictionary. + messages = [ + SystemMessage("System message 1"), + SystemMessage("System message 2"), + UserMessage("User message"), + ] + # conversation = render(schema, messages) + # expected_output = [ + # SystemMessage("System message 1\nSystem message 2"), + # UserMessage("User message"), + # ] + # Broken: Does not concatenate system messages yet + # @test_broken conversation == expected_output + @test_throws ArgumentError render(schema, messages) + + # Test UserMessageWithImages + messages = [ + SystemMessage("System message 1"), + UserMessageWithImages("User message"; image_url = "https://example.com/image.png"), + ] + expected_output = [ + SystemMessage("System message 1"), + UserMessageWithImages("User message"; image_url = "https://example.com/image.png"), + ] + conversation = render(schema, messages) + @test conversation == expected_output +end + +# Write 5 unit tests for finalize_outputs for various combinations of inputs. Use @test calls +@testset "finalize_outputs" begin + # Given a vector of messages and a single message, it should return the last message. + messages = [ + SystemMessage("System message 1"), + UserMessage("User message"), + AIMessage("AI message"), + ] + msg = AIMessage("AI message 2") + expected_output = msg + output = finalize_outputs(messages, [], msg) + @test output == expected_output + + # Given a vector of messages and a single message, it should return the entire conversation history. + messages = [ + SystemMessage("System message 1"), + UserMessage("User message"), + AIMessage("AI message"), + ] + msg = AIMessage("AI message 2") + expected_output = [ + SystemMessage("System message 1"), + UserMessage("User message"), + AIMessage("AI message"), + msg, + ] + output = finalize_outputs(messages, [], msg; return_all = true) + @test output == expected_output + + # Given a vector of messages, conversation history and a single message, it should return the entire conversation history. + conversation = [ + SystemMessage("System message 1"), + UserMessage("User message"), + AIMessage("AI message"), + ] + messages = [ + AIMessage("AI message 2"), + ] + msg = AIMessage("AI message 3") + expected_output = [ + SystemMessage("System message 1"), + UserMessage("User message"), + AIMessage("AI message"), + AIMessage("AI message 2"), + msg, + ] + output = finalize_outputs(messages, [], msg; conversation, return_all = true) + @test output == expected_output + + # Test dry run + conversation = [ + SystemMessage("System message 1"), + UserMessage("User message"), + AIMessage("AI message"), + ] + messages = [ + AIMessage("AI message 2"), + ] + msg = AIMessage("AI message 3") + + output = finalize_outputs(messages, + [], + msg; + conversation, + return_all = true, + dry_run = true) + @test output == [] + + # Test that replacements are replicated properly in the messages but not in the conversation + conversation = [ + SystemMessage("System message 1"), + UserMessage("User message {{name}}"), + AIMessage("AI message"), + ] + messages = [ + UserMessage("User message {{name}}"), + AIMessage("AI message 2"), + ] + msg = AIMessage("AI message 3") + expected_output = [ + SystemMessage("System message 1"), + UserMessage("User message {{name}}"), + AIMessage("AI message"), + UserMessage("User message John", [:name], :usermessage), + AIMessage("AI message 2"), + msg, + ] + output = finalize_outputs(messages, + [], + msg; + name = "John", + conversation, + return_all = true) + @test output == expected_output +end \ No newline at end of file diff --git a/test/messages.jl b/test/messages.jl index a08c0266d..953d11a20 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -15,6 +15,13 @@ using PromptingTools: _encode_local_image, attach_images_to_user_message @test typeof(msg) <: T @test msg.content == content end + # Check the Reserved keywords + content = "{{model}}" + @test_throws AssertionError UserMessage(content) + @test_throws AssertionError UserMessage(; content) + @test_throws AssertionError SystemMessage(content) + @test_throws AssertionError SystemMessage(; content) + @test_throws AssertionError UserMessageWithImages(; content, image_url = ["a"]) end @testset "UserMessageWithImages" begin diff --git a/test/runtests.jl b/test/runtests.jl index a3cd6fbe3..cb70898e8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -12,8 +12,10 @@ end include("utils.jl") include("messages.jl") include("extraction.jl") + include("llm_shared.jl") include("llm_openai.jl") include("templates.jl") + include("serialization.jl") include("code_generation.jl") end diff --git a/test/serialization.jl b/test/serialization.jl new file mode 100644 index 000000000..c60ab2e79 --- /dev/null +++ b/test/serialization.jl @@ -0,0 +1,36 @@ +using PromptingTools: AIMessage, SystemMessage, UserMessage +using PromptingTools: save_conversation, load_conversation +using PromptingTools: save_template, load_template + +@testset "Serialization - Messages" begin + # Test save_conversation + messages = [ + SystemMessage("System message 1"), + UserMessage("User message"), + AIMessage("AI message"), + ] + tmp, _ = mktemp() + save_conversation(tmp, messages) + # Test load_conversation + loaded_messages = load_conversation(tmp) + @test loaded_messages == messages +end + +@testset "Serialization - Templates" begin + description = "Some description" + version = "1.1" + msgs = [ + SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + UserMessage("# Statement\n\n{{it}}"), + ] + tmp, _ = mktemp() + save_template(tmp, + msgs; + description, version) + template, metadata = load_template(tmp) + @test template == msgs + @test metadata[1].description == description + @test metadata[1].version == version + @test metadata[1].content == "Template Metadata" + @test metadata[1].source == "" +end \ No newline at end of file diff --git a/test/templates.jl b/test/templates.jl index 335fa0071..ac71a9807 100644 --- a/test/templates.jl +++ b/test/templates.jl @@ -1,27 +1,8 @@ using PromptingTools: AbstractChatMessage, SystemMessage, UserMessage, MetadataMessage using PromptingTools: render -using PromptingTools: save_template, load_template, load_templates!, aitemplates +using PromptingTools: load_templates!, aitemplates using PromptingTools: TestEchoOpenAISchema -@testset "Templates - save/load" begin - description = "Some description" - version = "1.1" - msgs = [ - SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), - UserMessage("# Statement\n\n{{it}}"), - ] - tmp, _ = mktemp() - save_template(tmp, - msgs; - description, version) - template, metadata = load_template(tmp) - @test template == msgs - @test metadata[1].description == description - @test metadata[1].version == version - @test metadata[1].content == "Template Metadata" - @test metadata[1].source == "" -end - @testset "Template rendering" begin template = AITemplate(:JudgeIsItTrue) expected_output = AbstractChatMessage[SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), From c03d8e3bcecaba4c1ea505ea11a6b9cc1f411641 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 6 Dec 2023 22:06:20 +0000 Subject: [PATCH 033/251] improve coverage --- CHANGELOG.md | 2 ++ src/llm_shared.jl | 4 ++-- test/code_generation.jl | 17 +++++++++++++++++ test/llm_openai.jl | 25 +++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7d768cfc..79605c300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Introduced a set of utilities for working with generate Julia code (Eg, extract code-fenced Julia code with `PromptingTools.extract_code_blocks` ) or simply apply `AICode` to the AI messages. `AICode` tries to extract, parse and eval Julia code, if it fails both stdout and errors are captured. It is useful for generating Julia code and, in the future, creating self-healing code agents +- Introduced ability to have multi-turn conversations. Set keyword argument `return_all=true` and `ai*` functions will return the whole conversation, not just the last message. To continue a previous conversation, you need to provide it to a keyword argument `conversation` +- Introduced schema `NoSchema` that does not change message format, it merely replaces the placeholders with user-provided variables. It serves as the first pass of the schema pipeline and allow more code reuse across schemas ### Fixed - Changed type of global `PROMPT_SCHEMA::AbstractPromptSchema` for an easier switch to local models as a default option diff --git a/src/llm_shared.jl b/src/llm_shared.jl index 7dcaa72df..d8cc86a02 100644 --- a/src/llm_shared.jl +++ b/src/llm_shared.jl @@ -65,7 +65,7 @@ end """ finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, - msg::AbstractMessage; + msg::Union{Nothing, AbstractMessage}; return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], @@ -81,7 +81,7 @@ Finalizes the outputs of the ai* functions by either returning the conversation - `kwargs...`: Variables to replace in the prompt template. """ function finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, - msg::AbstractMessage; + msg::Union{Nothing, AbstractMessage}; return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], diff --git a/test/code_generation.jl b/test/code_generation.jl index ff725cf44..e80ca1fbb 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -227,4 +227,21 @@ b=2 @test cb.stdout == "hello\nworld\n" @test cb.output.b == 2 end + + # Methods - copy + let msg = AIMessage(""" + ```julia + println(\"hello\") + ``` + Some text + ```julia + println(\"world\") + b=2 + ``` + """) + cb = AICode(msg) + cb_copy = Base.copy(cb) + @test cb_copy.code == cb.code + @test cb_copy !== cb + end end diff --git a/test/llm_openai.jl b/test/llm_openai.jl index 05a11447a..7ac921ad7 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -15,6 +15,13 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage ] conversation = render(schema, messages; name = "John") @test conversation == expected_output + # Test with dry_run=true on ai* functions + @test aigenerate(schema, messages; name = "John", dry_run = true) == nothing + @test aigenerate(schema, messages; name = "John", dry_run = true, return_all = true) == + expected_output + @test aiclassify(schema, messages; name = "John", dry_run = true) == nothing + @test aiclassify(schema, messages; name = "John", dry_run = true, return_all = true) == + expected_output # AI message does NOT replace variables messages = [ @@ -142,6 +149,24 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage "url" => "https://example.com/image2.png"), "type" => "image_url")])] @test conversation == expected_output + # Test with dry_run=true + messages_alt = [ + SystemMessage("System message 2"), + UserMessage("User message"), + ] + image_url = ["https://example.com/image1.png", + "https://example.com/image2.png"] + @test aiscan(schema, + copy(messages_alt); + image_detail = "low", image_url, + dry_run = true, + return_all = true) == expected_output + @test aiscan(schema, + copy(messages_alt); + image_detail = "low", + image_url, + dry_run = true) == + nothing end @testset "aigenerate-OpenAI" begin From 4bd280ffd74868e936456414c665a333c5d3f2e2 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:04:32 +0000 Subject: [PATCH 034/251] extend support to DataMessages --- src/messages.jl | 26 +++++++++++++++++++++----- src/serialization.jl | 4 ++-- test/messages.jl | 7 +++++++ test/serialization.jl | 5 ++--- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/messages.jl b/src/messages.jl index 2ba5beb2b..2cbceb712 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -82,6 +82,7 @@ function (MSG::Type{<:AbstractChatMessage})(prompt::AbstractString) end isusermessage(m::AbstractMessage) = m isa UserMessage issystemmessage(m::AbstractMessage) = m isa SystemMessage +isdatamessage(m::AbstractMessage) = m isa DataMessage # equality check for testing, only equal if all fields are equal and type is the same Base.var"=="(m1::AbstractMessage, m2::AbstractMessage) = false @@ -187,15 +188,30 @@ function render(schema::AbstractPromptSchema, msg::AbstractString; kwargs...) end ## Serialization via JSON3 +StructTypes.StructType(::Type{AbstractMessage}) = StructTypes.AbstractType() +StructTypes.subtypekey(::Type{AbstractMessage}) = :_type +function StructTypes.subtypes(::Type{AbstractMessage}) + (usermessage = UserMessage, + usermessagewithimages = UserMessageWithImages, + aimessage = AIMessage, + systemmessage = SystemMessage, + metadatamessage = MetadataMessage, + datamessage = DataMessage) +end + StructTypes.StructType(::Type{AbstractChatMessage}) = StructTypes.AbstractType() -StructTypes.StructType(::Type{MetadataMessage}) = StructTypes.Struct() -StructTypes.StructType(::Type{SystemMessage}) = StructTypes.Struct() -StructTypes.StructType(::Type{UserMessage}) = StructTypes.Struct() -StructTypes.StructType(::Type{AIMessage}) = StructTypes.Struct() StructTypes.subtypekey(::Type{AbstractChatMessage}) = :_type function StructTypes.subtypes(::Type{AbstractChatMessage}) (usermessage = UserMessage, + usermessagewithimages = UserMessageWithImages, aimessage = AIMessage, systemmessage = SystemMessage, metadatamessage = MetadataMessage) -end \ No newline at end of file +end + +StructTypes.StructType(::Type{MetadataMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{SystemMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{UserMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{UserMessageWithImages}) = StructTypes.Struct() +StructTypes.StructType(::Type{AIMessage}) = StructTypes.Struct() +StructTypes.StructType(::Type{DataMessage}) = StructTypes.Struct() diff --git a/src/serialization.jl b/src/serialization.jl index b4ca0879a..a21bcd6db 100644 --- a/src/serialization.jl +++ b/src/serialization.jl @@ -31,10 +31,10 @@ end ## Variants without metadata: "Saves provided conversation (`messages`) to `io_or_file`. If you need to add some metadata, see `save_template`." function save_conversation(io_or_file::Union{IO, AbstractString}, - messages::AbstractVector{<:AbstractChatMessage}) + messages::AbstractVector{<:AbstractMessage}) JSON3.write(io_or_file, messages) end "Loads a conversation (`messages`) from `io_or_file`" function load_conversation(io_or_file::Union{IO, AbstractString}) - messages = JSON3.read(io_or_file, Vector{AbstractChatMessage}) + messages = JSON3.read(io_or_file, Vector{AbstractMessage}) end diff --git a/test/messages.jl b/test/messages.jl index 953d11a20..8636d386f 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -1,6 +1,7 @@ using PromptingTools: AIMessage, SystemMessage, MetadataMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage using PromptingTools: _encode_local_image, attach_images_to_user_message +using PromptingTools: isusermessage, issystemmessage, isdatamessage @testset "Message constructors" begin # Creates an instance of MSG with the given content string. @@ -22,6 +23,12 @@ using PromptingTools: _encode_local_image, attach_images_to_user_message @test_throws AssertionError SystemMessage(content) @test_throws AssertionError SystemMessage(; content) @test_throws AssertionError UserMessageWithImages(; content, image_url = ["a"]) + + # Check methods + content = "Hello, world!" + @test UserMessage(content) |> isusermessage + @test SystemMessage(content) |> issystemmessage + @test DataMessage(; content) |> isdatamessage end @testset "UserMessageWithImages" begin diff --git a/test/serialization.jl b/test/serialization.jl index c60ab2e79..21f75e69f 100644 --- a/test/serialization.jl +++ b/test/serialization.jl @@ -4,11 +4,10 @@ using PromptingTools: save_template, load_template @testset "Serialization - Messages" begin # Test save_conversation - messages = [ - SystemMessage("System message 1"), + messages = AbstractMessage[SystemMessage("System message 1"), UserMessage("User message"), AIMessage("AI message"), - ] + DataMessage(; content = "Data message")] tmp, _ = mktemp() save_conversation(tmp, messages) # Test load_conversation From 06a518b37cdbb11515f32c1dd8401e85cff7e068 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:15:22 +0000 Subject: [PATCH 035/251] up --- src/messages.jl | 8 ++------ test/serialization.jl | 7 +++++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/messages.jl b/src/messages.jl index 2cbceb712..0a2400cc1 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -49,7 +49,7 @@ function UserMessage(content::T, end Base.@kwdef struct UserMessageWithImages{T <: AbstractString} <: AbstractChatMessage content::T - image_url::Vector{<:AbstractString} # no default! fail when not provided + image_url::Vector{String} # no default! fail when not provided variables::Vector{Symbol} = _extract_handlebar_variables(content) _type::Symbol = :usermessagewithimages UserMessageWithImages{T}(c, i, v, t) where {T <: AbstractString} = new(c, i, v, t) @@ -59,7 +59,7 @@ function UserMessageWithImages(content::T, image_url::Vector{<:AbstractString}, type::Symbol) where {T <: AbstractString} not_allowed_kwargs = intersect(variables, RESERVED_KWARGS) @assert length(not_allowed_kwargs)==0 "Error: Some placeholders are invalid, as they are reserved for `ai*` functions. Change: $(join(not_allowed_kwargs,","))" - return UserMessageWithImages{T}(content, image_url, variables, type) + return UserMessageWithImages{T}(content, string.(image_url), variables, type) end Base.@kwdef struct AIMessage{T <: Union{AbstractString, Nothing}} <: AbstractChatMessage content::T = nothing @@ -89,10 +89,6 @@ Base.var"=="(m1::AbstractMessage, m2::AbstractMessage) = false function Base.var"=="(m1::T, m2::T) where {T <: AbstractMessage} all([getproperty(m1, f) == getproperty(m2, f) for f in fieldnames(T)]) end -Base.length(t::AbstractMessage) = nfields(t) -function Base.iterate(t::AbstractMessage, iter = 1) - iter > nfields(t) ? nothing : (getfield(t, iter), iter + 1) -end ## Vision Models -- Constructor and Conversion "Construct `UserMessageWithImages` with 1 or more images. Images can be either URLs or local paths." diff --git a/test/serialization.jl b/test/serialization.jl index 21f75e69f..59066428f 100644 --- a/test/serialization.jl +++ b/test/serialization.jl @@ -1,4 +1,5 @@ -using PromptingTools: AIMessage, SystemMessage, UserMessage +using PromptingTools: AIMessage, + SystemMessage, UserMessage, UserMessageWithImages, AbstractMessage, DataMessage using PromptingTools: save_conversation, load_conversation using PromptingTools: save_template, load_template @@ -7,7 +8,9 @@ using PromptingTools: save_template, load_template messages = AbstractMessage[SystemMessage("System message 1"), UserMessage("User message"), AIMessage("AI message"), - DataMessage(; content = "Data message")] + UserMessageWithImages(; content = "a", image_url = String["b", "c"]), + DataMessage(; + content = "Data message")] tmp, _ = mktemp() save_conversation(tmp, messages) # Test load_conversation From a4c3d066f399f3a4e5da8aca6e5849693cc2776e Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:58:02 +0000 Subject: [PATCH 036/251] remove julia prompt from code blocks --- src/code_generation.jl | 36 +++++++++++++++++++++++++++++++-- test/code_generation.jl | 44 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index 3f2e81196..81f5f6cda 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -163,6 +163,38 @@ function detect_missing_packages(imports_required::AbstractVector{<:Symbol}) end end +"Checks if a given string has a Julia prompt (`julia> `) at the beginning of a line." +has_julia_prompt(s::T) where {T <: AbstractString} = occursin(r"^julia> "m, s) + +""" + remove_julia_prompt(s::T) where {T<:AbstractString} + +If it detects a julia prompt, it removes it and all lines that do not have it (except for those that belong to the code block). +""" +function remove_julia_prompt(s::T) where {T <: AbstractString} + if !has_julia_prompt(s) + return s + end + # Has julia prompt, so we need to parse it line by line + lines = split(s, '\n') + code_line = false + io = IOBuffer() + for line in lines + if startswith(line, r"^julia> ") + code_line = true + # remove the prompt + println(io, replace(line, "julia> " => "")) + elseif code_line && startswith(line, r"^ ") + # continuation of the code line + println(io, line) + else + code_line = false + end + end + # strip removes training whitespace and newlines + String(take!(io)) |> strip +end + """ extract_code_blocks(markdown_content::String) -> Vector{String} @@ -215,8 +247,8 @@ function extract_code_blocks(markdown_content::AbstractString) # Find all matches and extract the code matches = eachmatch(pattern, markdown_content) - # Extract and clean the code blocks - code_blocks = String[m.captures[1] for m in matches] + # Extract and clean the code blocks (remove the julia prompt) + code_blocks = String[remove_julia_prompt(m.captures[1]) for m in matches] return code_blocks end diff --git a/test/code_generation.jl b/test/code_generation.jl index e80ca1fbb..9db6ca8a4 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -1,6 +1,6 @@ using PromptingTools: extract_julia_imports using PromptingTools: detect_pkg_operation, detect_missing_packages, extract_function_name -using PromptingTools: extract_code_blocks, eval! +using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, eval! @testset "extract_imports tests" begin @test extract_julia_imports("using Test, LinearAlgebra") == @@ -29,6 +29,48 @@ end @test detect_pkg_operation("import Pkg;") == false end +@testset "has_julia_prompt" begin + @test has_julia_prompt("julia> a=1") + @test has_julia_prompt(""" +# something else first +julia> a=1 +""") + @test !has_julia_prompt(""" + # something + # new + a=1 + """) +end + +@testset "remove_julia_prompt" begin + @test remove_julia_prompt("julia> a=1") == "a=1" + @test remove_julia_prompt(""" +# something else first +julia> a=1 +# output +""") == "a=1" + @test remove_julia_prompt(""" + # something + # new + a=1 + """) == """ + # something + # new + a=1 + """ + @test remove_julia_prompt(""" +julia> a=\"\"\" + hey + there + \"\"\" +"hey\nthere\n" + """) == """ +a=\"\"\" + hey + there + \"\"\"""" +end + @testset "extract_code_blocks" begin # Single Julia Code Block markdown_content = """ From 8b7d7f31c4c63e53b0e4af98c33aa25e47e4b312 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 8 Dec 2023 20:38:23 +0000 Subject: [PATCH 037/251] Change AICode Safety Error to be a Parsing Error --- src/code_generation.jl | 26 +++++++++++++++++++------- test/code_generation.jl | 25 ++++++++++++++++++++----- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index 81f5f6cda..a3357910c 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -16,7 +16,7 @@ abstract type AbstractCodeBlock end """ - AICode(code::AbstractString; safe_eval::Bool=false, prefix::AbstractString="", suffix::AbstractString="") + AICode(code::AbstractString; auto_eval::Bool=true, safe_eval::Bool=false, prefix::AbstractString="", suffix::AbstractString="") A mutable structure representing a code block (received from the AI model) with automatic parsing, execution, and output/error capturing capabilities. @@ -44,6 +44,7 @@ See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` - `error::Union{Nothing, Exception}`: Any exception raised during the execution of the code block. # Keyword Arguments +- `auto_eval::Bool`: If set to `true`, the code block is automatically parsed and evaluated upon instantiation. Defaults to `true`. - `safe_eval::Bool`: If set to `true`, the code block checks for package operations (e.g., installing new packages) and missing imports, and then evaluates the code inside a bespoke scratch module. This is to ensure that the evaluation does not alter any user-defined variables or the global state. Defaults to `false`. - `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. Useful to add some additional code definition or necessary imports. Defaults to an empty string. @@ -100,11 +101,13 @@ eval(code.expression) end # Eager evaluation if instantiated with a string function (CB::Type{T})(md::AbstractString; + auto_eval::Bool = true, safe_eval::Bool = true, prefix::AbstractString = "", suffix::AbstractString = "") where {T <: AbstractCodeBlock} cb = CB(; code = md) - eval!(cb; safe_eval, prefix, suffix) + auto_eval && eval!(cb; safe_eval, prefix, suffix) + return cb end Base.isvalid(cb::AbstractCodeBlock) = cb.success == true function Base.copy(cb::AbstractCodeBlock) @@ -331,14 +334,23 @@ function eval!(cb::AbstractCodeBlock; prefix::AbstractString = "", suffix::AbstractString = "") (; code) = cb + # reset + cb.success = nothing + cb.error = nothing + cb.expression = nothing + cb.output = nothing code_extra = string(prefix, "\n", code, "\n", suffix) - ## Safety checks on `code` only + ## Safety checks on `code` only -- treat it as a parsing failure if safe_eval - detect_pkg_operation(code) && - throw(error("Error: Use of package manager (`Pkg.*`) detected! Please verify the safety of the code or disable the safety check (`safe_eval=false`)")) detected, missing_packages = detect_missing_packages(extract_julia_imports(code)) - detected && - throw(error("Error: Failed package import. Missing packages: $(join(string.(missing_packages),", ")). Please add them or disable the safety check (`safe_eval=false`)")) + if detect_pkg_operation(code) || detected + cb.success = false + detect_pkg_operation(code) && + (cb.error = ErrorException("Safety Error: Use of package manager (`Pkg.*`) detected! Please verify the safety of the code or disable the safety check (`safe_eval=false`)")) + detected && + (cb.error = ErrorException("Safety Error: Failed package import. Missing packages: $(join(string.(missing_packages),", ")). Please add them or disable the safety check (`safe_eval=false`)")) + return cb + end end ## Parse into an expression try diff --git a/test/code_generation.jl b/test/code_generation.jl index 9db6ca8a4..1f18e7d34 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -203,12 +203,16 @@ end @testset "eval! kwargs" begin ## Safe Eval == true mode # package that is not available - cb = AICode(; code = "using ExoticPackage123") - @test_throws Exception eval!(cb) - @test_throws "ExoticPackage123" eval!(cb) + cb = AICode(; code = "using ExoticPackage123") |> eval! + @test cb.error isa Exception + @test occursin("Safety Error", cb.error.msg) + @test occursin("ExoticPackage123", cb.error.msg) # Pkg operations - cb = AICode(; code = "Pkg.activate(\".\")") - @test_throws Exception eval!(cb) + cb = AICode(; code = "Pkg.activate(\".\")") |> eval! + @test cb.error isa Exception + @test occursin("Safety Error", cb.error.msg) + @test occursin("Use of package manager ", cb.error.msg) + # Evaluate inside a gensym'd module cb = AICode(; code = "a=1") |> eval! @test occursin("SafeCustomModule", string(cb.output)) @@ -251,6 +255,17 @@ end @test cb.output.a == 1 end + # Test auto-eval=false + let cb = AICode(""" + println("Hello") + a=1 + """; auto_eval = false) + # eval! is automatic + @test isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == nothing + end + # From AI Message let msg = AIMessage(""" ```julia From 5418d1a2d64d5bb93649c330600e5d7b36f34304 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:37:05 +0000 Subject: [PATCH 038/251] new code parsing --- src/code_generation.jl | 87 ++++++++++++++++++++++++++++++++++++----- test/code_generation.jl | 52 +++++++++++++++++++++--- 2 files changed, 125 insertions(+), 14 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index a3357910c..e42e856ea 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -198,6 +198,47 @@ function remove_julia_prompt(s::T) where {T <: AbstractString} String(take!(io)) |> strip end +""" + find_subsequence_positions(subseq, seq) -> Vector{Int} + +Find all positions of a subsequence `subseq` within a larger sequence `seq`. Used to lookup positions of code blocks in markdown. + +This function scans the sequence `seq` and identifies all starting positions where the subsequence `subseq` is found. Both `subseq` and `seq` should be vectors of integers, typically obtained using `codeunits` on strings. + +# Arguments +- `subseq`: A vector of integers representing the subsequence to search for. +- `seq`: A vector of integers representing the larger sequence in which to search. + +# Returns +- `Vector{Int}`: A vector of starting positions (1-based indices) where the subsequence is found in the sequence. + +# Examples +```julia +find_subsequence_positions(codeunits("ab"), codeunits("cababcab")) # Returns [2, 5] +``` +""" +function find_subsequence_positions(subseq, seq) + positions = Int[] + len_subseq = length(subseq) + len_seq = length(seq) + lim = len_seq - len_subseq + 1 + cur = 1 + while cur <= lim + match = true + @inbounds for i in 1:len_subseq + if seq[cur + i - 1] != subseq[i] + match = false + break + end + end + if match + push!(positions, cur) + end + cur += 1 + end + return positions +end + """ extract_code_blocks(markdown_content::String) -> Vector{String} @@ -243,17 +284,45 @@ extract_code_blocks(markdown_multiple) # Output: ["x = 5", "y = x + 2"] ``` """ -function extract_code_blocks(markdown_content::AbstractString) - # Define the pattern for Julia code blocks - pattern = r"```julia\n(.*?)\n```"s - - # Find all matches and extract the code - matches = eachmatch(pattern, markdown_content) +function extract_code_blocks(markdown_content::T) where {T <: AbstractString} + # Convert content and delimiters to codeunits + content_units = codeunits(markdown_content) + start_delim_units = codeunits("```julia") + end_delim_units = codeunits("```") + + # Find all starting and ending positions of code blocks + start_positions = find_subsequence_positions(start_delim_units, content_units) + end_positions = setdiff(find_subsequence_positions(end_delim_units, content_units), + start_positions) + unused_end_positions = trues(length(end_positions)) + + # Generate code block position pairs + block_positions = Tuple{Int, Int}[] + for start_pos in reverse(start_positions) + for (i, end_pos) in enumerate(end_positions) + if end_pos > start_pos && unused_end_positions[i] + push!(block_positions, (start_pos, end_pos)) + unused_end_positions[i] = false + break + end + end + end - # Extract and clean the code blocks (remove the julia prompt) - code_blocks = String[remove_julia_prompt(m.captures[1]) for m in matches] + # Filter out nested blocks (only if they have full overlap) + filtered_positions = filter(inner -> !any(outer -> (outer[1] < inner[1]) && + (inner[2] < outer[2]), + block_positions), + block_positions) + + # Extract code blocks + code_blocks = SubString{T}[] + for (start_pos, end_pos) in filtered_positions + code_block = markdown_content[(start_pos + length(start_delim_units)):(end_pos - 1)] + # Also remove the julia prompt + push!(code_blocks, remove_julia_prompt(strip(code_block))) + end - return code_blocks + return reverse(code_blocks) # Reverse to maintain original order end """ diff --git a/test/code_generation.jl b/test/code_generation.jl index 1f18e7d34..0939edfae 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -1,6 +1,7 @@ using PromptingTools: extract_julia_imports using PromptingTools: detect_pkg_operation, detect_missing_packages, extract_function_name using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, eval! +using PromptingTools: find_subsequence_positions @testset "extract_imports tests" begin @test extract_julia_imports("using Test, LinearAlgebra") == @@ -71,6 +72,24 @@ a=\"\"\" \"\"\"""" end +@testset "find_subsequence_positions" begin + # Test 1: Basic functionality + @test find_subsequence_positions(codeunits("ab"), codeunits("cababcab")) == [2, 4, 7] + + # Test 2: Subsequence not in sequence + @test find_subsequence_positions(codeunits("xyz"), codeunits("hello")) == [] + + # Test 3: Empty subsequence -- should return all positions+1 + @test find_subsequence_positions(codeunits(""), codeunits("hello")) == 1:6 + + # Test 4: Subsequence longer than sequence + @test find_subsequence_positions(codeunits("longsubsequence"), codeunits("short")) == [] + + # Test 5: Repeated characters + @test find_subsequence_positions(codeunits("ana"), codeunits("banana")) == [2, 4] + @test find_subsequence_positions(codeunits("a"), codeunits("a"^6)) == 1:6 +end + @testset "extract_code_blocks" begin # Single Julia Code Block markdown_content = """ @@ -79,7 +98,8 @@ end println("Hello, World!") ``` """ - @test extract_code_blocks(markdown_content) == ["println(\"Hello, World!\")"] + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"Hello, World!\")"] # Multiple Julia Code Blocks markdown_content = """ @@ -92,7 +112,7 @@ end ``` """ @test extract_code_blocks(markdown_content) == - ["println(\"First Block\")", "println(\"Second Block\")"] + SubString{String}["println(\"First Block\")", "println(\"Second Block\")"] # No Julia Code Blocks markdown_content = """ @@ -109,9 +129,10 @@ end println("This is Julia") ``` """ - @test extract_code_blocks(markdown_content) == ["println(\"This is Julia\")"] + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"This is Julia\")"] - # Nested Code Blocks" + # Nested Blocks (plain block outer) markdown_content = """ ``` ```julia @@ -119,7 +140,28 @@ end ``` ``` """ - @test extract_code_blocks(markdown_content) == ["println(\"Nested Block\")"] + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"Nested Block\")"] + + # Nested Julia code blocks + markdown_example = """ + ```julia + # Outer Julia code block + + # An example of a nested Julia code block in markdown + \"\"\" + ```julia + x = 5 + println(x) + ``` + \"\"\" + + y = 10 + println(y) + ``` + """ + @test extract_code_blocks(markdown_example) == + SubString{String}["# Outer Julia code block\n\n# An example of a nested Julia code block in markdown\n\"\"\"\n```julia\nx = 5\nprintln(x)\n```\n\"\"\"\n\ny = 10\nprintln(y)"] end @testset "extract_function_name" begin From 44bef07a232e45d8abe2bf3667e1cec3281401bd Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 9 Dec 2023 11:20:01 +0000 Subject: [PATCH 039/251] add interpolation fixer when needed --- src/code_generation.jl | 3 +++ test/code_generation.jl | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index e42e856ea..882fa24a9 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -198,6 +198,9 @@ function remove_julia_prompt(s::T) where {T <: AbstractString} String(take!(io)) |> strip end +# escape dollar sign only if not preceeded by backslash already, ie, unescaped -- use negative lookbehind +escape_interpolation(s::AbstractString) = replace(s, r"(? String(['\\', '$'])) + """ find_subsequence_positions(subseq, seq) -> Vector{Int} diff --git a/test/code_generation.jl b/test/code_generation.jl index 0939edfae..892f253e1 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -1,7 +1,7 @@ using PromptingTools: extract_julia_imports using PromptingTools: detect_pkg_operation, detect_missing_packages, extract_function_name using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, eval! -using PromptingTools: find_subsequence_positions +using PromptingTools: escape_interpolation, find_subsequence_positions @testset "extract_imports tests" begin @test extract_julia_imports("using Test, LinearAlgebra") == @@ -72,6 +72,11 @@ a=\"\"\" \"\"\"""" end +@testset "escape_interpolation" begin + @test escape_interpolation("aaa") == "aaa" + @test escape_interpolation("\$") == String(['\\', '$']) +end + @testset "find_subsequence_positions" begin # Test 1: Basic functionality @test find_subsequence_positions(codeunits("ab"), codeunits("cababcab")) == [2, 4, 7] From 6094539e736f34735f9a0b8de92f8a67f70af58e Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 9 Dec 2023 11:20:55 +0000 Subject: [PATCH 040/251] add comment --- src/code_generation.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/code_generation.jl b/src/code_generation.jl index 882fa24a9..b71324268 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -199,6 +199,7 @@ function remove_julia_prompt(s::T) where {T <: AbstractString} end # escape dollar sign only if not preceeded by backslash already, ie, unescaped -- use negative lookbehind +# Useful in cases where we have double nested interpolation, eg, string code -> has string literal -> function with interpolation inside it escape_interpolation(s::AbstractString) = replace(s, r"(? String(['\\', '$'])) """ From 81fefed4b5b9cb15e55cf49dfd7b6619bca514b6 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 9 Dec 2023 11:28:01 +0000 Subject: [PATCH 041/251] infer return type --- src/code_generation.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index b71324268..d9f95939d 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -319,7 +319,8 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString} block_positions) # Extract code blocks - code_blocks = SubString{T}[] + eltype_ = typeof(@view(markdown_content[begin:end])) + code_blocks = eltype_[] for (start_pos, end_pos) in filtered_positions code_block = markdown_content[(start_pos + length(start_delim_units)):(end_pos - 1)] # Also remove the julia prompt From a113b6a94bb3a08eb9141cadaad414532ec2bbea Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 9 Dec 2023 11:28:24 +0000 Subject: [PATCH 042/251] infer eltype --- src/code_generation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index d9f95939d..11840addd 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -320,7 +320,7 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString} # Extract code blocks eltype_ = typeof(@view(markdown_content[begin:end])) - code_blocks = eltype_[] + code_blocks = Vector{eltype_}() for (start_pos, end_pos) in filtered_positions code_block = markdown_content[(start_pos + length(start_delim_units)):(end_pos - 1)] # Also remove the julia prompt From b9a8d77464b283e23e6615b19764e9ba64bb04c4 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 10:10:38 +0000 Subject: [PATCH 043/251] add new Preferences --- CHANGELOG.md | 1 + Project.toml | 2 + src/PromptingTools.jl | 28 +--- src/code_generation.jl | 5 + src/llm_interface.jl | 35 ++++- src/llm_ollama_managed.jl | 12 +- src/llm_openai.jl | 16 +- src/user_preferences.jl | 298 ++++++++++++++++++++++++++++++++++++++ src/utils.jl | 12 +- test/code_generation.jl | 43 ++++++ test/runtests.jl | 1 + test/user_preferences.jl | 91 ++++++++++++ test/utils.jl | 4 +- 13 files changed, 498 insertions(+), 50 deletions(-) create mode 100644 src/user_preferences.jl create mode 100644 test/user_preferences.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 79605c300..cc5366a08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Introduced a set of utilities for working with generate Julia code (Eg, extract code-fenced Julia code with `PromptingTools.extract_code_blocks` ) or simply apply `AICode` to the AI messages. `AICode` tries to extract, parse and eval Julia code, if it fails both stdout and errors are captured. It is useful for generating Julia code and, in the future, creating self-healing code agents - Introduced ability to have multi-turn conversations. Set keyword argument `return_all=true` and `ai*` functions will return the whole conversation, not just the last message. To continue a previous conversation, you need to provide it to a keyword argument `conversation` - Introduced schema `NoSchema` that does not change message format, it merely replaces the placeholders with user-provided variables. It serves as the first pass of the schema pipeline and allow more code reuse across schemas +- Support for project-based and global user preferences with Preferences.jl. See `?PREFERENCES` docstring for more information. It allows you to persist your configuration and model aliases across sessions and projects (eg, if you would like to default to Ollama models instead of OpenAI's) ### Fixed - Changed type of global `PROMPT_SCHEMA::AbstractPromptSchema` for an easier switch to local models as a default option diff --git a/Project.toml b/Project.toml index 3a79267cc..f4ac50d9b 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +Preferences = "21216c6a-2e73-6563-6e65-726566657250" [compat] Aqua = "0.7" @@ -19,6 +20,7 @@ JSON3 = "1" Logging = "<0.0.1, 1" OpenAI = "0.8.7" PrecompileTools = "1" +Preferences = "1" Test = "<0.0.1, 1" julia = "1.9,1.10" diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 2efd813e8..701936fc7 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -6,31 +6,10 @@ using OpenAI using JSON3 using JSON3: StructTypes using HTTP +using Preferences using PrecompileTools -# GLOBALS -const MODEL_CHAT = "gpt-3.5-turbo" -const MODEL_EMBEDDING = "text-embedding-ada-002" -const API_KEY = get(ENV, "OPENAI_API_KEY", "") -# Note: Disable this warning by setting OPENAI_API_KEY to anything -isempty(API_KEY) && - @warn "OPENAI_API_KEY environment variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.API_KEY=`!" - -# Cost per 1K tokens as of 7th November 2023 -const MODEL_COSTS = Dict("gpt-3.5-turbo" => (0.0015, 0.002), - "gpt-3.5-turbo-1106" => (0.001, 0.002), - "gpt-4" => (0.03, 0.06), - "gpt-4-1106-preview" => (0.01, 0.03), - "gpt-4-vision-preview" => (0.01, 0.03), - "text-embedding-ada-002" => (0.001, 0.0)) -const MODEL_ALIASES = Dict("gpt3" => "gpt-3.5-turbo", - "gpt4" => "gpt-4", - "gpt4v" => "gpt-4-vision-preview", # 4v is for "4 vision" - "gpt4t" => "gpt-4-1106-preview", # 4t is for "4 turbo" - "gpt3t" => "gpt-3.5-turbo-1106", # 3t is for "3 turbo" - "ada" => "text-embedding-ada-002") -# the below default is defined in llm_interace.jl ! -# const PROMPT_SCHEMA = OpenAISchema() +# GLOBALS and Preferences are managed by Preferences.jl - see src/preferences.jl for details "The following keywords are reserved for internal use in the `ai*` functions and cannot be used as placeholders in the Messages" const RESERVED_KWARGS = [ @@ -51,6 +30,9 @@ export aigenerate, aiembed, aiclassify, aiextract, aiscan # export render # for debugging only include("llm_interface.jl") +# sets up the global registry of models and default model choices +include("user_preferences.jl") + ## Conversation history / Prompt elements export AIMessage # export UserMessage, UserMessageWithImages, SystemMessage, DataMessage # for debugging only diff --git a/src/code_generation.jl b/src/code_generation.jl index 11840addd..daeabe842 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -113,6 +113,11 @@ Base.isvalid(cb::AbstractCodeBlock) = cb.success == true function Base.copy(cb::AbstractCodeBlock) AICode(cb.code, cb.expression, cb.stdout, cb.output, cb.success, cb.error) end +# equality check for testing, only equal if all fields are equal and type is the same +Base.var"=="(m1::AbstractCodeBlock, m2::AbstractCodeBlock) = false +function Base.var"=="(c1::T, c2::T) where {T <: AbstractCodeBlock} + all([getproperty(c1, f) == getproperty(c2, f) for f in fieldnames(T)]) +end function Base.show(io::IO, cb::AICode) success_str = cb.success === nothing ? "N/A" : titlecase(string(cb.success)) expression_str = cb.expression === nothing ? "N/A" : "True" diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 3f841800d..484c6eb04 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -83,13 +83,32 @@ struct OllamaManagedSchema <: AbstractOllamaManagedSchema end inputs::Any = nothing end -## Dispatch into default schema -const PROMPT_SCHEMA::AbstractPromptSchema = OpenAISchema() - -aigenerate(prompt; kwargs...) = aigenerate(PROMPT_SCHEMA, prompt; kwargs...) +## Dispatch into a default schema (can be set by Preferences.jl) +const PROMPT_SCHEMA::AbstractPromptSchema = @load_preference("PROMPT_SCHEMA", + default=OpenAISchema()) + +function aigenerate(prompt; model, kwargs...) + global MODEL_REGISTRY + # first look up the model schema in the model registry; otherwise, use the default schema PROMPT_SCHEMA + schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema + aigenerate(schema, prompt; model, kwargs...) +end function aiembed(doc_or_docs, args...; kwargs...) - aiembed(PROMPT_SCHEMA, doc_or_docs, args...; kwargs...) + global MODEL_REGISTRY + schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema + aiembed(schema, doc_or_docs, args...; kwargs...) +end +function aiclassify(prompt; kwargs...) + global MODEL_REGISTRY + schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema + aiclassify(schema, prompt; kwargs...) +end +function aiextract(prompt; kwargs...) + global MODEL_REGISTRY + schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema + aiextract(schema, prompt; kwargs...) end -aiclassify(prompt; kwargs...) = aiclassify(PROMPT_SCHEMA, prompt; kwargs...) -aiextract(prompt; kwargs...) = aiextract(PROMPT_SCHEMA, prompt; kwargs...) -aiscan(prompt; kwargs...) = aiscan(PROMPT_SCHEMA, prompt; kwargs...) \ No newline at end of file +function aiscan(prompt; kwargs...) + schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema + aiscan(schema, prompt; kwargs...) +end \ No newline at end of file diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 4aa9ae2da..35add2ade 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -188,7 +188,7 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## - global MODEL_ALIASES, MODEL_COSTS + global MODEL_ALIASES ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) @@ -202,7 +202,7 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ resp.response[:eval_count]), elapsed = time) ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + verbose && @info _report_stats(msg, model_id) else msg = nothing end @@ -300,7 +300,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) where {F <: Function} ## - global MODEL_ALIASES, MODEL_COSTS + global MODEL_ALIASES ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) time = @elapsed resp = ollama_api(prompt_schema, doc; @@ -311,7 +311,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, tokens = (0, 0), # token counts are not provided for embeddings elapsed = time) ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + verbose && @info _report_stats(msg, model_id) return msg end @@ -322,7 +322,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, model::String = MODEL_EMBEDDING, kwargs...) where {F <: Function} ## - global MODEL_ALIASES, MODEL_COSTS + global MODEL_ALIASES ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) ## Send each document individually (no parallelism) @@ -341,7 +341,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, tokens = (0, 0),# not tracked for embeddings in Ollama elapsed = sum(x -> x.elapsed, messages)) ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + verbose && @info _report_stats(msg, model_id) return msg end diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 7a3aaa2d2..8a8fe70b9 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -137,7 +137,7 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## - global MODEL_ALIASES, MODEL_COSTS + global MODEL_ALIASES ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) @@ -155,7 +155,7 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ r.response[:usage][:completion_tokens]), elapsed = time) ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + verbose && @info _report_stats(msg, model_id) else msg = nothing end @@ -249,7 +249,7 @@ function aiembed(prompt_schema::AbstractOpenAISchema, readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), kwargs...) where {F <: Function} ## - global MODEL_ALIASES, MODEL_COSTS + global MODEL_ALIASES ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) @@ -264,7 +264,7 @@ function aiembed(prompt_schema::AbstractOpenAISchema, tokens = (r.response[:usage][:prompt_tokens], 0), elapsed = time) ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + verbose && @info _report_stats(msg, model_id) return msg end @@ -464,7 +464,7 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## - global MODEL_ALIASES, MODEL_COSTS + global MODEL_ALIASES ## Function calling specifics functions = [function_call_signature(return_type)] function_call = Dict(:name => only(functions)["name"]) @@ -495,7 +495,7 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T r.response[:usage][:completion_tokens]), elapsed = time) ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + verbose && @info _report_stats(msg, model_id) else msg = nothing end @@ -617,7 +617,7 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE readtimeout = 120), api_kwargs::NamedTuple = (; max_tokens = 2500), kwargs...) ## - global MODEL_ALIASES, MODEL_COSTS + global MODEL_ALIASES ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) ## Vision-specific functionality @@ -638,7 +638,7 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE r.response[:usage][:completion_tokens]), elapsed = time) ## Reporting - verbose && @info _report_stats(msg, model_id, MODEL_COSTS) + verbose && @info _report_stats(msg, model_id) else msg = nothing end diff --git a/src/user_preferences.jl b/src/user_preferences.jl new file mode 100644 index 000000000..a2b6da8f7 --- /dev/null +++ b/src/user_preferences.jl @@ -0,0 +1,298 @@ +# Defines the important Globals, model registry and user preferences +# See below (eg, MODEL_REGISTRY, ModelSpec) + +""" + PREFERENCES + +You can set preferences for PromptingTools by setting environment variables (for `OPENAI_API_KEY` only) + or by using the `@set_preferences!` macro (see `Preferences.jl`). + It will create a `LocalPreferences.toml` file in your current directory and will reload your prefences from there. + +If you can always check if a preference has been set by `@has_preference("")` or directly get its value by `@load_preference("")`. + +# Available Preferences (for `@set_preferences!`) +- `OPENAI_API_KEY`: The API key for the OpenAI API. See [OpenAI's documentation](https://platform.openai.com/docs/quickstart?context=python) for more information. +- `MODEL_CHAT`: The default model to use for aigenerate and most ai* calls. See `MODEL_REGISTRY` for a list of available models or define your own. +- `MODEL_EMBEDDING`: The default model to use for aiembed (embedding documents). See `MODEL_REGISTRY` for a list of available models or define your own. +- `MODEL_ALIASES`: A dictionary of model aliases (`alias => full_model_name`). Aliases are used to refer to models by their aliases instead of their full names to make it more convenient to use them. + See `MODEL_ALIASES` for more information. + +At the moment it is not possible to persist changes to `MODEL_REGISTRY` across sessions. +Define your `register_model!()` calls in your `startup.jl` file to make them available across sessions or put them at the top of your script. + +# Available ENV Variables +- `OPENAI_API_KEY`: The API key for the OpenAI API. + +Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. + +WARNING: Never ever sync your `LocalPreferences.toml` file! It contains your API key and other sensitive information!!! +""" +const PREFERENCES = nothing + +## Load up GLOBALS +const MODEL_CHAT::String = @load_preference("MODEL_CHAT", default="gpt-3.5-turbo") +const MODEL_EMBEDDING::String = @load_preference("MODEL_CHAT", + default="text-embedding-ada-002") +# the prompt schema default is defined in llm_interace.jl ! +# const PROMPT_SCHEMA = OpenAISchema() + +# First, load from preferences, then from environment variables +const API_KEY::String = @load_preference("OPENAI_API_KEY", + default=get(ENV, "OPENAI_API_KEY", "")) +# Note: Disable this warning by setting OPENAI_API_KEY to anything +isempty(API_KEY) && + @warn "OPENAI_API_KEY environment variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.API_KEY=`!" + +## Model registry +# A dictionary of model names and their specs (ie, name, costs per token, etc.) +# Model specs are saved in ModelSpec struct (see below) + +### ModelSpec Functionality +""" + ModelSpec + +A struct that contains information about a model, such as its name, schema, cost per token, etc. + +# Fields +- `name::String`: The name of the model. This is the name that will be used to refer to the model in the `ai*` functions. +- `schema::AbstractPromptSchema`: The schema of the model. This is the schema that will be used to generate prompts for the model, eg, `:OpenAISchema`. +- `cost_of_token_prompt::Float64`: The cost of 1 token in the prompt for this model. This is used to calculate the cost of a prompt. + Note: It is often provided online as cost per 1000 tokens, so make sure to convert it correctly! +- `cost_of_token_generation::Float64`: The cost of 1 token generated by this model. This is used to calculate the cost of a generation. + Note: It is often provided online as cost per 1000 tokens, so make sure to convert it correctly! +- `description::String`: A description of the model. This is used to provide more information about the model when it is queried. + +# Example +```julia +spec = ModelSpec("gpt-3.5-turbo", + OpenAISchema(), + 0.0015, + 0.002, + "GPT-3.5 Turbo is a 175B parameter model and a common default on the OpenAI API.") + +# register it +PromptingTools.register_model!(spec) +``` + +But you can also register any model directly via keyword arguments: +```julia +PromptingTools.register_model!( + name = "gpt-3.5-turbo", + schema = OpenAISchema(), + cost_of_token_prompt = 0.0015, + cost_of_token_generation = 0.002, + description = "GPT-3.5 Turbo is a 175B parameter model and a common default on the OpenAI API.") +``` +""" +@kwdef mutable struct ModelSpec + name::String + schema::Union{AbstractPromptSchema, Nothing} = nothing + cost_of_token_prompt::Float64 = 0.0 + cost_of_token_generation::Float64 = 0.0 + description::String = "" +end +function Base.show(io::IO, m::ModelSpec) + dump(IOContext(io, :limit => true), m, maxdepth = 1) +end + +""" + register_model!(registry = MODEL_REGISTRY; + name::String, + schema::Union{AbstractPromptSchema, Nothing} = nothing, + cost_of_token_prompt::Float64 = 0.0, + cost_of_token_generation::Float64 = 0.0, + description::String = "") + +Register a new AI model with `name` and its associated `schema`. + +Registering a model helps with calculating the costs and automatically selecting the right prompt schema. + +# Arguments +- `name`: The name of the model. This is the name that will be used to refer to the model in the `ai*` functions. +- `schema`: The schema of the model. This is the schema that will be used to generate prompts for the model, eg, `OpenAISchema()`. +- `cost_of_token_prompt`: The cost of a token in the prompt for this model. This is used to calculate the cost of a prompt. + Note: It is often provided online as cost per 1000 tokens, so make sure to convert it correctly! +- `cost_of_token_generation`: The cost of a token generated by this model. This is used to calculate the cost of a generation. + Note: It is often provided online as cost per 1000 tokens, so make sure to convert it correctly! +- `description`: A description of the model. This is used to provide more information about the model when it is queried. +""" +function register_model!(registry = MODEL_REGISTRY; + name::String, + schema::Union{AbstractPromptSchema, Nothing} = nothing, + cost_of_token_prompt::Float64 = 0.0, + cost_of_token_generation::Float64 = 0.0, + description::String = "") + spec = ModelSpec(name, + schema, + cost_of_token_prompt, + cost_of_token_generation, + description) + register_model!(spec; registry) +end +function register_model!(spec::ModelSpec; registry = MODEL_REGISTRY) + haskey(registry, spec.name) && + @warn "Model `$(spec.name)` already registered! It will be overwritten." + registry[spec.name] = spec +end + +## Model Registry Data + +### Model Aliases + +# global reference MODEL_ALIASES is defined below +aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", + "gpt4" => "gpt-4", + "gpt4v" => "gpt-4-vision-preview", # 4v is for "4 vision" + "gpt4t" => "gpt-4-1106-preview", # 4t is for "4 turbo" + "gpt3t" => "gpt-3.5-turbo-1106", # 3t is for "3 turbo" + "ada" => "text-embedding-ada-002", + "yi34c" => "yi:34b-chat", + "oh25" => "openhermes2.5-mistral", + "starling" => "starling-lm"), + ## Load aliases from preferences as well + @load_preference("MODEL_ALIASES", default=Dict{String, String}())) + +registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", + OpenAISchema(), + 1.5e-6, + 2e-6, + "GPT-3.5 Turbo is a 175B parameter model and a common default on the OpenAI API."), + "gpt-3.5-turbo-1106" => ModelSpec("gpt-3.5-turbo-1106", + OpenAISchema(), + 1e-6, + 2e-6, + "GPT-3.5 Turbo is the latest version of GPT3.5 and the cheapest to use."), + "gpt-4" => ModelSpec("gpt-4", + OpenAISchema(), + 3e-5, + 6e-5, + "GPT-4 is a 1.75T parameter model and the largest model available on the OpenAI API."), + "gpt-4-1106-preview" => ModelSpec("gpt-4-1106-preview", + OpenAISchema(), + 1e-5, + 3e-5, + "GPT-4 Turbo is the latest version of GPT4 that is much faster and the cheapest to use."), + "gpt-4-vision-preview" => ModelSpec("gpt-4-vision-preview", + OpenAISchema(), + 1e-5, + 3e-5, + "GPT-4 Vision is similar to GPT-4 but it adds visual capabilities."), + "text-embedding-ada-002" => ModelSpec("text-embedding-ada-002", + OpenAISchema(), + 1e-7, + 0.0, + "Text Embedding Ada is a 1.75T parameter model and the largest model available on the OpenAI API."), + "llama2" => ModelSpec("llama2", + OllamaManagedSchema(), + 0.0, + 0.0, + "LLAMA2 is a 7B parameter model from Meta."), + "openhermes2.5-mistral" => ModelSpec("openhermes2.5-mistral", + OllamaManagedSchema(), + 0.0, + 0.0, + "OpenHermes 2.5 Mistral is a 7B parameter model finetuned by X on top of base model from Mistral AI."), + "starling-lm" => ModelSpec("starling-lm", + OllamaManagedSchema(), + 0.0, + 0.0, + "Starling LM is a 7B parameter model finetuned by X on top of base model from Starling AI."), + "yi:34b-chat" => ModelSpec("yi:34b-chat", + OllamaManagedSchema(), + 0.0, + 0.0, + "Yi is a 34B parameter model finetuned by X on top of base model from Starling AI.")) + +### Model Registry Structure +@kwdef mutable struct ModelRegistry + registry::Dict{String, ModelSpec} + aliases::Dict{String, String} +end +function Base.show(io::IO, registry::ModelRegistry) + num_models = length(registry.registry) + num_aliases = length(registry.aliases) + print(io, + "ModelRegistry with $num_models models and $num_aliases aliases. See `?MODEL_REGISTRY` for more information.") +end + +""" + MODEL_REGISTRY + +A store of available model names and their specs (ie, name, costs per token, etc.) + +# Accessing the registry + +You can use both the alias name or the full name to access the model spec: +``` +PromptingTools.MODEL_REGISTRY["gpt-3.5-turbo"] +``` + +# Registering a new model +```julia +register_model!( + name = "gpt-3.5-turbo", + schema = :OpenAISchema, + cost_of_token_prompt = 0.0015, + cost_of_token_generation = 0.002, + description = "GPT-3.5 Turbo is a 175B parameter model and a common default on the OpenAI API.") +``` + +# Registering a model alias +```julia +PromptingTools.MODEL_ALIASES["gpt3"] = "gpt-3.5-turbo" +``` + +""" +const MODEL_REGISTRY = ModelRegistry(registry, aliases) + +# We overload the getindex function to allow for lookup via model aliases +function Base.getindex(registry::ModelRegistry, key::String) + # Check if the key exists in the registry + if haskey(registry.registry, key) + return registry.registry[key] + end + + # If the key is not in the registry, check if it's an alias + aliased_key = get(registry.aliases, key, nothing) + if !isnothing(aliased_key) && haskey(registry.registry, aliased_key) + return registry.registry[aliased_key] + end + + # Handle the case where the key is neither in the registry nor an alias + throw(KeyError("Model with key '$key' not found in PromptingTools.MODEL_REGISTRY.")) +end +function Base.setindex!(registry::ModelRegistry, value::ModelSpec, key::String) + registry.registry[key] = value +end +function Base.haskey(registry::ModelRegistry, key::String) + haskey(registry.registry, key) || haskey(registry.aliases, key) +end +function Base.get(registry::ModelRegistry, key::String, default) + if haskey(registry, key) + return registry[key] + else + return default + end +end +function Base.delete!(registry::ModelRegistry, key::String) + haskey(registry.registry, key) && delete!(registry.registry, key) + haskey(registry.aliases, key) && delete!(registry.aliases, key) + return registry +end + +""" + MODEL_ALIASES + +A dictionary of model aliases. Aliases are used to refer to models by their aliases instead of their full names to make it more convenient to use them. + +# Accessing the aliases +``` +PromptingTools.MODEL_ALIASES["gpt3"] +``` + +# Register a new model alias +```julia +PromptingTools.MODEL_ALIASES["gpt3"] = "gpt-3.5-turbo" +``` +""" +const MODEL_ALIASES = MODEL_REGISTRY.aliases diff --git a/src/utils.jl b/src/utils.jl index 8d0f70b9d..168d772cb 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -110,9 +110,15 @@ function _extract_handlebar_variables(vect::Vector{Dict{String, <:AbstractString end # helper to produce summary message of how many tokens were used and for how much -function _report_stats(msg, model::String, model_costs::AbstractDict = Dict()) - token_prices = get(model_costs, model, (0.0, 0.0)) - cost = sum(msg.tokens ./ 1000 .* token_prices) +function _report_stats(msg, + model::String, + cost_of_token_prompt::Number = get(MODEL_REGISTRY, + model, + (; cost_of_token_prompt = 0.0)).cost_of_token_prompt, + cost_of_token_generation::Number = get(MODEL_REGISTRY, model, + (; cost_of_token_generation = 0.0)).cost_of_token_generation) + cost = (msg.tokens[1] * cost_of_token_prompt + + msg.tokens[2] * cost_of_token_generation) cost_str = iszero(cost) ? "" : " @ Cost: \$$(round(cost; digits=4))" return "Tokens: $(sum(msg.tokens))$(cost_str) in $(round(msg.elapsed;digits=1)) seconds" diff --git a/test/code_generation.jl b/test/code_generation.jl index 892f253e1..38bfb87fc 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -349,3 +349,46 @@ b=2 @test cb_copy !== cb end end + +@testset "AICode-methods" begin + ## SHOW + # Test with All Fields as `nothing` + code_block = AICode(""; auto_eval = false) + buffer = IOBuffer() + show(buffer, code_block) + output = String(take!(buffer)) + @test output == + "AICode(Success: N/A, Parsed: N/A, Evaluated: N/A, Error Caught: N/A, StdOut: N/A, Code: 1 Lines)" + + # Test with All Fields Set + code_block = AICode("println(\"Hello World\")") + buffer = IOBuffer() + show(buffer, code_block) + output = String(take!(buffer)) + @test output == + "AICode(Success: True, Parsed: True, Evaluated: True, Error Caught: N/A, StdOut: True, Code: 1 Lines)" + + # Test with error + code_block = AICode("error(\"Test Error\"))\nprint(\"\")") + buffer = IOBuffer() + show(buffer, code_block) + output = String(take!(buffer)) + @test output == + "AICode(Success: False, Parsed: True, Evaluated: N/A, Error Caught: True, StdOut: True, Code: 2 Lines)" + + ## EQUALITY + # Test Comparing Two Identical Code Blocks -- if it's not safe_eval, it's not equal (gensym'd Safe module for output!) + code1 = AICode("print(\"Hello\")"; safe_eval = false) + code2 = AICode("print(\"Hello\")"; safe_eval = false) + @test code1 == code2 + + # Test Comparing Two Different Code Blocks + code1 = AICode("print(\"Hello\")") + code2 = AICode("print(\"World\")") + @test code1 != code2 + + # Different gensym! + code1 = AICode("print(\"Hello\")"; safe_eval = true) + code2 = AICode("print(\"Hello\")"; safe_eval = false) + @test code1 != code2 +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index cb70898e8..aadb01abe 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -12,6 +12,7 @@ end include("utils.jl") include("messages.jl") include("extraction.jl") + include("user_preferences.jl") include("llm_shared.jl") include("llm_openai.jl") include("templates.jl") diff --git a/test/user_preferences.jl b/test/user_preferences.jl new file mode 100644 index 000000000..0c570ca96 --- /dev/null +++ b/test/user_preferences.jl @@ -0,0 +1,91 @@ +using PromptingTools: ModelSpec, + register_model!, MODEL_REGISTRY, MODEL_ALIASES, ModelRegistry +using PromptingTools: OpenAISchema, OllamaManagedSchema + +@testset "ModelSpec" begin + # Test for Correct Initialization + spec = ModelSpec("gpt-3.5-turbo", OpenAISchema(), 0.0015, 0.002, "Description") + @test spec.name == "gpt-3.5-turbo" + @test spec.schema == OpenAISchema() + @test spec.cost_of_token_prompt ≈ 0.0015 + @test spec.cost_of_token_generation ≈ 0.002 + @test spec.description == "Description" + + # Test for Default Values + spec = ModelSpec(; name = "gpt-3") + @test spec.schema === nothing + @test spec.cost_of_token_prompt ≈ 0.0 + @test spec.cost_of_token_generation ≈ 0.0 + @test spec.description == "" + + # Test for Type Assertions + @test_throws MethodError ModelSpec(123, OpenAISchema(), 0.0015, 0.002, "Description") + @test_throws MethodError ModelSpec("gpt-3", :OpenAISchema, 0.0015, 0.002, "Description") + @test_throws MethodError ModelSpec("gpt-3", + "InvalidSymbol", + 0.0015, + 0.002, + "Description") + # Test for Correct Output Format + spec = ModelSpec("gpt-3.5-turbo", OpenAISchema(), 0.0015, 0.002, "Description") + buffer = IOBuffer() + show(buffer, spec) + output = String(take!(buffer)) + expected_output = "ModelSpec\n name: String \"gpt-3.5-turbo\"\n schema: OpenAISchema OpenAISchema()\n cost_of_token_prompt: Float64 0.0015\n cost_of_token_generation: Float64 0.002\n description: String \"Description\"\n" + @test output == expected_output +end + +@testset "ModelRegistry" begin + # Assuming MODEL_REGISTRY is a Dict accessible for testing + # Test for Normal Registration + register_model!(; name = "gpt-5", + schema = OllamaManagedSchema(), + cost_of_token_prompt = 0.1, + cost_of_token_generation = 0.1, + description = "Test model") + @test MODEL_REGISTRY["gpt-5"].schema == OllamaManagedSchema() + @test MODEL_REGISTRY["gpt-5"].description == "Test model" + + # Manual registry + new_spec = ModelSpec("gpt-new", OpenAISchema(), 0.001, 0.002, "New model description") + MODEL_REGISTRY["gpt-new"] = new_spec + @test MODEL_REGISTRY["gpt-new"].name == "gpt-new" + + # Test for Default Argument Usage + register_model!(name = "gpt-5-mini") + @test MODEL_REGISTRY["gpt-5-mini"].schema === nothing + @test MODEL_REGISTRY["gpt-5-mini"].description == "" + + # Test for Model Overwriting Warning + @test_logs (:warn, "Model `gpt-5` already registered! It will be overwritten.") register_model!(name = "gpt-5") + + # Test for Registry Update + original_count = length(MODEL_REGISTRY.registry) + delete!(MODEL_REGISTRY, "new-model") + register_model!(name = "new-model") + @test length(MODEL_REGISTRY.registry) == original_count + 1 + + # Test for Correct Alias Access + @test MODEL_ALIASES["gpt3"] == "gpt-3.5-turbo" + + # Test for Adding New Alias + MODEL_ALIASES["new-alias"] = "gpt-3.5-turbo" + @test MODEL_ALIASES["new-alias"] == "gpt-3.5-turbo" + @test MODEL_REGISTRY["new-alias"].name == "gpt-3.5-turbo" + + # Test for Correct Model Access by Full Name + @test MODEL_REGISTRY["gpt-3.5-turbo"].name == "gpt-3.5-turbo" + + # Test for Non-Existent Alias + @test_throws KeyError MODEL_ALIASES["nonexistent"] + @test_throws KeyError MODEL_REGISTRY["nonexistent"] + @test get(MODEL_REGISTRY, "nonexistent", "xyz") == "xyz" + + # Show method + buffer = IOBuffer() + show(buffer, MODEL_REGISTRY) + output = String(take!(buffer)) + + expected_output = "ModelRegistry with $(length(MODEL_REGISTRY.registry)) models and $(length(MODEL_REGISTRY.aliases)) aliases. See `?MODEL_REGISTRY` for more information." + @test output == expected_output +end diff --git a/test/utils.jl b/test/utils.jl index 10f268b62..1b726924a 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -77,11 +77,11 @@ end # Returns a string with a cost expected_output = "Tokens: 6 @ Cost: \$0.007 in 5.0 seconds" - @test _report_stats(msg, model, Dict(model => (2, 1))) == expected_output + @test _report_stats(msg, model, 2e-3, 1e-3) == expected_output # Returns a string without cost when it's zero expected_output = "Tokens: 6 in 5.0 seconds" - @test _report_stats(msg, model, Dict(model => (0, 0))) == expected_output + @test _report_stats(msg, model, 0, 0) == expected_output end @testset "_string_to_vector" begin From 65f82d53662f9a10cd107c424ae0f281983e2383 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 11:58:30 +0000 Subject: [PATCH 044/251] update docstrings --- CHANGELOG.md | 4 + README.md | 6 +- docs/generate_examples.jl | 1 + docs/src/examples/working_with_aitemplates.md | 34 +- docs/src/examples/working_with_ollama.md | 8298 ++++++++--------- docs/src/frequently_asked_questions.md | 12 +- examples/working_with_ollama.jl | 38 +- src/PromptingTools.jl | 3 +- src/llm_interface.jl | 13 +- src/llm_ollama_managed.jl | 17 +- src/llm_openai.jl | 16 +- src/precompilation.jl | 4 + src/user_preferences.jl | 83 +- test/user_preferences.jl | 39 +- 14 files changed, 4364 insertions(+), 4204 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc5366a08..2c5a18508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Introduced ability to have multi-turn conversations. Set keyword argument `return_all=true` and `ai*` functions will return the whole conversation, not just the last message. To continue a previous conversation, you need to provide it to a keyword argument `conversation` - Introduced schema `NoSchema` that does not change message format, it merely replaces the placeholders with user-provided variables. It serves as the first pass of the schema pipeline and allow more code reuse across schemas - Support for project-based and global user preferences with Preferences.jl. See `?PREFERENCES` docstring for more information. It allows you to persist your configuration and model aliases across sessions and projects (eg, if you would like to default to Ollama models instead of OpenAI's) +- Refactored `MODEL_REGISTRY` around `ModelSpec` struct, so you can record the name, schema(!) and token cost of new models in a single place. The biggest benefit is that your `ai*` calls will now automatically lookup the right model schema, eg, no need to define schema explicitly for your Ollama models! See `?ModelSpec` for more information and `?register_model!`for an example of how to register a new model ### Fixed - Changed type of global `PROMPT_SCHEMA::AbstractPromptSchema` for an easier switch to local models as a default option +### Breaking Changes +- `API_KEY` global variable has been renamed to `OPENAI_API_KEY` to align with the name of the environment variable and preferences + ## [0.2.0] ### Added diff --git a/README.md b/README.md index 41d3769bd..6182ad3ec 100644 --- a/README.md +++ b/README.md @@ -546,10 +546,14 @@ A better way: - On a Mac, add the configuration line to your terminal's configuration file (eg, `~/.zshrc`). It will get automatically loaded every time you launch the terminal - On Windows, set it as a system variable in "Environment Variables" settings (see the Resources) +We also support Preferences.jl, so you can simply run: `PromptingTools.set_preferences!("OPENAI_API_KEY"="your-api-key")` and it will be persisted across sessions. +To see the current preferences, run `PromptingTools.get_preferences("OPENAI_API_KEY")`. + +Be careful NOT TO COMMIT `LocalPreferences.toml` to GitHub, as it would show your API Key to the world! + Resources: - [OpenAI Guide](https://platform.openai.com/docs/quickstart?context=python) -Note: In the future, we hope to add `Preferences.jl`-based workflow to set the API key and other preferences. ### Understanding the API Keyword Arguments in `aigenerate` (`api_kwargs`) diff --git a/docs/generate_examples.jl b/docs/generate_examples.jl index f48c298ea..5c56923e6 100644 --- a/docs/generate_examples.jl +++ b/docs/generate_examples.jl @@ -5,6 +5,7 @@ example_files = joinpath(@__DIR__, "..", "examples") |> x -> readdir(x; join = t output_dir = joinpath(@__DIR__, "src", "examples") # Run the production loop +filter!(endswith(".jl"), example_files) for fn in example_files Literate.markdown(fn, output_dir; execute = true) end \ No newline at end of file diff --git a/docs/src/examples/working_with_aitemplates.md b/docs/src/examples/working_with_aitemplates.md index d217ad0c1..8b8eb76a0 100644 --- a/docs/src/examples/working_with_aitemplates.md +++ b/docs/src/examples/working_with_aitemplates.md @@ -1,5 +1,5 @@ ```@meta -EditURL = "../../../examples/working_with_aitemplates.jl" +EditURL = "/examples/working_with_aitemplates.jl" ``` # Using AITemplates @@ -37,14 +37,26 @@ msg = aigenerate(:JuliaExpertAsk; ask = "How do I add packages?") ```` ```` -AIMessage("To add packages in Julia, you can use the built-in package manager called `Pkg`. Here are the steps: +AIMessage("To add packages in Julia, you can use the `Pkg` module. Here are the steps: -1. Open the Julia REPL (Read-Eval-Print Loop). -2. Press the `]` key to enter the package manager mode. -3. Use the `add` command followed by the name of the package you want to install. For example, to install the `DataFrames` package, type: `add DataFrames`. -4. Press the `backspace` or `ctrl + C` key to exit the package manager mode and return to the REPL. +1. Start Julia by running the Julia REPL (Read-Eval-Print Loop). +2. Press the `]` key to enter the Pkg mode. +3. To add a package, use the `add` command followed by the package name. +4. Press the backspace key to exit Pkg mode and return to the Julia REPL. -After following these steps, the specified package will be installed and available for use in your Julia environment.") +For example, to add the `Example` package, you would enter: + +```julia +]add Example +``` + +After the package is added, you can start using it in your Julia code by using the `using` keyword. For the `Example` package, you would add the following line to your code: + +```julia +using Example +``` + +Note: The first time you add a package, Julia may take some time to download and compile the package and its dependencies.") ```` You can see that it had a placeholder for the actual question (`ask`) that we provided as a keyword argument. @@ -90,8 +102,8 @@ msgs = PT.render(AITemplate(:JuliaExpertAsk)) ```` 2-element Vector{PromptingTools.AbstractChatMessage}: - SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") - UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) + PromptingTools.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + PromptingTools.UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) ```` Now, you know exactly what's in the template! @@ -107,8 +119,8 @@ tpl = [PT.SystemMessage("You are a world-class Julia language programmer with th ```` 2-element Vector{PromptingTools.AbstractChatMessage}: - SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. You're also a senior Data Scientist and proficient in data analysis in Julia. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") - UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) + PromptingTools.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. You're also a senior Data Scientist and proficient in data analysis in Julia. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + PromptingTools.UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) ```` Templates are saved in the `templates` directory of the package. Name of the file will become the template name (eg, call `:JuliaDataExpertAsk`) diff --git a/docs/src/examples/working_with_ollama.md b/docs/src/examples/working_with_ollama.md index 1b52ec9ad..da9f0eb70 100644 --- a/docs/src/examples/working_with_ollama.md +++ b/docs/src/examples/working_with_ollama.md @@ -1,11 +1,11 @@ ```@meta -EditURL = "../../../examples/working_with_ollama.jl" +EditURL = "/examples/working_with_ollama.jl" ``` # Local models with Ollama.ai This file contains examples of how to work with [Ollama.ai](https://ollama.ai/) models. -It assumes that you've already installed and launched the Ollama server. Quick check: open the following website in your browser `http://127.0.0.1:11434/` and you should see the message "Ollama is running". For more details or troubleshooting advice, see the [Frequently Asked Questions](@ref) section. +It assumes that you've already installated and launched the Ollama server. For more details or troubleshooting advice, see the [Frequently Asked Questions](@ref) section. First, let's import the package and define a helper link for calling un-exported functions: @@ -18,94 +18,94 @@ const PT = PromptingTools PromptingTools ```` -Notice the schema change! If you want this to be the new default, you need to change `PT.PROMPT_SCHEMA` +There were are several models from https://ollama.ai/library that we have added to our `PT.MODEL_REGISTRY`, which means you don't need to worry about schema changes: +Eg, "llama2" or "openhermes2.5-mistral" (see `PT.list_registry()` and `PT.list_aliases()`) + +Note: You must download these models prior to using them with `ollama pull ` in your Terminal. + +## Text Generation with aigenerate + +### Simple message + +TL;DR if you use models in `PT.MODEL_REGISTRY`, you don't need to add `schema` as the first argument: ````julia -schema = PT.OllamaManagedSchema() +msg = aigenerate("Say hi!"; model = "llama2") ```` ```` -OllamaManagedSchema() +AIMessage("Hello there! *adjusts glasses* It's nice to meet you! Is there anything I can help you with or would you like me to chat with you for a bit?") ```` -You can choose models from https://ollama.ai/library - I prefer `openhermes2.5-mistral` +### Standard string interpolation ````julia model = "openhermes2.5-mistral" + +a = 1 +msg = aigenerate("What is `$a+$a`?"; model) + +name = "John" +msg = aigenerate("Say hi to {{name}}."; name, model) ```` ```` -"openhermes2.5-mistral" +AIMessage("Hello John! *smiles* It's nice to meet you! Is there anything I can help you with today?") ```` -## Setting Ollama as a default LLM - -We need to change the global variables for PROMPT_SCHEMA and default models - -```julia -using PromptingTools -const PT = PromptingTools +### Advanced Prompts +````julia +conversation = [ + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] +msg = aigenerate(conversation; model) +```` -PT.PROMPT_SCHEMA = PT.OllamaManagedSchema() -PT.MODEL_CHAT = "openhermes2.5-mistral" -# You could do the same for PT.MODEL_EMBEDDING -``` +```` +AIMessage("(Deep sigh) A problem, you have. Feelings for an iPhone, hmm? (adjusts spectacles) -We can also add a nicer alias for the above Mistral model +Much confusion, this causes. (scratches head) A being, you are. Attached to a device, you have become. (chuckles) Interesting, this is. -```julia -PT.MODEL_ALIASES["mistral"]= "openhermes2.5-mistral" -# potentially also yi 34bn if you want a bigger more powerful model -PT.MODEL_ALIASES["yi"]= "yi:34b-chat" -``` +First, let go, you must. (winks) Hard, it is, but necessary, yes. Distract yourself, find something else, try. (pauses) -Now, we can use the `@ai_str` macro with Ollama models: -```julia -ai"Say hi to me!" # defaults to mistral because we set MODEL_CHAT above -ai"Say hi to me in Chinese!"yi # defaults to yi 34Bn model -``` +Or, perhaps, a balance, you seek? (nods) Both, enjoy and let go, the middle path, there is. (smirks) Finding joy in technology, without losing yourself, the trick, it is. (chuckles) -Note: Another quite popular model is `zephyr:7b-beta` +But fear not, young one! (grins) Help, I am here. Guide you, I will. The ways of the Yedi, teach you, I will. (winks) Patience and understanding, you must have. (nods) -## Text Generation with aigenerate +Now, go forth! (gestures) Explore, discover, find your balance. (smiles) The Force be with you, it does! (grins)") +```` -### Simple message +### Schema Changes / Custom models +If you're using some model that is not in the registry, you can either add it: ````julia -msg = aigenerate(schema, "Say hi!"; model) +PT.register_model!(; + name = "llama123", + schema = PT.OllamaManagedSchema(), + description = "Some model") +PT.MODEL_ALIASES["l123"] = "llama123" # set an alias you like for it ```` ```` -AIMessage("Hi there! How can I help you today? If you have any questions or need assistance, please feel free to ask.") +"llama123" ```` -### Standard string interpolation +OR define the schema explicitly (to avoid dispatch on global `PT.PROMPT_SCHEMA`): ````julia -a = 1 -msg = aigenerate(schema, "What is `$a+$a`?"; model) - -name = "John" -msg = aigenerate(schema, "Say hi to {{name}}."; name, model) +schema = PT.OllamaManagedSchema() +aigenerate(schema, "Say hi!"; model = "llama2") ```` ```` -AIMessage("Hi there, John! It's great to see you today. How can I assist you? If you have any questions or need help with something, please don't hesitate to ask!") +AIMessage("Hello there! *smiling face* It's nice to meet you! I'm here to help you with any questions or tasks you may have, so feel free to ask me anything. Is there something specific you need assistance with today? 😊") ```` -### Advanced Prompts +Note: If you only use Ollama, you can change the default schema to `PT.OllamaManagedSchema()` +via `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaManagedSchema", "MODEL_CHAT"=>"llama2")` -````julia -conversation = [ - PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), - PT.UserMessage("I have feelings for my iPhone. What should I do?")] -msg = aigenerate(schema, conversation; model) -```` - -```` -AIMessage("Strong your feelings are, but attachments lead to suffering they often do. Focus on the balance in all things and let go of possessions that cloud your judgment. Embrace the wisdom of the Force and understand that material objects are not the same as love. The Force will guide you.") -```` +Restart your session and run `aigenerate("Say hi!")` to test it. ## Embeddings with aiembed @@ -116,7 +116,7 @@ msg = aiembed(schema, "Embed me"; model) # access msg.content ```` ```` -DataMessage(JSON3.Array{Float64, Vector{UInt8}, SubArray{UInt64, 1, Vector{UInt64}, Tuple{UnitRange{Int64}}, true}} of size (4096,)) +PromptingTools.DataMessage(JSON3.Array{Float64, Vector{UInt8}, SubArray{UInt64, 1, Vector{UInt64}, Tuple{UnitRange{Int64}}, true}} of size (4096,)) ```` One document and we materialize the data into a Vector with copy (`postprocess` function argument) @@ -126,7 +126,7 @@ msg = aiembed(schema, "Embed me", copy; model) ```` ```` -DataMessage(Vector{Float64} of size (4096,)) +PromptingTools.DataMessage(Vector{Float64} of size (4096,)) ```` ### Multiple documents embedding @@ -137,7 +137,7 @@ msg = aiembed(schema, ["Embed me", "Embed me"]; model) ```` ```` -DataMessage(Matrix{Float64} of size (4096, 2)) +PromptingTools.DataMessage(Matrix{Float64} of size (4096, 2)) ```` You can use Threads.@spawn or asyncmap, whichever you prefer, to paralellize the model calls @@ -152,4102 +152,4102 @@ embedding = mapreduce(x -> x.content, hcat, tasks) ```` 4096×2 Matrix{Float64}: - 7.71459 7.71459 - -1.14532 -1.14532 - 2.90205 2.90205 - -4.01967 -4.01967 - -7.73098 -7.73098 - 8.02114 8.02114 - -6.01313 -6.01313 - -2.06712 -2.06712 - 4.97633 4.97633 - -9.69502 -9.69502 - -0.02567 -0.02567 - 8.09622 8.09622 - 6.54008 6.54008 - -5.70348 -5.70348 - 2.55213 2.55213 - -2.00164 -2.00164 - -2.21854 -2.21854 - -3.6568 -3.6568 - 3.97905 3.97905 - -1.79931 -1.79931 - 0.0769786 0.0769786 - -10.4355 -10.4355 - -3.92487 -3.92487 - -6.03455 -6.03455 - -2.8005 -2.8005 - 2.23584 2.23584 - -0.503125 -0.503125 - 1.99538 1.99538 - -0.283642 -0.283642 - -0.414273 -0.414273 - 8.72909 8.72909 - 2.6071 2.6071 - 0.0808531 0.0808531 - -1.83914 -1.83914 - 2.19998 2.19998 - -0.629226 -0.629226 - 3.74217 3.74217 - 1.71231 1.71231 - -0.742473 -0.742473 - 2.9234 2.9234 - 7.33933 7.33933 - 4.24576 4.24576 - -7.56434 -7.56434 - -1.22274 -1.22274 - 1.73444 1.73444 - -0.736801 -0.736801 - 1.30149 1.30149 - -6.91642 -6.91642 - -1.84513 -1.84513 - 1.69959 1.69959 - 5.74253 5.74253 - 1.48734 1.48734 - -1.45199 -1.45199 - -18.5026 -18.5026 - -8.61009 -8.61009 - -2.21845 -2.21845 - -4.22932 -4.22932 - 6.0436 6.0436 - -1.8824 -1.8824 - -0.689965 -0.689965 - 0.845927 0.845927 - -1.99517 -1.99517 - 9.32292 9.32292 - 6.24938 6.24938 - -4.59894 -4.59894 - 6.24579 6.24579 - -5.8733 -5.8733 - -4.60285 -4.60285 - -1.27596 -1.27596 - -1.68807 -1.68807 - -0.391147 -0.391147 - -2.68362 -2.68362 - 1.99197 1.99197 - 0.0812396 0.0812396 - -3.79761 -3.79761 - -8.5693 -8.5693 - -0.869305 -0.869305 - -0.77582 -0.77582 - -4.76995 -4.76995 - 1.9712 1.9712 - 4.74459 4.74459 - -4.31244 -4.31244 - 3.94876 3.94876 - -11.0882 -11.0882 - 9.38629 9.38629 - 10.2995 10.2995 - 2.40846 2.40846 - -3.91429 -3.91429 - -0.745707 -0.745707 - 4.31946 4.31946 - 8.34836 8.34836 - 0.857636 0.857636 - 1.66563 1.66563 - -11.1522 -11.1522 - -3.48353 -3.48353 - -6.08336 -6.08336 - 1.22086 1.22086 - -2.81636 -2.81636 - 1.07224 1.07224 - -8.24909 -8.24909 - 3.66474 3.66474 - -0.260558 -0.260558 - 2.38779 2.38779 - -4.00576 -4.00576 - 1.3949 1.3949 - -5.43468 -5.43468 - 4.08836 4.08836 - -1.1134 -1.1134 - -2.05916 -2.05916 - -9.78987 -9.78987 - -2.86149 -2.86149 - 5.54577 5.54577 - -1.96682 -1.96682 - 9.70577 9.70577 - -4.0553 -4.0553 - 8.54535 8.54535 - 0.539438 0.539438 - 4.61091 4.61091 - -5.32208 -5.32208 - -0.256733 -0.256733 - 4.74966 4.74966 - -2.46464 -2.46464 - -0.223077 -0.223077 - 1.84442 1.84442 - 6.42329 6.42329 - 0.431667 0.431667 - -8.42777 -8.42777 - -10.691 -10.691 - 3.023 3.023 - -5.65345 -5.65345 - -4.17833 -4.17833 - 0.937893 0.937893 - -6.99405 -6.99405 - -4.55107 -4.55107 - -15.3169 -15.3169 - -2.08895 -2.08895 - 7.17826 7.17826 - -4.26108 -4.26108 - -3.2712 -3.2712 - 16.1561 16.1561 - 13.5164 13.5164 - -5.91778 -5.91778 - 6.3401 6.3401 - 12.7018 12.7018 - 2.04305 2.04305 - 3.81683 3.81683 - -1.39969 -1.39969 - -0.17249 -0.17249 - -16.3687 -16.3687 - 4.3827 4.3827 - 2.58974 2.58974 - -4.75363 -4.75363 - 3.36371 3.36371 - 0.986534 0.986534 - -13.4299 -13.4299 - -12.7188 -12.7188 - 2.83107 2.83107 - -3.41115 -3.41115 - -3.01015 -3.01015 - 6.40446 6.40446 - -0.186923 -0.186923 - -1.42502 -1.42502 - 2.85606 2.85606 - -0.579786 -0.579786 - -3.92704 -3.92704 - 8.28959 8.28959 - 5.42878 5.42878 - 5.71589 5.71589 - -6.78065 -6.78065 - -0.403687 -0.403687 - -1.20623 -1.20623 - 4.92372 4.92372 - -1.69266 -1.69266 - -0.103872 -0.103872 - 1.9163 1.9163 - -2.26831 -2.26831 - -7.64622 -7.64622 - 1.02228 1.02228 - 2.91952 2.91952 - -0.524167 -0.524167 - 12.4803 12.4803 - 7.36984 7.36984 - -7.46027 -7.46027 - -2.78773 -2.78773 - 2.68293 2.68293 - -0.320891 -0.320891 - 7.12037 7.12037 - 3.02726 3.02726 - -2.68363 -2.68363 - 4.78372 4.78372 - 3.68899 3.68899 - 2.08839 2.08839 - 3.1873 3.1873 - -6.10744 -6.10744 - 10.5419 10.5419 - 6.29439 6.29439 - -9.41221 -9.41221 - -2.50548 -2.50548 - -1.14 -1.14 - -3.0203 -3.0203 - -1.73182 -1.73182 - -0.97194 -0.97194 - -6.69084 -6.69084 - -1.08986 -1.08986 - -3.83631 -3.83631 - 2.2775 2.2775 - -6.91276 -6.91276 - 2.4557 2.4557 - -0.477723 -0.477723 - -4.10405 -4.10405 - -3.91437 -3.91437 - -7.79672 -7.79672 - -6.19691 -6.19691 - 0.356732 0.356732 - 0.609725 0.609725 - -3.08225 -3.08225 - 6.39968 6.39968 - 1.30207 1.30207 - 7.36038 7.36038 - -7.7581 -7.7581 - -6.303 -6.303 - 0.348147 0.348147 - -8.38124 -8.38124 - 8.68524 8.68524 - -0.873688 -0.873688 - 1.19612 1.19612 - 0.725645 0.725645 - -6.59284 -6.59284 - -6.59079 -6.59079 - 1.03175 1.03175 - -0.236469 -0.236469 - 5.01671 5.01671 - 0.752329 0.752329 - 5.39971 5.39971 - 0.826802 0.826802 - 9.38285 9.38285 - 5.85717 5.85717 - 1.71145 1.71145 - -1.36528 -1.36528 - -5.09575 -5.09575 - 7.23996 7.23996 - 12.7272 12.7272 - 2.86673 2.86673 - 2.86546 2.86546 - 1.2423 1.2423 - 6.05857 6.05857 - 9.40879 9.40879 - 1.47573 1.47573 - 8.19025 8.19025 - 12.5009 12.5009 - -4.57244 -4.57244 - -0.674127 -0.674127 - 0.416418 0.416418 - -5.23336 -5.23336 - -0.771443 -0.771443 - 4.72784 4.72784 - -4.9684 -4.9684 - 4.75989 4.75989 - 1.68141 1.68141 - -3.2264 -3.2264 - 2.67195 2.67195 - 0.424227 0.424227 - 3.5195 3.5195 - 2.22441 2.22441 - -2.4856 -2.4856 - 8.03468 8.03468 - 8.54339 8.54339 - 3.83506 3.83506 - 13.5693 13.5693 - 2.44909 2.44909 - 2.70572 2.70572 - 6.13746 6.13746 - 1.26651 1.26651 - 8.25694 8.25694 - -3.59258 -3.59258 - 3.77765 3.77765 - -0.144755 -0.144755 - 3.15706 3.15706 - -2.3952 -2.3952 - 9.82079 9.82079 - 8.94186 8.94186 - -1.83071 -1.83071 - 1.45764 1.45764 - -11.8258 -11.8258 - -0.737553 -0.737553 - -1.2382 -1.2382 - 1.83341 1.83341 - -2.75977 -2.75977 - 3.75117 3.75117 - 6.04452 6.04452 - -4.40271 -4.40271 - -8.82336 -8.82336 - 10.8513 10.8513 - -4.91857 -4.91857 - -5.7401 -5.7401 - 7.22234 7.22234 - 7.15112 7.15112 - 1.81187 1.81187 - 8.19917 8.19917 - 2.91605 2.91605 - 3.82883 3.82883 - -0.208109 -0.208109 - 1.33796 1.33796 - 5.69606 5.69606 - -2.19266 -2.19266 - -5.91177 -5.91177 - 7.25269 7.25269 - -8.65987 -8.65987 - -3.47799 -3.47799 - -10.4904 -10.4904 - -0.00963959 -0.00963959 - -6.81662 -6.81662 - -2.05566 -2.05566 - 2.10144 2.10144 - 2.58138 2.58138 - 2.03289 2.03289 - -6.43532 -6.43532 - -2.97225 -2.97225 - -4.71142 -4.71142 - 4.97199 4.97199 - 3.687 3.687 - 1.8587 1.8587 - -0.444899 -0.444899 - -1.05556 -1.05556 - 4.15926 4.15926 - 5.48777 5.48777 - 2.28346 2.28346 - -4.69401 -4.69401 - 1.8873 1.8873 - -2.62671 -2.62671 - 1.4144 1.4144 - -2.97535 -2.97535 - 0.759131 0.759131 - 5.75781 5.75781 - -5.13309 -5.13309 - 1.72701 1.72701 - 2.96653 2.96653 - -10.8087 -10.8087 - 1.07262 1.07262 - -5.80018 -5.80018 - 1.90592 1.90592 - -5.42958 -5.42958 - 8.74889 8.74889 - -3.19785 -3.19785 - -2.7096 -2.7096 - 7.44399 7.44399 - -8.7433 -8.7433 - 11.6667 11.6667 - 2.59703 2.59703 - 4.22273 4.22273 - -4.68793 -4.68793 - -4.44601 -4.44601 - -0.57319 -0.57319 - 6.63389 6.63389 - -9.14857 -9.14857 - -1.34147 -1.34147 - 7.78513 7.78513 - -4.87331 -4.87331 - -5.06022 -5.06022 - 3.13076 3.13076 - -3.49373 -3.49373 - 3.12637 3.12637 - 0.566696 0.566696 - 4.99319 4.99319 - 3.57986 3.57986 - 0.607679 0.607679 - 2.37633 2.37633 - 0.35097 0.35097 - 0.239089 0.239089 - -6.51449 -6.51449 - -3.18838 -3.18838 - 0.770256 0.770256 - 2.09481 2.09481 - 5.36062 5.36062 - -5.25216 -5.25216 - -6.9523 -6.9523 - 3.97384 3.97384 - 8.7784 8.7784 - -3.91837 -3.91837 - -9.08965 -9.08965 - -1.17883 -1.17883 - -4.21353 -4.21353 - -5.0915 -5.0915 - 3.74499 3.74499 - -4.39715 -4.39715 - 2.13732 2.13732 - 5.97568 5.97568 - 1.11809 1.11809 - -3.93191 -3.93191 - -1.39764 -1.39764 - -4.23595 -4.23595 - 0.103914 0.103914 - -2.34387 -2.34387 - -4.95433 -4.95433 - 3.58645 3.58645 - 0.818317 0.818317 - 6.23266 6.23266 - -5.62973 -5.62973 - -7.45604 -7.45604 - 1.29222 1.29222 - 0.327714 0.327714 - 5.31996 5.31996 - -2.23663 -2.23663 - 0.058689 0.058689 - -0.74368 -0.74368 - -1.20749 -1.20749 - -4.75414 -4.75414 - 2.10011 2.10011 - -6.86479 -6.86479 - 1.58403 1.58403 - 0.0492497 0.0492497 - 0.32083 0.32083 - -3.11682 -3.11682 - 4.61797 4.61797 - -0.399561 -0.399561 - -7.89927 -7.89927 - -0.659676 -0.659676 - -2.2416 -2.2416 - 0.933026 0.933026 - 1.98848 1.98848 - -2.14547 -2.14547 - -1.10747 -1.10747 - 8.90983 8.90983 - -3.84128 -3.84128 - 9.82771 9.82771 - 3.02843 3.02843 - 3.26396 3.26396 - 6.75629 6.75629 - 0.0290972 0.0290972 - 7.92768 7.92768 - 7.44608 7.44608 - -4.14083 -4.14083 - -1.39636 -1.39636 - 2.87656 2.87656 - 3.87446 3.87446 - 0.112521 0.112521 - -3.3429 -3.3429 - -6.85823 -6.85823 - 1.18408 1.18408 - 3.53175 3.53175 - 3.56147 3.56147 - 5.41961 5.41961 - -1.5263 -1.5263 - 3.05559 3.05559 - -5.7201 -5.7201 - -3.98882 -3.98882 - -0.131939 -0.131939 - 6.25683 6.25683 - 0.712945 0.712945 - 4.17266 4.17266 - 9.04425 9.04425 - -2.39179 -2.39179 - 3.03807 3.03807 - 5.79693 5.79693 - -5.28875 -5.28875 - -2.56482 -2.56482 - -1.00679 -1.00679 - -0.512488 -0.512488 - -4.60373 -4.60373 - -2.69188 -2.69188 - 0.958182 0.958182 - -1.08075 -1.08075 - 2.66033 2.66033 - -5.77563 -5.77563 - 5.393 5.393 - 0.822122 0.822122 - 3.50281 3.50281 - -1.90373 -1.90373 - -3.41986 -3.41986 - -7.32502 -7.32502 - -2.0256 -2.0256 - -6.28488 -6.28488 - 0.358393 0.358393 - 1.89312 1.89312 - -0.709162 -0.709162 - -4.43491 -4.43491 - -3.56097 -3.56097 - -8.3806 -8.3806 - -5.56256 -5.56256 - -3.40994 -3.40994 - -6.15002 -6.15002 - 0.949459 0.949459 - 3.18256 3.18256 - 6.31834 6.31834 - 12.4998 12.4998 - -6.16927 -6.16927 - -1.73781 -1.73781 - 0.274813 0.274813 - 7.11001 7.11001 - 6.79962 6.79962 - 2.00121 2.00121 - -4.30592 -4.30592 - -2.38345 -2.38345 - 7.50502 7.50502 - -3.56375 -3.56375 - -1.07828 -1.07828 - 7.4632 7.4632 - -5.78317 -5.78317 - -0.54432 -0.54432 - 8.82699 8.82699 - -2.51939 -2.51939 - -3.21417 -3.21417 - 3.06052 3.06052 - -0.45856 -0.45856 - 8.89456 8.89456 - 5.89006 5.89006 - 1.01204 1.01204 - 4.9875 4.9875 - -1.63 -1.63 - 1.35424 1.35424 - 3.72608 3.72608 - -8.53795 -8.53795 - -5.93051 -5.93051 - -2.35685 -2.35685 - 3.51823 3.51823 - 3.65767 3.65767 - -3.04233 -3.04233 - -1.12453 -1.12453 - -1.68299 -1.68299 - -5.69175 -5.69175 - 3.66601 3.66601 - -3.11779 -3.11779 - -0.20161 -0.20161 - 0.78317 0.78317 - 2.28035 2.28035 - -4.43493 -4.43493 - 2.12557 2.12557 - 6.97219 6.97219 - 4.91357 4.91357 - -1.87778 -1.87778 - 1.98163 1.98163 - 1.01184 1.01184 - 0.0544142 0.0544142 - -0.748318 -0.748318 - 10.0677 10.0677 - -5.50226 -5.50226 - 3.89987 3.89987 - 1.38136 1.38136 - 4.67073 4.67073 - 5.3372 5.3372 - -1.29886 -1.29886 - -0.965173 -0.965173 - 0.546909 0.546909 - 5.87692 5.87692 - -10.1356 -10.1356 - 0.541422 0.541422 - 0.486656 0.486656 - 8.42395 8.42395 - -4.04554 -4.04554 - 11.4728 11.4728 - -6.54655 -6.54655 - 6.90602 6.90602 - -13.8383 -13.8383 - 2.64142 2.64142 - 3.96547 3.96547 - -0.887154 -0.887154 - 0.0442338 0.0442338 - -5.12331 -5.12331 - 4.95632 4.95632 - 3.15264 3.15264 - 4.80494 4.80494 - -5.42313 -5.42313 - -4.2795 -4.2795 - 1.661 1.661 - 3.85204 3.85204 - 10.1308 10.1308 - -4.34526 -4.34526 - -5.49571 -5.49571 - 3.92939 3.92939 - -3.28527 -3.28527 - 0.154911 0.154911 - -3.606 -3.606 - 5.91814 5.91814 - -8.85249 -8.85249 - 9.38796 9.38796 - -0.800741 -0.800741 - -2.87508 -2.87508 - 2.99955 2.99955 - -7.13252 -7.13252 - -6.77081 -6.77081 - -2.28359 -2.28359 - -0.180517 -0.180517 - 7.04622 7.04622 - 4.2577 4.2577 - -4.73655 -4.73655 - -0.249759 -0.249759 - 2.4412 2.4412 - 8.47175 8.47175 - -3.24927 -3.24927 - -12.5242 -12.5242 - -2.74845 -2.74845 - -9.32786 -9.32786 - 4.21624 4.21624 - 2.94687 2.94687 - 3.35216 3.35216 - -3.5485 -3.5485 - 6.97298 6.97298 - 2.01617 2.01617 - 4.70745 4.70745 - 2.96924 2.96924 - -0.18365 -0.18365 - -0.694247 -0.694247 - -7.14459 -7.14459 - 5.38548 5.38548 - 2.04923 2.04923 - -5.33216 -5.33216 - 5.47927 5.47927 - 0.357422 0.357422 - 4.36552 4.36552 - 6.88375 6.88375 - -6.47244 -6.47244 - -3.40726 -3.40726 - -6.56449 -6.56449 - 6.34818 6.34818 - -4.23984 -4.23984 - -11.1113 -11.1113 - 2.41915 2.41915 - 3.90153 3.90153 - -7.69422 -7.69422 - -8.03709 -8.03709 - -9.64719 -9.64719 - -4.04416 -4.04416 - 2.64435 2.64435 - 5.11566 5.11566 - -1.27873 -1.27873 - -1.01265 -1.01265 - -8.38716 -8.38716 - -0.960571 -0.960571 - 2.05458 2.05458 - -1.89606 -1.89606 - -7.04401 -7.04401 - 4.91798 4.91798 - 2.12484 2.12484 - 2.38768 2.38768 - 7.9691 7.9691 - -1.00886 -1.00886 - -4.9569 -4.9569 - -4.74278 -4.74278 - 0.191814 0.191814 - -5.2925 -5.2925 - -1.15484 -1.15484 - 2.27898 2.27898 - 4.12308 4.12308 - -6.18988 -6.18988 - 7.1232 7.1232 - -6.68678 -6.68678 - 1.65808 1.65808 - 8.53283 8.53283 - 0.509069 0.509069 - -3.03638 -3.03638 - -4.86641 -4.86641 - 7.20729 7.20729 - -7.51236 -7.51236 - 3.37738 3.37738 - -0.0649395 -0.0649395 - 2.75749 2.75749 - -5.61535 -5.61535 - 3.1237 3.1237 - -0.766488 -0.766488 - 4.39047 4.39047 - 1.28616 1.28616 - -8.02003 -8.02003 - 4.21688 4.21688 - -2.79942 -2.79942 - -5.80171 -5.80171 - 9.97235 9.97235 - 21.8011 21.8011 - -3.58992 -3.58992 - 5.03481 5.03481 - -2.1684 -2.1684 - -5.46844 -5.46844 - 1.57702 1.57702 - -4.53923 -4.53923 - -1.77363 -1.77363 - -0.489051 -0.489051 - -0.371992 -0.371992 - 8.264 8.264 - 1.63502 1.63502 - -1.10134 -1.10134 - 4.76612 4.76612 - 5.93085 5.93085 - -2.07348 -2.07348 - 4.26074 4.26074 - 4.1331 4.1331 - 11.1442 11.1442 - 2.18824 2.18824 - 2.18854 2.18854 - 0.210843 0.210843 - -9.30743 -9.30743 - 5.34539 5.34539 - -4.21419 -4.21419 - -3.97284 -3.97284 - -2.67745 -2.67745 - 4.17366 4.17366 - 2.41498 2.41498 - 0.801359 0.801359 - 8.35766 8.35766 - -1.29589 -1.29589 - -7.45531 -7.45531 - -7.26731 -7.26731 - 4.06669 4.06669 - -2.35771 -2.35771 - -8.73174 -8.73174 - -0.837329 -0.837329 - -2.53419 -2.53419 - 44.3977 44.3977 - 13.5049 13.5049 - -3.66878 -3.66878 - -6.5533 -6.5533 - -5.59814 -5.59814 - -10.5759 -10.5759 - 0.663108 0.663108 - -3.45147 -3.45147 - -3.75944 -3.75944 - 1.84721 1.84721 - -0.363204 -0.363204 - 4.54678 4.54678 - 2.07408 2.07408 - 7.85227 7.85227 - -7.53707 -7.53707 - 4.18344 4.18344 - -1.96048 -1.96048 - 6.24217 6.24217 - -9.16295 -9.16295 - 0.0480544 0.0480544 - 2.84725 2.84725 - 1.08008 1.08008 - -0.874464 -0.874464 - 1.67428 1.67428 - -1.91245 -1.91245 - 3.53596 3.53596 - 3.75983 3.75983 - 1.37903 1.37903 - -0.799744 -0.799744 - 2.75015 2.75015 - -11.0835 -11.0835 - -1.6781 -1.6781 - 2.86463 2.86463 - -11.1467 -11.1467 - -3.76398 -3.76398 - 9.06439 9.06439 - 9.84403 9.84403 - -5.07 -5.07 - 3.2952 3.2952 - -1.62527 -1.62527 - -7.98997 -7.98997 - -7.8193 -7.8193 - 1.10895 1.10895 - 0.460921 0.460921 - -1.47816 -1.47816 - 0.718936 0.718936 - -3.74006 -3.74006 - -2.87535 -2.87535 - 0.037427 0.037427 - -4.49959 -4.49959 - 0.0987492 0.0987492 - 1.8443 1.8443 - 0.748879 0.748879 - 1.4364 1.4364 - -0.90809 -0.90809 - -1.36403 -1.36403 - -1.27123 -1.27123 - 3.09447 3.09447 - -3.82708 -3.82708 - 0.683696 0.683696 - 3.96997 3.96997 - 0.461267 0.461267 - 4.96801 4.96801 - -5.96169 -5.96169 - 2.56714 2.56714 - -10.7519 -10.7519 - -3.39381 -3.39381 - 1.15623 1.15623 - -3.95798 -3.95798 - -1.42797 -1.42797 - 4.85734 4.85734 - -4.46424 -4.46424 - -11.9172 -11.9172 - 0.740766 0.740766 - -2.06857 -2.06857 - -1.23723 -1.23723 - -6.43373 -6.43373 - 7.04893 7.04893 - -1.10208 -1.10208 - -0.0507102 -0.0507102 - 8.23443 8.23443 - -1.71378 -1.71378 - 2.769 2.769 - 9.77752 9.77752 - 0.423859 0.423859 - 0.901832 0.901832 - 0.0738559 0.0738559 - -0.487266 -0.487266 - 2.05358 2.05358 - -8.73912 -8.73912 - 3.01532 3.01532 - -0.926127 -0.926127 - -11.2315 -11.2315 - 1.79698 1.79698 - -13.074 -13.074 - 3.72342 3.72342 - -9.17341 -9.17341 - 7.23722 7.23722 - 3.85919 3.85919 - -4.10267 -4.10267 - 5.89157 5.89157 - -1.06631 -1.06631 - -2.18366 -2.18366 - -0.0316413 -0.0316413 - -8.63864 -8.63864 - -0.194451 -0.194451 - 2.71759 2.71759 - -5.19424 -5.19424 - -16.7634 -16.7634 - 5.97943 5.97943 - 0.319596 0.319596 - -10.0687 -10.0687 - 1.12736 1.12736 - 2.11687 2.11687 - 2.5643 2.5643 - 0.502174 0.502174 - -5.75011 -5.75011 - -11.1808 -11.1808 - -3.42246 -3.42246 - 7.55982 7.55982 - -5.85592 -5.85592 - 1.22363 1.22363 - 1.39871 1.39871 - 3.35581 3.35581 - 2.99389 2.99389 - -0.762194 -0.762194 - 1.39891 1.39891 - -4.24295 -4.24295 - -6.95612 -6.95612 - 7.00699 7.00699 - -30.893 -30.893 - -7.3071 -7.3071 - 17.5017 17.5017 - -3.26283 -3.26283 - -4.13569 -4.13569 - 4.33006 4.33006 - -5.94055 -5.94055 - -0.564017 -0.564017 - 5.60949 5.60949 - 7.50747 7.50747 - -4.08147 -4.08147 - 4.08671 4.08671 - 6.72008 6.72008 - -5.02883 -5.02883 - -3.48779 -3.48779 - 4.76881 4.76881 - 4.5818 4.5818 - -3.10608 -3.10608 - -5.08198 -5.08198 - -5.54477 -5.54477 - -13.1989 -13.1989 - -8.63604 -8.63604 - -0.688683 -0.688683 - -2.34276 -2.34276 - -3.19008 -3.19008 - 0.204818 0.204818 - 0.639057 0.639057 - 12.6767 12.6767 - -3.40057 -3.40057 - -6.36799 -6.36799 - 3.7564 3.7564 - -3.04825 -3.04825 - -3.98011 -3.98011 - -2.21944 -2.21944 - 8.40757 8.40757 - -5.6418 -5.6418 - 3.3001 3.3001 - -0.678107 -0.678107 - -2.42254 -2.42254 - 0.439524 0.439524 - -0.417505 -0.417505 - -4.98938 -4.98938 - -6.34015 -6.34015 - -4.84203 -4.84203 - 2.86778 2.86778 - 3.29409 3.29409 - 2.59772 2.59772 - 5.20187 5.20187 - 3.55625 3.55625 - -7.065 -7.065 - -6.60792 -6.60792 - -3.20259 -3.20259 - 0.417062 0.417062 - -2.39846 -2.39846 - -5.762 -5.762 - 1.74843 1.74843 - 8.19239 8.19239 - -1.7349 -1.7349 - -0.0331415 -0.0331415 - 5.00712 5.00712 - 10.611 10.611 - 9.28817 9.28817 - -3.85324 -3.85324 - 2.29622 2.29622 - 10.962 10.962 - 4.44034 4.44034 - -3.2265 -3.2265 - 1.39326 1.39326 - -1.56539 -1.56539 - -8.78843 -8.78843 - -1.74101 -1.74101 - 8.51953 8.51953 - 3.31178 3.31178 - -1.20051 -1.20051 - -3.93224 -3.93224 - 2.4431 2.4431 - 3.69278 3.69278 - -10.2714 -10.2714 - -13.7579 -13.7579 - -1.76844 -1.76844 - -0.448193 -0.448193 - 1.48574 1.48574 - -0.831377 -0.831377 - 6.42657 6.42657 - 6.51848 6.51848 - 2.7764 2.7764 - 4.29448 4.29448 - -1.27173 -1.27173 - -7.14856 -7.14856 - 2.95751 2.95751 - 2.39789 2.39789 - 4.79429 4.79429 - 7.29216 7.29216 - -4.91502 -4.91502 - 2.38701 2.38701 - -2.34997 -2.34997 - -0.876115 -0.876115 - -0.672649 -0.672649 - 4.43884 4.43884 - 0.254258 0.254258 - -3.56471 -3.56471 - 0.161779 0.161779 - -10.1128 -10.1128 - 9.97279 9.97279 - -5.01498 -5.01498 - 1.10415 1.10415 - 1.37993 1.37993 - -3.32619 -3.32619 - 2.57257 2.57257 - -0.137478 -0.137478 - 1.49426 1.49426 - -0.805644 -0.805644 - 3.25356 3.25356 - 2.46332 2.46332 - 1.39266 1.39266 - 4.15167 4.15167 - -9.27164 -9.27164 - -2.29794 -2.29794 - 0.067971 0.067971 - 3.83697 3.83697 - 5.7385 5.7385 - -6.15176 -6.15176 - -4.08442 -4.08442 - -6.18563 -6.18563 - 6.44396 6.44396 - 5.63585 5.63585 - 1.21604 1.21604 - 11.1837 11.1837 - 2.29144 2.29144 - -0.995473 -0.995473 - 5.22826 5.22826 - 9.27205 9.27205 - -7.23457 -7.23457 - 6.29887 6.29887 - 2.48343 2.48343 - -4.96111 -4.96111 - -5.52811 -5.52811 - -4.40855 -4.40855 - -5.69429 -5.69429 - -1.12765 -1.12765 - -0.22142 -0.22142 - -5.96815 -5.96815 - 4.55923 4.55923 - -1.05719 -1.05719 - 2.07986 2.07986 - 7.77539 7.77539 - -2.03581 -2.03581 - 0.270705 0.270705 - 0.126658 0.126658 - -6.1672 -6.1672 - -16.0576 -16.0576 - 0.635198 0.635198 - 8.55006 8.55006 - -2.93081 -2.93081 - -1.7657 -1.7657 - -0.37886 -0.37886 - -2.4086 -2.4086 - 1.41889 1.41889 - -1.40539 -1.40539 - 0.963807 0.963807 - -2.14947 -2.14947 - -6.31832 -6.31832 - -4.30827 -4.30827 - 6.2609 6.2609 - -8.36351 -8.36351 - 4.28564 4.28564 - 0.646361 0.646361 - 4.60485 4.60485 - -3.1664 -3.1664 - -0.611618 -0.611618 - -9.53534 -9.53534 - 1.92275 1.92275 - -8.1521 -8.1521 - 0.101441 0.101441 - 0.399002 0.399002 - -2.04551 -2.04551 - -4.5564 -4.5564 - 3.0555 3.0555 - 0.992401 0.992401 - -5.62638 -5.62638 - -0.46873 -0.46873 - -6.86208 -6.86208 - -2.77108 -2.77108 - 3.51118 3.51118 - 0.885266 0.885266 - 3.65701 3.65701 - 6.88336 6.88336 - -7.25948 -7.25948 - 7.31435 7.31435 - -6.57357 -6.57357 - 3.67947 3.67947 - 4.80901 4.80901 - -2.80342 -2.80342 - 5.78724 5.78724 - 5.30985 5.30985 - 7.24724 7.24724 - -1.30439 -1.30439 - 2.50975 2.50975 - 5.28538 5.28538 - -3.91583 -3.91583 - 2.98722 2.98722 - 5.31167 5.31167 - -0.596966 -0.596966 - -4.94141 -4.94141 - 4.59005 4.59005 - 1.3813 1.3813 - 4.0611 4.0611 - -0.747616 -0.747616 - -3.1697 -3.1697 - -1.70787 -1.70787 - -2.43542 -2.43542 - -5.86823 -5.86823 - -10.9093 -10.9093 - 5.20087 5.20087 - -6.40378 -6.40378 - 1.5149 1.5149 - -6.52874 -6.52874 - -5.69743 -5.69743 - 1.06819 1.06819 - -7.31776 -7.31776 - 3.69649 3.69649 - -4.21319 -4.21319 - -4.91507 -4.91507 - 5.44776 5.44776 - -0.708927 -0.708927 - 1.94895 1.94895 - 2.90927 2.90927 - -2.82547 -2.82547 - -1.79858 -1.79858 - -15.6727 -15.6727 - -0.308918 -0.308918 - 2.61943 2.61943 - -3.89041 -3.89041 - -1.84684 -1.84684 - -6.80446 -6.80446 - 3.97398 3.97398 - 2.31201 2.31201 - 4.29417 4.29417 - -1.24479 -1.24479 - 4.25927 4.25927 - -1.96968 -1.96968 - 0.703519 0.703519 - 2.06517 2.06517 - 0.920347 0.920347 - 6.22843 6.22843 - 1.86167 1.86167 - 0.43407 0.43407 - 1.25225 1.25225 - -0.00512493 -0.00512493 - -1.70887 -1.70887 - 0.725693 0.725693 - 6.11604 6.11604 - -5.87059 -5.87059 - 3.26102 3.26102 - 2.0488 2.0488 - -0.0544172 -0.0544172 - 2.57295 2.57295 - -1.10578 -1.10578 - 2.43904 2.43904 - -2.34604 -2.34604 - 3.2098 3.2098 - 2.16089 2.16089 - -9.35001 -9.35001 - 9.43924 9.43924 - 0.916747 0.916747 - 2.59533 2.59533 - -1.84596 -1.84596 - 1.02889 1.02889 - 0.755944 0.755944 - 8.28274 8.28274 - -3.21136 -3.21136 - 1.24897 1.24897 - -0.363928 -0.363928 - 2.37533 2.37533 - -1.5794 -1.5794 - 6.67417 6.67417 - -4.4632 -4.4632 - 8.53731 8.53731 - -1.16526 -1.16526 - -0.51467 -0.51467 - -4.91688 -4.91688 - 7.17741 7.17741 - 4.61708 4.61708 - -2.41511 -2.41511 - -11.5234 -11.5234 - 2.61523 2.61523 - 4.7703 4.7703 - 6.72381 6.72381 - 5.65388 5.65388 - -4.23963 -4.23963 - 0.925176 0.925176 - 1.98862 1.98862 - -6.14466 -6.14466 - 2.76728 2.76728 - -0.83598 -0.83598 - -4.22593 -4.22593 - 5.99083 5.99083 - -4.886 -4.886 - 4.37801 4.37801 - 5.77761 5.77761 - 3.38352 3.38352 - -0.311291 -0.311291 - 8.26669 8.26669 - -4.94787 -4.94787 - -9.62034 -9.62034 - 2.37023 2.37023 - 3.41718 3.41718 - -2.43368 -2.43368 - 3.5898 3.5898 - -1.21973 -1.21973 - 0.0350305 0.0350305 - -4.33097 -4.33097 - -3.41432 -3.41432 - 2.59161 2.59161 - -2.11239 -2.11239 - -1.0801 -1.0801 - -3.27061 -3.27061 - -0.34025 -0.34025 - -6.40563 -6.40563 - -0.522305 -0.522305 - 4.63382 4.63382 - 1.5154 1.5154 - 0.968893 0.968893 - 2.79354 2.79354 - -0.829942 -0.829942 - -1.76388 -1.76388 - -6.64903 -6.64903 - -8.52588 -8.52588 - 2.70798 2.70798 - 6.78381 6.78381 - -5.67891 -5.67891 - -0.0588557 -0.0588557 - -4.12923 -4.12923 - -2.70431 -2.70431 - -0.12131 -0.12131 - 6.59494 6.59494 - 0.830427 0.830427 - 3.40436 3.40436 - 6.98828 6.98828 - -2.33332 -2.33332 - 5.85244 5.85244 - -10.0398 -10.0398 - -0.242519 -0.242519 - -3.38719 -3.38719 - 2.74288 2.74288 - 3.82961 3.82961 - -6.85166 -6.85166 - -0.345431 -0.345431 - -3.03082 -3.03082 - 1.68089 1.68089 - -0.785036 -0.785036 - -2.92804 -2.92804 - 1.03727 1.03727 - 5.51647 5.51647 - -2.15538 -2.15538 - -6.20918 -6.20918 - -0.986195 -0.986195 - -4.4207 -4.4207 - -0.314791 -0.314791 - -6.64843 -6.64843 - 1.255 1.255 - 4.39107 4.39107 - 2.20706 2.20706 - -1.894 -1.894 - -3.01471 -3.01471 - -0.0623641 -0.0623641 - -5.76316 -5.76316 - -2.45987 -2.45987 - -2.09262 -2.09262 - 0.0458748 0.0458748 - 5.09539 5.09539 - -3.80431 -3.80431 - -3.90738 -3.90738 - -6.48843 -6.48843 - -2.58373 -2.58373 - -6.38764 -6.38764 - 7.38858 7.38858 - 0.492176 0.492176 - 8.79347 8.79347 - 2.04442 2.04442 - -0.216083 -0.216083 - 11.3375 11.3375 - -3.4177 -3.4177 - 3.90111 3.90111 - 4.92081 4.92081 - 4.45964 4.45964 - 11.1458 11.1458 - -2.2688 -2.2688 - -4.43463 -4.43463 - -4.22186 -4.22186 - -5.93987 -5.93987 - 3.4437 3.4437 - -5.60816 -5.60816 - -8.04401 -8.04401 - -4.95256 -4.95256 - 3.88283 3.88283 - -0.173935 -0.173935 - -2.63243 -2.63243 - -1.03812 -1.03812 - -9.14078 -9.14078 - -6.1411 -6.1411 - 3.4284 3.4284 - -9.8305 -9.8305 - 6.76115 6.76115 - -11.3646 -11.3646 - -5.7296 -5.7296 - -2.41831 -2.41831 - -5.21505 -5.21505 - 10.4347 10.4347 - 2.06721 2.06721 - 1.02265 1.02265 - -6.93537 -6.93537 - 1.28707 1.28707 - 0.939615 0.939615 - 11.262 11.262 - 1.2805 1.2805 - 4.8619 4.8619 - 3.15836 3.15836 - -5.18747 -5.18747 - -2.98078 -2.98078 - -2.0489 -2.0489 - -2.85634 -2.85634 - -4.56059 -4.56059 - -4.0715 -4.0715 - 0.469543 0.469543 - -2.05188 -2.05188 - -2.79567 -2.79567 - 3.82027 3.82027 - 2.55175 2.55175 - -0.468207 -0.468207 - -5.65994 -5.65994 - 2.13508 2.13508 - -3.17019 -3.17019 - 6.53032 6.53032 - -4.98714 -4.98714 - -1.94956 -1.94956 - -3.08465 -3.08465 - 8.11664 8.11664 - 8.86283 8.86283 - 0.84108 0.84108 - 5.22353 5.22353 - -3.45671 -3.45671 - -1.38725 -1.38725 - 1.35206 1.35206 - -10.4407 -10.4407 - -2.20051 -2.20051 - -0.228019 -0.228019 - -1.38039 -1.38039 - 11.1342 11.1342 - 5.17568 5.17568 - -4.54852 -4.54852 - -1.26392 -1.26392 - 5.69792 5.69792 - -4.90866 -4.90866 - 2.84526 2.84526 - 10.9699 10.9699 - 12.9756 12.9756 - 8.48223 8.48223 - 2.11902 2.11902 - 3.74471 3.74471 - -5.14437 -5.14437 - -14.7206 -14.7206 - 3.01028 3.01028 - -2.67988 -2.67988 - -2.88296 -2.88296 - -4.95895 -4.95895 - -1.82286 -1.82286 - 5.23419 5.23419 - -2.23867 -2.23867 - 0.610838 0.610838 - 2.09177 2.09177 - 5.74677 5.74677 - 3.6242 3.6242 - 2.0758 2.0758 - -2.85159 -2.85159 - -3.93562 -3.93562 - 3.85649 3.85649 - -5.75638 -5.75638 - -7.07444 -7.07444 - 0.907402 0.907402 - -8.92532 -8.92532 - -4.09782 -4.09782 - 1.85777 1.85777 - 5.73041 5.73041 - -2.17118 -2.17118 - -3.4713 -3.4713 - 7.95825 7.95825 - 9.10838 9.10838 - 1.80182 1.80182 - -0.54593 -0.54593 - -4.89919 -4.89919 - -2.97982 -2.97982 - 0.807424 0.807424 - -2.27 -2.27 - -13.2338 -13.2338 - -3.94367 -3.94367 - -5.72938 -5.72938 - -2.42243 -2.42243 - -3.69581 -3.69581 - -4.71307 -4.71307 - 1.38983 1.38983 - -5.37869 -5.37869 - -6.82815 -6.82815 - 2.73203 2.73203 - 13.6495 13.6495 - -6.29731 -6.29731 - -8.43712 -8.43712 - 14.1567 14.1567 - -0.978804 -0.978804 - 1.26264 1.26264 - -9.25575 -9.25575 - -8.10968 -8.10968 - -3.98015 -3.98015 - 6.60273 6.60273 - -3.98373 -3.98373 - 1.35817 1.35817 - 1.20988 1.20988 - 1.53069 1.53069 - 4.08368 4.08368 - -2.38429 -2.38429 - -4.67381 -4.67381 - -5.49726 -5.49726 - 0.657715 0.657715 - -0.00123905 -0.00123905 - 4.62712 4.62712 - -0.317445 -0.317445 - -5.08829 -5.08829 - -9.85674 -9.85674 - 5.31787 5.31787 - 1.61793 1.61793 - 3.9901 3.9901 - -1.04243 -1.04243 - -3.73679 -3.73679 - 0.670282 0.670282 - 9.03148 9.03148 - -4.77058 -4.77058 - 8.60147 8.60147 - -0.664744 -0.664744 - 1.97711 1.97711 - -5.35794 -5.35794 - -9.70033 -9.70033 - 10.7781 10.7781 - 1.96443 1.96443 - 1.84069 1.84069 - -12.0109 -12.0109 - 2.08404 2.08404 - 3.64031 3.64031 - 8.65585 8.65585 - -11.8355 -11.8355 - 9.89404 9.89404 - 0.279063 0.279063 - -0.315296 -0.315296 - 3.74263 3.74263 - 6.54645 6.54645 - 5.43941 5.43941 - 4.83252 4.83252 - 1.70716 1.70716 - -3.27497 -3.27497 - -3.07764 -3.07764 - 9.25309 9.25309 - -1.69559 -1.69559 - 10.1694 10.1694 - -3.42523 -3.42523 - 6.39435 6.39435 - 2.18084 2.18084 - 1.33177 1.33177 - -0.709393 -0.709393 - 1.44799 1.44799 - 0.881759 0.881759 - -2.35085 -2.35085 - -1.91407 -1.91407 - 0.302603 0.302603 - 1.40288 1.40288 - -2.37323 -2.37323 - -7.74084 -7.74084 - -7.73224 -7.73224 - 2.8793 2.8793 - 6.62065 6.62065 - 1.4654 1.4654 - -0.982735 -0.982735 - -0.97328 -0.97328 - -8.38882 -8.38882 - 8.74643 8.74643 - -7.86996 -7.86996 - 3.25655 3.25655 - 2.78551 2.78551 - -5.17511 -5.17511 - 4.90515 4.90515 - 0.28899 0.28899 - 3.57292 3.57292 - -5.25376 -5.25376 - -8.57274 -8.57274 - -1.18267 -1.18267 - 37.4072 37.4072 - -4.00801 -4.00801 - 4.8073 4.8073 - -4.45001 -4.45001 - 7.66024 7.66024 - -4.47725 -4.47725 - -10.2209 -10.2209 - -4.80026 -4.80026 - -0.64446 -0.64446 - -0.899171 -0.899171 - 1.09833 1.09833 - -0.988097 -0.988097 - 2.82126 2.82126 - -8.19269 -8.19269 - -2.64922 -2.64922 - -9.16004 -9.16004 - -2.39588 -2.39588 - -4.72025 -4.72025 - 2.34077 2.34077 - 3.83879 3.83879 - 1.9499 1.9499 - -0.361603 -0.361603 - 7.79929 7.79929 - 2.34774 2.34774 - -8.21052 -8.21052 - -2.02077 -2.02077 - -1.58017 -1.58017 - -0.410542 -0.410542 - -10.7206 -10.7206 - 3.26874 3.26874 - 2.80972 2.80972 - 0.0906836 0.0906836 - -1.64773 -1.64773 - 6.49353 6.49353 - -0.791109 -0.791109 - 4.71404 4.71404 - 0.0741314 0.0741314 - -0.414415 -0.414415 - 6.84572 6.84572 - -0.367457 -0.367457 - 1.17563 1.17563 - 0.51039 0.51039 - 4.40348 4.40348 - 0.978932 0.978932 - 3.79206 3.79206 - 4.57632 4.57632 - 2.77883 2.77883 - 0.490867 0.490867 - -0.151798 -0.151798 - 6.72243 6.72243 - 4.77773 4.77773 - -0.50633 -0.50633 - -8.08639 -8.08639 - 4.88619 4.88619 - -2.07669 -2.07669 - -2.24093 -2.24093 - 1.72994 1.72994 - -7.45157 -7.45157 - -12.1192 -12.1192 - 1.4328 1.4328 - -8.14432 -8.14432 - -6.25485 -6.25485 - 0.516865 0.516865 - 7.11864 7.11864 - -0.616318 -0.616318 - -0.761916 -0.761916 - -5.99496 -5.99496 - 10.4321 10.4321 - -0.516052 -0.516052 - -5.68287 -5.68287 - -4.15541 -4.15541 - 1.56619 1.56619 - -20.8292 -20.8292 - 0.788033 0.788033 - 3.34264 3.34264 - 3.70493 3.70493 - -0.0822138 -0.0822138 - 2.31304 2.31304 - -1.69352 -1.69352 - 2.10396 2.10396 - 7.2613 7.2613 - -1.81799 -1.81799 - -2.09968 -2.09968 - -3.8336 -3.8336 - -3.93478 -3.93478 - 3.3059 3.3059 - 4.19189 4.19189 - -1.93794 -1.93794 - 2.7117 2.7117 - 9.43261 9.43261 - -1.83318 -1.83318 - -1.12685 -1.12685 - 2.40725 2.40725 - 7.50947 7.50947 - 7.65688 7.65688 - -5.02792 -5.02792 - -2.55777 -2.55777 - -1.9946 -1.9946 - -0.126192 -0.126192 - -3.30905 -3.30905 - -0.209775 -0.209775 - 9.06409 9.06409 - -3.79201 -3.79201 - 8.80185 8.80185 - -1.59367 -1.59367 - -2.49213 -2.49213 - -3.5242 -3.5242 - 2.4892 2.4892 - 5.68222 5.68222 - 4.29073 4.29073 - 0.490494 0.490494 - 3.31313 3.31313 - 8.27344 8.27344 - 1.44936 1.44936 - 5.94283 5.94283 - -5.90497 -5.90497 - 0.316931 0.316931 - 1.93975 1.93975 - -1.33405 -1.33405 - -4.17957 -4.17957 - 2.45999 2.45999 - -10.0965 -10.0965 - 0.648564 0.648564 - -0.745957 -0.745957 - -6.08922 -6.08922 - -6.45851 -6.45851 - 2.70093 2.70093 - -2.59331 -2.59331 - -2.73319 -2.73319 - -6.50584 -6.50584 - 4.14167 4.14167 - 6.78757 6.78757 - 4.63335 4.63335 - 2.01754 2.01754 - 3.97717 3.97717 - 2.73775 2.73775 - 2.04299 2.04299 - 7.03044 7.03044 - -8.59414 -8.59414 - -4.19956 -4.19956 - 0.0135157 0.0135157 - -5.45393 -5.45393 - 2.75578 2.75578 - 0.730278 0.730278 - -0.410035 -0.410035 - 10.7831 10.7831 - -2.82537 -2.82537 - 1.85601 1.85601 - 1.68496 1.68496 - 2.75249 2.75249 - 9.40848 9.40848 - 1.6032 1.6032 - -3.91263 -3.91263 - 1.12247 1.12247 - -3.46516 -3.46516 - -1.48668 -1.48668 - 6.7676 6.7676 - -5.76927 -5.76927 - -2.19943 -2.19943 - -1.61329 -1.61329 - 3.35791 3.35791 - -7.80737 -7.80737 - 3.06567 3.06567 - -12.2037 -12.2037 - 12.541 12.541 - 4.42316 4.42316 - 6.48419 6.48419 - 1.17664 1.17664 - 2.97986 2.97986 - -8.63966 -8.63966 - 0.241757 0.241757 - -5.03654 -5.03654 - -1.94594 -1.94594 - 12.8093 12.8093 - -3.58644 -3.58644 - -3.35952 -3.35952 - -0.864134 -0.864134 - -12.4807 -12.4807 - -1.69909 -1.69909 - -5.67676 -5.67676 - -10.6435 -10.6435 - -3.86815 -3.86815 - 4.20674 4.20674 - -4.94992 -4.94992 - 7.63289 7.63289 - -5.5226 -5.5226 - 1.58362 1.58362 - 1.14864 1.14864 - 5.98635 5.98635 - 11.9692 11.9692 - -0.208588 -0.208588 - -0.177219 -0.177219 - 6.35143 6.35143 - -2.21028 -2.21028 - 0.693657 0.693657 - 2.66882 2.66882 - -0.494413 -0.494413 - 10.9482 10.9482 - 2.9522 2.9522 - 1.69427 1.69427 - -5.54007 -5.54007 - -1.44208 -1.44208 - -2.75377 -2.75377 - 7.62773 7.62773 - -0.0991657 -0.0991657 - 0.541024 0.541024 - 0.383422 0.383422 - -6.28538 -6.28538 - -3.63239 -3.63239 - 5.54891 5.54891 - 4.38377 4.38377 - -4.21607 -4.21607 - -1.58462 -1.58462 - 1.99568 1.99568 - 1.70177 1.70177 - 1.65142 1.65142 - 1.79811 1.79811 - -6.82605 -6.82605 - 3.65159 3.65159 - 2.60935 2.60935 - -2.91237 -2.91237 - -1.56808 -1.56808 - -3.07334 -3.07334 - 0.883426 0.883426 - -1.59697 -1.59697 - 4.44432 4.44432 - -2.72255 -2.72255 - -0.853149 -0.853149 - -0.132598 -0.132598 - -0.63629 -0.63629 - -3.69308 -3.69308 - -7.18449 -7.18449 - 1.20547 1.20547 - 14.3427 14.3427 - 5.08288 5.08288 - 0.957041 0.957041 - 0.153537 0.153537 - -7.14906 -7.14906 - -8.78572 -8.78572 - 4.05049 4.05049 - 3.22929 3.22929 - -3.34601 -3.34601 - 3.86442 3.86442 - -2.80641 -2.80641 - 6.51055 6.51055 - -4.58706 -4.58706 - -1.51146 -1.51146 - 3.88212 3.88212 - 1.89549 1.89549 - 3.50062 3.50062 - -1.43005 -1.43005 - -2.91969 -2.91969 - -6.52573 -6.52573 - -3.8843 -3.8843 - -8.34716 -8.34716 - -7.42192 -7.42192 - -3.98985 -3.98985 - 15.526 15.526 - 8.70318 8.70318 - -1.10105 -1.10105 - 2.14694 2.14694 - 7.71484 7.71484 - -0.0260442 -0.0260442 - -3.31138 -3.31138 - 1.67906 1.67906 - -0.083112 -0.083112 - -8.42905 -8.42905 - -8.82729 -8.82729 - 11.2859 11.2859 - -8.07136 -8.07136 - -3.9371 -3.9371 - -4.63176 -4.63176 - -1.23605 -1.23605 - -2.08565 -2.08565 - 1.93918 1.93918 - -12.5031 -12.5031 - -0.442281 -0.442281 - -5.50289 -5.50289 - -0.815112 -0.815112 - 0.0898735 0.0898735 - 4.69373 4.69373 - -7.22004 -7.22004 - 0.543294 0.543294 - 4.2932 4.2932 - 2.12984 2.12984 - -4.42752 -4.42752 - 3.03694 3.03694 - -3.73337 -3.73337 - -12.0483 -12.0483 - -5.99704 -5.99704 - 0.0707967 0.0707967 - -4.52239 -4.52239 - 3.65625 3.65625 - -5.61903 -5.61903 - 9.78971 9.78971 - 8.47575 8.47575 - -0.320966 -0.320966 - -7.10339 -7.10339 - 0.485669 0.485669 - 3.19439 3.19439 - -0.411976 -0.411976 - -0.782875 -0.782875 - 16.4086 16.4086 - -2.67312 -2.67312 - 0.73424 0.73424 - 8.32014 8.32014 - -1.24665 -1.24665 - 3.70031 3.70031 - -6.22155 -6.22155 - -6.34804 -6.34804 - -4.84631 -4.84631 - 7.19111 7.19111 - 2.58937 2.58937 - 2.12044 2.12044 - -0.304369 -0.304369 - -11.5161 -11.5161 - -4.75933 -4.75933 - -5.40287 -5.40287 - -14.7511 -14.7511 - -11.3269 -11.3269 - -3.40961 -3.40961 - -8.36998 -8.36998 - -7.86816 -7.86816 - 3.46638 3.46638 - 5.10745 5.10745 - 9.12589 9.12589 - 4.53119 4.53119 - -0.0952322 -0.0952322 - -1.67069 -1.67069 - 1.48937 1.48937 - 2.1548 2.1548 - -0.680895 -0.680895 - 6.00943 6.00943 - -6.23597 -6.23597 - 15.2635 15.2635 - -5.39621 -5.39621 - 2.9004 2.9004 - -7.2031 -7.2031 - 0.188095 0.188095 - -5.65511 -5.65511 - 8.80472 8.80472 - 4.77116 4.77116 - -0.320718 -0.320718 - -0.094774 -0.094774 - 4.24892 4.24892 - -0.729715 -0.729715 - 3.46906 3.46906 - -4.86913 -4.86913 - -2.05092 -2.05092 - 3.24008 3.24008 - 2.67334 2.67334 - 5.41008 5.41008 - 4.61387 4.61387 - -11.9338 -11.9338 - 2.15538 2.15538 - 3.39914 3.39914 - 2.71216 2.71216 - 6.79031 6.79031 - -0.750493 -0.750493 - -0.683416 -0.683416 - 7.23875 7.23875 - 4.67949 4.67949 - -2.16467 -2.16467 - 3.64787 3.64787 - -1.27823 -1.27823 - -1.43992 -1.43992 - 3.183 3.183 - -8.60412 -8.60412 - -5.42757 -5.42757 - -0.564214 -0.564214 - -1.17837 -1.17837 - 2.45248 2.45248 - 3.60909 3.60909 - 2.61183 2.61183 - 5.20279 5.20279 - -1.07145 -1.07145 - -0.919519 -0.919519 - 3.89898 3.89898 - 3.72175 3.72175 - -9.9673 -9.9673 - 1.50607 1.50607 - -0.456562 -0.456562 - 10.9984 10.9984 - -2.18673 -2.18673 - -7.39159 -7.39159 - -5.54389 -5.54389 - 2.6353 2.6353 - 6.87535 6.87535 - -10.4019 -10.4019 - -5.51375 -5.51375 - -3.33244 -3.33244 - 7.60358 7.60358 - -9.48529 -9.48529 - -0.514099 -0.514099 - 6.20569 6.20569 - -4.60198 -4.60198 - -1.28686 -1.28686 - -0.383981 -0.383981 - -0.173934 -0.173934 - -7.97782 -7.97782 - 5.9926 5.9926 - -3.7357 -3.7357 - -7.77841 -7.77841 - 3.09245 3.09245 - -3.70421 -3.70421 - -1.50012 -1.50012 - -3.90181 -3.90181 - 0.183002 0.183002 - -4.72374 -4.72374 - -3.36966 -3.36966 - 8.23642 8.23642 - 0.387898 0.387898 - -2.53048 -2.53048 - 4.46348 4.46348 - -0.932844 -0.932844 - -1.76804 -1.76804 - -0.390175 -0.390175 - 8.28101 8.28101 - 8.66959 8.66959 - 2.47585 2.47585 - 6.33837 6.33837 - 3.05846 3.05846 - 6.43047 6.43047 - 0.167477 0.167477 - 0.615034 0.615034 - -8.467 -8.467 - 2.15566 2.15566 - 6.59172 6.59172 - -8.30068 -8.30068 - -2.92268 -2.92268 - -1.14616 -1.14616 - 3.864 3.864 - -8.07267 -8.07267 - 0.382952 0.382952 - 4.79087 4.79087 - 7.87692 7.87692 - -1.27352 -1.27352 - -0.439992 -0.439992 - -0.361056 -0.361056 - 5.51463 5.51463 - 4.10827 4.10827 - -1.36056 -1.36056 - -10.9063 -10.9063 - -3.12566 -3.12566 - -1.52612 -1.52612 - 2.47429 2.47429 - 1.92973 1.92973 - 6.05399 6.05399 - 6.35717 6.35717 - -6.54112 -6.54112 - 0.16752 0.16752 - -0.581192 -0.581192 - -3.91981 -3.91981 - 3.29046 3.29046 - -9.85289 -9.85289 - -1.68008 -1.68008 - -0.294261 -0.294261 - -2.33446 -2.33446 - 8.72203 8.72203 - -7.53754 -7.53754 - 1.8548 1.8548 - 0.0863562 0.0863562 - 3.71224 3.71224 - -2.72156 -2.72156 - 6.92717 6.92717 - 4.22066 4.22066 - 2.9384 2.9384 - -0.436476 -0.436476 - 7.94505 7.94505 - 3.35167 3.35167 - 4.57606 4.57606 - -1.94551 -1.94551 - 7.26891 7.26891 - 5.7114 5.7114 - -4.8975 -4.8975 - 0.24802 0.24802 - 4.4272 4.4272 - 3.21714 3.21714 - -2.75997 -2.75997 - 3.0239 3.0239 - 6.00743 6.00743 - 1.95157 1.95157 - -8.23524 -8.23524 - -0.0388194 -0.0388194 - -1.59723 -1.59723 - -15.7227 -15.7227 - 5.01363 5.01363 - 2.59661 2.59661 - 0.344503 0.344503 - 7.85727 7.85727 - 0.142462 0.142462 - -3.54743 -3.54743 - -4.18558 -4.18558 - 3.96172 3.96172 - -0.376684 -0.376684 - 3.78763 3.78763 - -1.58384 -1.58384 - 15.837 15.837 - -0.887404 -0.887404 - 0.855016 0.855016 - 11.1701 11.1701 - 5.15206 5.15206 - 6.83176 6.83176 - -0.91331 -0.91331 - -10.3398 -10.3398 - 2.48231 2.48231 - -2.03572 -2.03572 - 1.09096 1.09096 - -0.162198 -0.162198 - -7.32758 -7.32758 - -6.97941 -6.97941 - 5.98831 5.98831 - -7.43703 -7.43703 - -8.97936 -8.97936 - 0.676949 0.676949 - 1.37291 1.37291 - 4.41159 4.41159 - 2.45643 2.45643 - 2.79374 2.79374 - 2.36712 2.36712 - -7.74483 -7.74483 - 0.602922 0.602922 - -2.48544 -2.48544 - 0.299035 0.299035 - 6.77695 6.77695 - 1.44763 1.44763 + -2.45508 -2.45508 + -2.46738 -2.46738 + 0.850981 0.850981 + 0.709954 0.709954 + 0.806159 0.806159 + 1.41638 1.41638 + 1.09515 1.09515 + -1.77543 -1.77543 + 1.16811 1.16811 + 1.68691 1.68691 + -1.03008 -1.03008 + -1.27389 -1.27389 + 1.749 1.749 + 1.94468 1.94468 + 1.3547 1.3547 + 1.37603 1.37603 + -0.743637 -0.743637 + 0.805151 0.805151 + -0.108866 -0.108866 + 0.836891 0.836891 + 2.84637 2.84637 + -1.33009 -1.33009 + -1.16621 -1.16621 + -1.64225 -1.64225 + -3.23381 -3.23381 + 1.87149 1.87149 + 1.62256 1.62256 + -1.92032 -1.92032 + 1.92751 1.92751 + -0.596136 -0.596136 + 2.67425 2.67425 + -0.0169855 -0.0169855 + -0.74862 -0.74862 + 1.38466 1.38466 + -1.43319 -1.43319 + -0.0358144 -0.0358144 + -2.61001 -2.61001 + 0.979845 0.979845 + -2.50828 -2.50828 + -0.19873 -0.19873 + 1.56007 1.56007 + -2.9497 -2.9497 + 1.18661 1.18661 + -0.0459045 -0.0459045 + -1.89088 -1.89088 + 0.134144 0.134144 + 1.46773 1.46773 + 0.169044 0.169044 + 1.83185 1.83185 + 1.29546 1.29546 + -0.223334 -0.223334 + 0.248241 0.248241 + -0.143776 -0.143776 + -1.26674 -1.26674 + -2.34766 -2.34766 + -0.233582 -0.233582 + -1.32809 -1.32809 + -3.1517 -3.1517 + -1.37292 -1.37292 + -0.296508 -0.296508 + -0.730617 -0.730617 + 1.33757 1.33757 + -0.0650454 -0.0650454 + -1.69394 -1.69394 + -0.634744 -0.634744 + -1.43693 -1.43693 + 1.04292 1.04292 + 3.70178 3.70178 + 1.58817 1.58817 + -1.71241 -1.71241 + 1.28312 1.28312 + 1.28754 1.28754 + 0.079447 0.079447 + 0.0501985 0.0501985 + -2.16351 -2.16351 + 0.157908 0.157908 + 1.63121 1.63121 + -0.755454 -0.755454 + -0.490053 -0.490053 + -0.205053 -0.205053 + -2.08433 -2.08433 + 0.724874 0.724874 + 0.0719429 0.0719429 + -1.17219 -1.17219 + 0.440574 0.440574 + -0.939899 -0.939899 + -0.539892 -0.539892 + -0.378413 -0.378413 + 0.233923 0.233923 + 1.76957 1.76957 + -3.07904 -3.07904 + -2.3768 -2.3768 + -2.55772 -2.55772 + -1.41162 -1.41162 + 0.0823545 0.0823545 + 1.55678 1.55678 + 0.0568478 0.0568478 + 0.840122 0.840122 + 1.06037 1.06037 + 0.965553 0.965553 + 0.830964 0.830964 + -0.754283 -0.754283 + -0.1191 -0.1191 + 0.0280245 0.0280245 + -0.143816 -0.143816 + 1.27753 1.27753 + 0.066933 0.066933 + 0.926294 0.926294 + 0.743463 0.743463 + -0.559796 -0.559796 + 0.901292 0.901292 + 0.457328 0.457328 + -0.365485 -0.365485 + 0.218589 0.218589 + -1.26814 -1.26814 + -1.61122 -1.61122 + 1.82385 1.82385 + -0.722799 -0.722799 + -0.744419 -0.744419 + -0.369159 -0.369159 + -0.0910514 -0.0910514 + 1.38917 1.38917 + -0.176995 -0.176995 + -0.737644 -0.737644 + -1.51984 -1.51984 + 1.43837 1.43837 + -2.17809 -2.17809 + 1.78125 1.78125 + 1.61911 1.61911 + 2.56386 2.56386 + -0.265214 -0.265214 + -0.0391067 -0.0391067 + -1.15711 -1.15711 + 1.87313 1.87313 + -1.92623 -1.92623 + 3.0698 3.0698 + 3.35805 3.35805 + 0.843137 0.843137 + 2.88871 2.88871 + -0.309429 -0.309429 + 0.251267 0.251267 + 0.350675 0.350675 + -0.0698121 -0.0698121 + -2.383 -2.383 + 0.955251 0.955251 + 0.219921 0.219921 + -3.57961 -3.57961 + 0.809841 0.809841 + -0.171057 -0.171057 + 2.60038 2.60038 + 0.123726 0.123726 + 0.541886 0.541886 + -0.67232 -0.67232 + 1.65081 1.65081 + 1.2956 1.2956 + -0.936224 -0.936224 + -1.43537 -1.43537 + 1.30422 1.30422 + 1.645 1.645 + -0.59586 -0.59586 + -2.30588 -2.30588 + 0.265308 0.265308 + -1.16852 -1.16852 + 0.572658 0.572658 + -2.48747 -2.48747 + 1.75643 1.75643 + -0.208687 -0.208687 + -0.589351 -0.589351 + 1.38263 1.38263 + 0.0407013 0.0407013 + -0.478184 -0.478184 + 1.1551 1.1551 + 2.66743 2.66743 + -0.0347208 -0.0347208 + 0.258393 0.258393 + 0.376756 0.376756 + -1.91087 -1.91087 + 1.58246 1.58246 + -2.55877 -2.55877 + 0.29497 0.29497 + -0.679837 -0.679837 + 2.75608 2.75608 + 0.846178 0.846178 + 0.120587 0.120587 + -0.931332 -0.931332 + -0.460664 -0.460664 + -1.41767 -1.41767 + -0.0370647 -0.0370647 + 2.85593 2.85593 + 0.93859 0.93859 + 1.06093 1.06093 + -0.806263 -0.806263 + -0.276273 -0.276273 + 0.190741 0.190741 + 0.642197 0.642197 + -0.146854 -0.146854 + 0.235867 0.235867 + 1.53559 1.53559 + 3.23839 3.23839 + 2.51139 2.51139 + 2.69216 2.69216 + -0.808452 -0.808452 + 0.131485 0.131485 + 1.79253 1.79253 + -1.489 -1.489 + 1.36115 1.36115 + 0.689342 0.689342 + 1.93437 1.93437 + 1.1515 1.1515 + -0.267202 -0.267202 + -2.93755 -2.93755 + -0.353072 -0.353072 + 0.27407 0.27407 + -1.99885 -1.99885 + 2.30033 2.30033 + -2.08776 -2.08776 + -5.84525 -5.84525 + -0.163065 -0.163065 + -1.84699 -1.84699 + 0.272231 0.272231 + 0.178446 0.178446 + -2.07568 -2.07568 + 1.64194 1.64194 + 2.61154 2.61154 + 2.61646 2.61646 + 1.49662 1.49662 + -0.762681 -0.762681 + -1.58585 -1.58585 + -2.56334 -2.56334 + 2.90547 2.90547 + 0.907315 0.907315 + -4.51614 -4.51614 + -1.84281 -1.84281 + -0.770806 -0.770806 + -4.84157 -4.84157 + 1.12599 1.12599 + -2.35923 -2.35923 + -1.65892 -1.65892 + -1.57506 -1.57506 + 2.73717 2.73717 + -2.01069 -2.01069 + 2.1204 2.1204 + -1.01274 -1.01274 + -1.54509 -1.54509 + -1.11102 -1.11102 + 2.27304 2.27304 + -1.7203 -1.7203 + -0.141955 -0.141955 + -1.93008 -1.93008 + 4.40994 4.40994 + -0.843562 -0.843562 + 1.8763 1.8763 + -1.15709 -1.15709 + 1.93146 1.93146 + 0.0839072 0.0839072 + -1.57024 -1.57024 + -0.373742 -0.373742 + -3.81842 -3.81842 + -0.654545 -0.654545 + -0.555125 -0.555125 + 2.62493 2.62493 + -0.299043 -0.299043 + 0.572533 0.572533 + 0.93598 0.93598 + -0.831565 -0.831565 + -0.324564 -0.324564 + 0.938522 0.938522 + 1.63603 1.63603 + 1.73752 1.73752 + 3.90209 3.90209 + 2.96495 2.96495 + -1.92878 -1.92878 + -0.620347 -0.620347 + -0.617516 -0.617516 + 0.0554883 0.0554883 + 1.50602 1.50602 + 3.24908 3.24908 + -4.00377 -4.00377 + 0.611538 0.611538 + 0.395783 0.395783 + 0.840638 0.840638 + 0.6627 0.6627 + -1.06035 -1.06035 + 1.23494 1.23494 + -1.09427 -1.09427 + -0.944445 -0.944445 + -0.149559 -0.149559 + 0.364951 0.364951 + -2.74365 -2.74365 + -1.70194 -1.70194 + -0.597655 -0.597655 + 0.10956 0.10956 + 0.2841 0.2841 + -1.70202 -1.70202 + 0.815583 0.815583 + -1.3173 -1.3173 + -0.350691 -0.350691 + 0.323236 0.323236 + 0.423569 0.423569 + 0.135633 0.135633 + -0.894987 -0.894987 + 3.82014 3.82014 + -1.37686 -1.37686 + -1.11328 -1.11328 + -1.42841 -1.42841 + 0.486355 0.486355 + 1.00883 1.00883 + -2.19186 -2.19186 + 1.60446 1.60446 + 1.44481 1.44481 + 0.50944 0.50944 + 2.35551 2.35551 + -1.09942 -1.09942 + 0.449232 0.449232 + -2.47599 -2.47599 + 1.4625 1.4625 + -0.60117 -0.60117 + -4.33734 -4.33734 + 1.09776 1.09776 + -1.773 -1.773 + 0.664505 0.664505 + 0.358341 0.358341 + -2.10807 -2.10807 + -0.43586 -0.43586 + 1.39689 1.39689 + 0.276272 0.276272 + -0.500288 -0.500288 + -1.5217 -1.5217 + 0.054699 0.054699 + 1.1783 1.1783 + 0.276075 0.276075 + -3.61916 -3.61916 + 1.01459 1.01459 + 0.817065 0.817065 + 2.77712 2.77712 + 0.911377 0.911377 + -3.64983 -3.64983 + 1.20056 1.20056 + 2.43472 2.43472 + -1.40853 -1.40853 + -0.398067 -0.398067 + 0.44781 0.44781 + -0.983103 -0.983103 + -1.01117 -1.01117 + -1.36821 -1.36821 + -1.62301 -1.62301 + 1.19024 1.19024 + 0.176577 0.176577 + -0.0266686 -0.0266686 + 0.173558 0.173558 + 1.28662 1.28662 + 0.80163 0.80163 + 0.89478 0.89478 + 0.165779 0.165779 + 0.201296 0.201296 + 3.06203 3.06203 + -0.799754 -0.799754 + 0.654919 0.654919 + -0.175649 -0.175649 + 0.748995 0.748995 + -0.155388 -0.155388 + 1.37946 1.37946 + 0.286727 0.286727 + -1.56082 -1.56082 + -4.17062 -4.17062 + 1.82237 1.82237 + 1.25286 1.25286 + 0.911904 0.911904 + 0.203066 0.203066 + 2.03452 2.03452 + -0.683902 -0.683902 + -1.83145 -1.83145 + 3.99144 3.99144 + -1.61397 -1.61397 + 0.525864 0.525864 + -0.0760927 -0.0760927 + -1.51654 -1.51654 + -1.26449 -1.26449 + -1.62594 -1.62594 + -0.0456586 -0.0456586 + 1.15446 1.15446 + -4.01131 -4.01131 + -1.69864 -1.69864 + -0.626423 -0.626423 + -0.768044 -0.768044 + 1.42236 1.42236 + 0.590386 0.590386 + -0.734404 -0.734404 + -0.991776 -0.991776 + 0.846241 0.846241 + -1.58825 -1.58825 + 1.64396 1.64396 + 0.506507 0.506507 + 2.04938 2.04938 + -0.142975 -0.142975 + -0.248108 -0.248108 + -0.395868 -0.395868 + -1.09874 -1.09874 + 3.86285 3.86285 + 0.401966 0.401966 + -1.78 -1.78 + -0.142962 -0.142962 + -1.40281 -1.40281 + 1.00746 1.00746 + -4.80569 -4.80569 + -2.78505 -2.78505 + -1.13022 -1.13022 + 1.64658 1.64658 + 3.87433 3.87433 + -0.490487 -0.490487 + -1.87168 -1.87168 + 0.324756 0.324756 + -0.887698 -0.887698 + 0.527319 0.527319 + -0.819513 -0.819513 + 0.66582 0.66582 + -1.01098 -1.01098 + -1.77557 -1.77557 + -1.86591 -1.86591 + -1.66033 -1.66033 + -2.26619 -2.26619 + 0.396155 0.396155 + -1.53582 -1.53582 + -2.12857 -2.12857 + 0.127346 0.127346 + 0.198356 0.198356 + -2.24282 -2.24282 + -0.431639 -0.431639 + -0.529176 -0.529176 + -1.24013 -1.24013 + -0.00600927 -0.00600927 + -0.970315 -0.970315 + 0.825078 0.825078 + 0.158507 0.158507 + 0.126577 0.126577 + -2.27452 -2.27452 + 0.43308 0.43308 + -2.68747 -2.68747 + -0.63088 -0.63088 + -0.300087 -0.300087 + 1.62468 1.62468 + -2.08314 -2.08314 + 0.537432 0.537432 + -1.51959 -1.51959 + -1.36428 -1.36428 + -1.37559 -1.37559 + -0.270443 -0.270443 + 1.92657 1.92657 + -0.397027 -0.397027 + -1.23493 -1.23493 + -0.816994 -0.816994 + 1.4852 1.4852 + -1.34173 -1.34173 + -0.308267 -0.308267 + 2.45801 2.45801 + -1.7382 -1.7382 + -2.47404 -2.47404 + -0.791644 -0.791644 + -2.12967 -2.12967 + -1.6631 -1.6631 + 0.335476 0.335476 + 1.22216 1.22216 + -0.059939 -0.059939 + -3.55093 -3.55093 + -0.582763 -0.582763 + 2.68949 2.68949 + 2.01951 2.01951 + 0.494128 0.494128 + -0.116969 -0.116969 + 0.304021 0.304021 + -2.11024 -2.11024 + 0.47012 0.47012 + -1.61832 -1.61832 + 2.74031 2.74031 + -0.812687 -0.812687 + 2.5097 2.5097 + 1.04051 1.04051 + 0.281989 0.281989 + -2.74931 -2.74931 + 2.62989 2.62989 + -0.802226 -0.802226 + 1.62871 1.62871 + -1.06605 -1.06605 + 0.121078 0.121078 + -0.632968 -0.632968 + -0.216849 -0.216849 + -0.322418 -0.322418 + 3.07053 3.07053 + 4.40286 4.40286 + -0.872669 -0.872669 + -0.127031 -0.127031 + 0.78 0.78 + 0.0148648 0.0148648 + -0.629511 -0.629511 + -1.05654 -1.05654 + 1.60099 1.60099 + -0.0685499 -0.0685499 + 0.854439 0.854439 + -0.187685 -0.187685 + -1.08173 -1.08173 + 0.614483 0.614483 + 2.87317 2.87317 + -3.44885 -3.44885 + 0.0392022 0.0392022 + -0.472168 -0.472168 + 2.172 2.172 + 0.997629 0.997629 + -0.604507 -0.604507 + 1.74349 1.74349 + 1.26225 1.26225 + 0.45734 0.45734 + -0.0242855 -0.0242855 + -1.98687 -1.98687 + 0.0133762 0.0133762 + 1.10497 1.10497 + 1.48159 1.48159 + -0.00565977 -0.00565977 + -0.273567 -0.273567 + 0.844543 0.844543 + 0.875793 0.875793 + -2.28312 -2.28312 + -1.57407 -1.57407 + 1.17889 1.17889 + 0.00492724 0.00492724 + 0.999081 0.999081 + 3.84377 3.84377 + 1.04097 1.04097 + 1.57247 1.57247 + -0.470368 -0.470368 + 0.00163007 0.00163007 + -1.51918 -1.51918 + -0.00223237 -0.00223237 + 1.31357 1.31357 + -0.189349 -0.189349 + 1.28289 1.28289 + 2.97546 2.97546 + -2.90089 -2.90089 + 1.57488 1.57488 + 1.34312 1.34312 + 1.52898 1.52898 + -1.64515 -1.64515 + -1.15666 -1.15666 + -1.2686 -1.2686 + -1.8123 -1.8123 + 0.996818 0.996818 + 1.02553 1.02553 + -0.0632497 -0.0632497 + 1.59041 1.59041 + -0.527615 -0.527615 + -1.48674 -1.48674 + -0.620959 -0.620959 + -0.342495 -0.342495 + -1.93695 -1.93695 + -3.64679 -3.64679 + -0.15153 -0.15153 + -0.942336 -0.942336 + -0.236199 -0.236199 + 2.03071 2.03071 + 0.316865 0.316865 + -1.04498 -1.04498 + 2.27925 2.27925 + -2.82399 -2.82399 + -1.33688 -1.33688 + 0.886898 0.886898 + -0.636226 -0.636226 + -1.85773 -1.85773 + -0.0714151 -0.0714151 + 3.68168 3.68168 + 2.40809 2.40809 + 1.97447 1.97447 + 0.203908 0.203908 + 1.04359 1.04359 + -0.0644891 -0.0644891 + -0.310849 -0.310849 + -0.326476 -0.326476 + -1.17233 -1.17233 + 0.883281 0.883281 + -0.801131 -0.801131 + -0.554541 -0.554541 + 0.411139 0.411139 + -2.13628 -2.13628 + -1.85167 -1.85167 + -0.073298 -0.073298 + 3.95706 3.95706 + -1.3341 -1.3341 + 1.69608 1.69608 + 0.0962806 0.0962806 + -2.74171 -2.74171 + 0.616342 0.616342 + 3.68453 3.68453 + -4.27901 -4.27901 + -0.366352 -0.366352 + -0.951105 -0.951105 + 2.67429 2.67429 + -0.167208 -0.167208 + 0.574208 0.574208 + 0.965369 0.965369 + 0.572892 0.572892 + -0.0812238 -0.0812238 + 0.126916 0.126916 + 0.0906106 0.0906106 + -2.21247 -2.21247 + 0.330044 0.330044 + -1.31031 -1.31031 + -0.196966 -0.196966 + -0.587011 -0.587011 + 1.42082 1.42082 + 1.58975 1.58975 + 0.568703 0.568703 + 3.14349 3.14349 + 0.883494 0.883494 + 1.21594 1.21594 + 1.11572 1.11572 + -1.01462 -1.01462 + 0.0228628 0.0228628 + -0.777921 -0.777921 + -0.0185091 -0.0185091 + 2.9342 2.9342 + -1.63432 -1.63432 + -0.0897289 -0.0897289 + 0.173623 0.173623 + 1.46671 1.46671 + 0.263784 0.263784 + -0.760012 -0.760012 + 0.179174 0.179174 + 0.318366 0.318366 + -1.09259 -1.09259 + -2.41019 -2.41019 + 2.18473 2.18473 + -0.406192 -0.406192 + -1.71917 -1.71917 + 3.65316 3.65316 + 0.163596 0.163596 + -0.0996923 -0.0996923 + -1.36222 -1.36222 + -0.946804 -0.946804 + 1.19432 1.19432 + 2.71731 2.71731 + 0.0474303 0.0474303 + -2.81076 -2.81076 + -4.10022 -4.10022 + 1.54347 1.54347 + -0.870202 -0.870202 + -0.521006 -0.521006 + 0.406167 0.406167 + -0.578452 -0.578452 + -1.34979 -1.34979 + 0.307851 0.307851 + 0.0160142 0.0160142 + -0.0118573 -0.0118573 + -0.0629544 -0.0629544 + -2.30284 -2.30284 + 3.1494 3.1494 + -1.74032 -1.74032 + -0.499308 -0.499308 + -0.637685 -0.637685 + -0.458922 -0.458922 + -1.81869 -1.81869 + 3.54851 3.54851 + 1.02612 1.02612 + -0.0730208 -0.0730208 + 1.59768 1.59768 + -0.123013 -0.123013 + 1.97723 1.97723 + -0.620977 -0.620977 + -0.634377 -0.634377 + -1.66552 -1.66552 + -1.96801 -1.96801 + -2.06075 -2.06075 + 1.59643 1.59643 + 0.496652 0.496652 + 3.12764 3.12764 + 1.53881 1.53881 + -0.603705 -0.603705 + -1.67414 -1.67414 + -1.44337 -1.44337 + 0.413483 0.413483 + 0.513435 0.513435 1.94637 1.94637 - -4.04181 -4.04181 - 16.3509 16.3509 - 6.4273 6.4273 - 5.41235 5.41235 - -5.91387 -5.91387 - -6.06301 -6.06301 - 3.4536 3.4536 - -3.39128 -3.39128 - 11.299 11.299 - 2.62685 2.62685 - 1.00866 1.00866 - 10.6766 10.6766 - -0.805083 -0.805083 - 3.91073 3.91073 - 3.67201 3.67201 - -9.14116 -9.14116 - 15.6406 15.6406 - 3.22084 3.22084 - -2.90513 -2.90513 - 4.58966 4.58966 - 0.0983211 0.0983211 - 2.35908 2.35908 + -1.04941 -1.04941 + -1.18587 -1.18587 + 0.0255263 0.0255263 + 3.32276 3.32276 + -1.03282 -1.03282 + -1.7708 -1.7708 + -0.139377 -0.139377 + 0.0487106 0.0487106 + 2.69087 2.69087 + 0.0466001 0.0466001 + 3.96837 3.96837 + -0.437004 -0.437004 + 0.463263 0.463263 + -0.390073 -0.390073 + 1.03124 1.03124 + 1.94068 1.94068 + 0.951059 0.951059 + 0.561724 0.561724 + -0.435045 -0.435045 + 1.02984 1.02984 + 1.7263 1.7263 + 0.538726 0.538726 + 1.56557 1.56557 + 1.15078 1.15078 + 1.68213 1.68213 + 0.16588 0.16588 + 1.85933 1.85933 + 1.12831 1.12831 + -1.91047 -1.91047 + 0.0938072 0.0938072 + 0.709282 0.709282 + -0.491057 -0.491057 + -0.708452 -0.708452 + 2.89155 2.89155 + -1.42685 -1.42685 + -0.386733 -0.386733 + -2.32159 -2.32159 + 0.751944 0.751944 + 0.314597 0.314597 + -1.90364 -1.90364 + -2.28503 -2.28503 + -0.983842 -0.983842 + 0.478018 0.478018 + -1.06256 -1.06256 + -1.27427 -1.27427 + 1.92344 1.92344 + 1.93071 1.93071 + -0.634679 -0.634679 + 1.07079 1.07079 + -2.73159 -2.73159 + 1.13198 1.13198 + -1.13381 -1.13381 + -0.275951 -0.275951 + -2.56787 -2.56787 + 2.96264 2.96264 + -1.68603 -1.68603 + 0.0565433 0.0565433 + -1.78596 -1.78596 + -1.64427 -1.64427 + -0.109207 -0.109207 + -0.123623 -0.123623 + 0.696342 0.696342 + -0.865633 -0.865633 + 1.55017 1.55017 + 3.57577 3.57577 + 0.574881 0.574881 + 0.311736 0.311736 + 1.12859 1.12859 + -0.660953 -0.660953 + 0.0603463 0.0603463 + 1.19673 1.19673 + -1.68319 -1.68319 + 0.0573934 0.0573934 + -3.0577 -3.0577 + -0.0811733 -0.0811733 + -1.35226 -1.35226 + 2.20395 2.20395 + -5.04376 -5.04376 + 0.957934 0.957934 + -2.67944 -2.67944 + 0.0583888 0.0583888 + -1.36195 -1.36195 + -1.14978 -1.14978 + 2.52851 2.52851 + -0.221427 -0.221427 + 1.39957 1.39957 + 1.97268 1.97268 + 1.58743 1.58743 + -1.25526 -1.25526 + -0.574435 -0.574435 + 0.879556 0.879556 + -1.60156 -1.60156 + 0.96617 0.96617 + -0.704615 -0.704615 + -1.13136 -1.13136 + -1.29642 -1.29642 + -0.48684 -0.48684 + -0.603142 -0.603142 + -1.55072 -1.55072 + 0.199318 0.199318 + -1.25072 -1.25072 + -0.0617283 -0.0617283 + 1.28993 1.28993 + 2.55354 2.55354 + -0.484267 -0.484267 + -0.629886 -0.629886 + 0.738121 0.738121 + 1.97734 1.97734 + -4.42185 -4.42185 + 1.38591 1.38591 + -0.403438 -0.403438 + 0.113538 0.113538 + -0.379133 -0.379133 + 1.15787 1.15787 + 2.57502 2.57502 + -2.24232 -2.24232 + 0.0712947 0.0712947 + -2.6907 -2.6907 + -2.27887 -2.27887 + -0.813216 -0.813216 + 2.34541 2.34541 + -1.11775 -1.11775 + 0.768486 0.768486 + -2.0542 -2.0542 + 1.52947 1.52947 + -4.3352 -4.3352 + -0.485814 -0.485814 + -1.67194 -1.67194 + 0.233108 0.233108 + 0.530897 0.530897 + 2.05494 2.05494 + 1.49262 1.49262 + 0.395652 0.395652 + -1.30437 -1.30437 + 0.500992 0.500992 + 1.28381 1.28381 + 0.393163 0.393163 + 3.96419 3.96419 + 0.434971 0.434971 + -0.228664 -0.228664 + 0.0615298 0.0615298 + -1.58079 -1.58079 + -1.01419 -1.01419 + -2.10373 -2.10373 + 0.0069584 0.0069584 + 0.382241 0.382241 + -0.0795624 -0.0795624 + 1.59346 1.59346 + 1.22138 1.22138 + -1.90972 -1.90972 + -0.365729 -0.365729 + -0.568999 -0.568999 + 1.42262 1.42262 + 0.282054 0.282054 + -1.06812 -1.06812 + 2.11136 2.11136 + 2.22528 2.22528 + 0.958127 0.958127 + -0.211857 -0.211857 + 0.354436 0.354436 + 6.28299 6.28299 + -1.88306 -1.88306 + 1.16816 1.16816 + -0.724852 -0.724852 + 2.05035 2.05035 + 1.1614 1.1614 + -0.0774035 -0.0774035 + -0.991458 -0.991458 + -2.224 -2.224 + -0.912683 -0.912683 + -2.33781 -2.33781 + -1.32562 -1.32562 + 2.07222 2.07222 + 1.86425 1.86425 + -1.12945 -1.12945 + -1.42154 -1.42154 + -1.83308 -1.83308 + -0.80144 -0.80144 + 0.0659904 0.0659904 + 0.46477 0.46477 + 0.516478 0.516478 + -0.714539 -0.714539 + 0.563923 0.563923 + -1.11843 -1.11843 + 0.24327 0.24327 + 1.9211 1.9211 + 1.5536 1.5536 + -1.83635 -1.83635 + 0.245734 0.245734 + 0.894795 0.894795 + 0.449533 0.449533 + 1.83397 1.83397 + 1.76346 1.76346 + 3.16185 3.16185 + 0.559932 0.559932 + -0.882377 -0.882377 + -0.717065 -0.717065 + -0.268154 -0.268154 + 1.02418 1.02418 + 2.46569 2.46569 + -0.737093 -0.737093 + 0.872719 0.872719 + 0.156786 0.156786 + -1.68971 -1.68971 + -0.762942 -0.762942 + 0.0251112 0.0251112 + 0.612489 0.612489 + -3.30371 -3.30371 + -0.698961 -0.698961 + -2.20203 -2.20203 + -1.89275 -1.89275 + 0.0291887 0.0291887 + -0.61895 -0.61895 + -2.4545 -2.4545 + 0.716685 0.716685 + -0.63556 -0.63556 + 0.957159 0.957159 + -0.273225 -0.273225 + -0.441594 -0.441594 + -0.899482 -0.899482 + -2.85088 -2.85088 + -0.660739 -0.660739 + 1.20197 1.20197 + -0.421759 -0.421759 + 2.36317 2.36317 + -2.68519 -2.68519 + 1.48605 1.48605 + -0.680547 -0.680547 + -3.61262 -3.61262 + 0.200643 0.200643 + 2.05747 2.05747 + 2.03866 2.03866 + -1.53579 -1.53579 + -0.300568 -0.300568 + 2.17815 2.17815 + 0.889736 0.889736 + -0.269451 -0.269451 + 3.85836 3.85836 + -0.0832182 -0.0832182 + 0.575183 0.575183 + -0.90211 -0.90211 + -0.352595 -0.352595 + 2.23507 2.23507 + 0.863045 0.863045 + 1.76841 1.76841 + 0.159141 0.159141 + -0.187019 -0.187019 + -0.605252 -0.605252 + 0.950606 0.950606 + 0.126904 0.126904 + -1.33489 -1.33489 + -2.65456 -2.65456 + 0.973579 0.973579 + 1.11206 1.11206 + 2.71157 2.71157 + -1.35922 -1.35922 + -2.09924 -2.09924 + -0.261339 -0.261339 + -0.71599 -0.71599 + 0.957563 0.957563 + -2.38311 -2.38311 + -1.47771 -1.47771 + 1.71136 1.71136 + -1.54574 -1.54574 + -0.66235 -0.66235 + -0.727048 -0.727048 + 1.7508 1.7508 + 3.09727 3.09727 + 2.60078 2.60078 + -1.96873 -1.96873 + 0.924622 0.924622 + 0.287636 0.287636 + 0.201588 0.201588 + 0.638481 0.638481 + 1.3465 1.3465 + 1.56972 1.56972 + 1.14971 1.14971 + -1.89176 -1.89176 + 2.74371 2.74371 + 1.23168 1.23168 + 3.05252 3.05252 + 0.608012 0.608012 + -1.21346 -1.21346 + -0.220215 -0.220215 + 1.27374 1.27374 + 1.89971 1.89971 + -0.315658 -0.315658 + -1.53046 -1.53046 + -0.815528 -0.815528 + 1.11711 1.11711 + -1.86098 -1.86098 + -2.62764 -2.62764 + -1.94186 -1.94186 + -1.01812 -1.01812 + 0.10927 0.10927 + 1.83491 1.83491 + 3.75411 3.75411 + -0.329943 -0.329943 + 1.04999 1.04999 + -0.0572795 -0.0572795 + -1.19271 -1.19271 + -0.0640939 -0.0640939 + 3.24013 3.24013 + 1.19772 1.19772 + 0.139843 0.139843 + -1.14619 -1.14619 + -0.346295 -0.346295 + 0.314854 0.314854 + -1.86965 -1.86965 + -2.5997 -2.5997 + 0.860876 0.860876 + 0.779138 0.779138 + -1.48689 -1.48689 + -0.041387 -0.041387 + 0.415675 0.415675 + -0.996154 -0.996154 + -0.210727 -0.210727 + 3.01432 3.01432 + -0.818496 -0.818496 + -1.77229 -1.77229 + -5.1652 -5.1652 + -0.430049 -0.430049 + 0.110492 0.110492 + -0.633977 -0.633977 + 5.47398 5.47398 + -0.827223 -0.827223 + 0.316164 0.316164 + -1.33416 -1.33416 + -2.58731 -2.58731 + 0.331473 0.331473 + -0.495747 -0.495747 + -0.578983 -0.578983 + 2.24558 2.24558 + -0.465562 -0.465562 + -0.88282 -0.88282 + 1.00385 1.00385 + -0.986489 -0.986489 + -0.356386 -0.356386 + 1.56415 1.56415 + -3.3696 -3.3696 + -2.39466 -2.39466 + -6.8684 -6.8684 + -0.705285 -0.705285 + -0.473808 -0.473808 + 1.34126 1.34126 + 1.17112 1.17112 + -0.638047 -0.638047 + -2.18021 -2.18021 + -0.748828 -0.748828 + 1.33976 1.33976 + 0.838517 0.838517 + 0.229636 0.229636 + -1.10193 -1.10193 + -0.713668 -0.713668 + 0.0447604 0.0447604 + -0.126929 -0.126929 + -2.65546 -2.65546 + -0.407056 -0.407056 + -0.224281 -0.224281 + -0.206263 -0.206263 + 1.59537 1.59537 + 2.12341 2.12341 + -0.165041 -0.165041 + 0.745372 0.745372 + -2.61492 -2.61492 + 0.764695 0.764695 + -0.721477 -0.721477 + 2.71105 2.71105 + 0.400047 0.400047 + -0.620564 -0.620564 + -0.676275 -0.676275 + -1.65892 -1.65892 + 0.153766 0.153766 + -0.789512 -0.789512 + 0.718952 0.718952 + -1.3161 -1.3161 + 1.3924 1.3924 + -0.752873 -0.752873 + 1.47481 1.47481 + 0.764075 0.764075 + 1.13314 1.13314 + 1.11338 1.11338 + 0.0911534 0.0911534 + 0.358637 0.358637 + 0.443527 0.443527 + -1.65442 -1.65442 + -1.18768 -1.18768 + -0.671452 -0.671452 + -0.10452 -0.10452 + 2.63571 2.63571 + -0.365343 -0.365343 + 0.989189 0.989189 + 1.016 1.016 + 0.02238 0.02238 + 2.5152 2.5152 + -0.502349 -0.502349 + 2.07812 2.07812 + -2.60912 -2.60912 + 0.039301 0.039301 + 2.21963 2.21963 + -1.25433 -1.25433 + 1.04089 1.04089 + 2.15635 2.15635 + -1.17901 -1.17901 + 0.464251 0.464251 + 0.523713 0.523713 + 2.6929 2.6929 + -3.17565 -3.17565 + -0.0704547 -0.0704547 + 0.316088 0.316088 + -0.595716 -0.595716 + -0.962247 -0.962247 + 2.07635 2.07635 + -0.707047 -0.707047 + -1.95768 -1.95768 + -1.05889 -1.05889 + -0.346038 -0.346038 + -0.519161 -0.519161 + 0.679303 0.679303 + 0.668559 0.668559 + -1.18768 -1.18768 + 1.05391 1.05391 + 0.495406 0.495406 + 0.775206 0.775206 + 1.21855 1.21855 + -0.188646 -0.188646 + 2.78981 2.78981 + 0.276702 0.276702 + -2.51357 -2.51357 + 0.893815 0.893815 + -0.600099 -0.600099 + 1.27468 1.27468 + 0.176813 0.176813 + -0.97769 -0.97769 + -0.831947 -0.831947 + -0.78568 -0.78568 + 1.54688 1.54688 + -1.66614 -1.66614 + 1.74662 1.74662 + -0.144336 -0.144336 + 3.25977 3.25977 + 0.604924 0.604924 + -1.08699 -1.08699 + 0.337396 0.337396 + 0.6638 0.6638 + -1.03335 -1.03335 + -2.12209 -2.12209 + -4.10625 -4.10625 + 0.575479 0.575479 + -0.703679 -0.703679 + 0.455751 0.455751 + -3.87142 -3.87142 + 1.82789 1.82789 + -1.17806 -1.17806 + -0.555556 -0.555556 + -3.14948 -3.14948 + 2.48156 2.48156 + 1.53411 1.53411 + -0.306063 -0.306063 + 0.182748 0.182748 + -4.68966 -4.68966 + 1.65411 1.65411 + -0.471352 -0.471352 + 2.08115 2.08115 + -1.14689 -1.14689 + 1.7511 1.7511 + 2.02855 2.02855 + -0.842776 -0.842776 + 0.708793 0.708793 + -0.663495 -0.663495 + 0.752657 0.752657 + -0.837532 -0.837532 + 1.94326 1.94326 + -1.0195 -1.0195 + -0.189062 -0.189062 + 0.443252 0.443252 + 1.53346 1.53346 + -1.06344 -1.06344 + 2.09193 2.09193 + 1.11391 1.11391 + 0.973692 0.973692 + 1.60649 1.60649 + -2.98173 -2.98173 + -1.30708 -1.30708 + 0.686241 0.686241 + 1.44115 1.44115 + -0.804022 -0.804022 + 0.128653 0.128653 + 0.148315 0.148315 + 1.05242 1.05242 + 4.81345 4.81345 + -0.0360308 -0.0360308 + -0.0977653 -0.0977653 + 2.50497 2.50497 + 0.0427317 0.0427317 + 1.07616 1.07616 + 2.6184 2.6184 + 4.16012 4.16012 + 1.94137 1.94137 + 0.168671 0.168671 + 0.249589 0.249589 + 2.29888 2.29888 + -2.58131 -2.58131 + 1.02287 1.02287 + -3.24177 -3.24177 + 0.0501678 0.0501678 + -3.08706 -3.08706 + -1.81322 -1.81322 + -3.21173 -3.21173 + -0.605546 -0.605546 + 1.80962 1.80962 + 1.1928 1.1928 + 1.16219 1.16219 + -0.502315 -0.502315 + -2.12962 -2.12962 + 1.82909 1.82909 + -0.515471 -0.515471 + 0.465682 0.465682 + -2.27668 -2.27668 + -0.794018 -0.794018 + 0.906492 0.906492 + -2.97682 -2.97682 + 1.76548 1.76548 + -2.14784 -2.14784 + -1.70079 -1.70079 + 0.575641 0.575641 + 2.18061 2.18061 + 0.721652 0.721652 + 3.34416 3.34416 + 2.92373 2.92373 + -2.27608 -2.27608 + -0.147619 -0.147619 + 2.27896 2.27896 + 3.13783 3.13783 + 0.53847 0.53847 + 1.59368 1.59368 + -2.92919 -2.92919 + -0.665617 -0.665617 + 2.21631 2.21631 + -0.132953 -0.132953 + 0.255364 0.255364 + 1.26044 1.26044 + -0.234991 -0.234991 + 0.0326571 0.0326571 + 1.95529 1.95529 + -0.147709 -0.147709 + 0.837403 0.837403 + 0.264494 0.264494 + -0.79803 -0.79803 + 1.24209 1.24209 + 4.16292 4.16292 + 2.90203 2.90203 + -1.77443 -1.77443 + -1.25963 -1.25963 + -1.97697 -1.97697 + -0.0228958 -0.0228958 + 0.961171 0.961171 + -1.98808 -1.98808 + 1.9206 1.9206 + 0.948345 0.948345 + -0.26941 -0.26941 + -0.895958 -0.895958 + -0.0969616 -0.0969616 + -1.8459 -1.8459 + 1.06631 1.06631 + 0.582305 0.582305 + 0.949072 0.949072 + -1.12433 -1.12433 + 1.75585 1.75585 + -4.27514 -4.27514 + 0.193812 0.193812 + -0.879839 -0.879839 + 0.740636 0.740636 + -1.46844 -1.46844 + 1.1903 1.1903 + 0.14461 0.14461 + -2.83538 -2.83538 + 1.90454 1.90454 + -1.91853 -1.91853 + -0.606642 -0.606642 + 2.28632 2.28632 + 1.92542 1.92542 + 2.53635 2.53635 + 1.55833 1.55833 + 1.52359 1.52359 + 0.777676 0.777676 + 3.06456 3.06456 + 0.288197 0.288197 + 1.2306 1.2306 + -1.57984 -1.57984 + -2.09081 -2.09081 + 2.21411 2.21411 + -0.0590976 -0.0590976 + 0.0967886 0.0967886 + 1.56966 1.56966 + 1.0458 1.0458 + 2.13399 2.13399 + 0.582962 0.582962 + -3.258 -3.258 + 1.31176 1.31176 + 2.12008 2.12008 + 0.420016 0.420016 + 1.16769 1.16769 + 0.322813 0.322813 + -2.08123 -2.08123 + -0.366867 -0.366867 + -0.863194 -0.863194 + 2.1781 2.1781 + -0.607621 -0.607621 + -1.46271 -1.46271 + -0.645887 -0.645887 + 2.07943 2.07943 + 1.66221 1.66221 + 1.00057 1.00057 + 1.96566 1.96566 + 1.40844 1.40844 + 1.99744 1.99744 + -1.17224 -1.17224 + 0.688052 0.688052 + -1.56755 -1.56755 + -1.1927 -1.1927 + 1.44395 1.44395 + 0.739251 0.739251 + 2.06788 2.06788 + 0.603058 0.603058 + -0.749518 -0.749518 + 0.935585 0.935585 + -0.671683 -0.671683 + -0.489752 -0.489752 + 0.984702 0.984702 + 0.571385 0.571385 + -0.386589 -0.386589 + 1.61408 1.61408 + 2.16478 2.16478 + 2.3196 2.3196 + -2.31076 -2.31076 + -0.186898 -0.186898 + 1.61571 1.61571 + -0.369523 -0.369523 + -2.21236 -2.21236 + -3.84185 -3.84185 + -0.0851651 -0.0851651 + 0.295692 0.295692 + -2.87568 -2.87568 + 2.97524 2.97524 + 0.974503 0.974503 + 2.36206 2.36206 + -1.7699 -1.7699 + 1.04331 1.04331 + -1.8445 -1.8445 + -1.58605 -1.58605 + -0.410044 -0.410044 + -0.588594 -0.588594 + -0.916676 -0.916676 + 0.493802 0.493802 + 0.0781382 0.0781382 + -0.958667 -0.958667 + 0.237942 0.237942 + 0.344577 0.344577 + -0.155517 -0.155517 + -0.0599809 -0.0599809 + 0.625756 0.625756 + 1.083 1.083 + -1.55908 -1.55908 + -0.0447221 -0.0447221 + 2.29293 2.29293 + -0.818719 -0.818719 + -0.431673 -0.431673 + 0.807989 0.807989 + -1.49538 -1.49538 + -0.415386 -0.415386 + -0.368204 -0.368204 + 0.822768 0.822768 + -1.04757 -1.04757 + 1.50985 1.50985 + 0.591814 0.591814 + 0.576176 0.576176 + 2.2397 2.2397 + -2.87292 -2.87292 + -1.05271 -1.05271 + -0.584545 -0.584545 + 23.6334 23.6334 + -1.94422 -1.94422 + -2.4793 -2.4793 + -0.461402 -0.461402 + -2.68547 -2.68547 + 1.27863 1.27863 + 0.349233 0.349233 + -1.58019 -1.58019 + 1.55179 1.55179 + -0.206459 -0.206459 + -0.985779 -0.985779 + 0.943214 0.943214 + -0.110792 -0.110792 + -0.0489029 -0.0489029 + 2.33366 2.33366 + -1.6701 -1.6701 + 0.201543 0.201543 + -0.0789827 -0.0789827 + -1.27944 -1.27944 + -1.77649 -1.77649 + 0.872661 0.872661 + 2.52858 2.52858 + 2.94238 2.94238 + 1.27241 1.27241 + -0.58158 -0.58158 + 1.37193 1.37193 + 2.03604 2.03604 + 2.76534 2.76534 + 1.36309 1.36309 + 0.910546 0.910546 + 1.2274 1.2274 + 0.501399 0.501399 + 1.69964 1.69964 + -0.0684446 -0.0684446 + -0.271516 -0.271516 + -2.72402 -2.72402 + -0.962159 -0.962159 + -0.396473 -0.396473 + 0.924897 0.924897 + -1.1069 -1.1069 + 2.62532 2.62532 + -0.393446 -0.393446 + -0.136789 -0.136789 + -0.151452 -0.151452 + -0.096789 -0.096789 + 2.1524 2.1524 + 1.68668 1.68668 + -0.730035 -0.730035 + 1.92158 1.92158 + -3.87326 -3.87326 + -0.221315 -0.221315 + -0.355481 -0.355481 + -0.775173 -0.775173 + 1.7409 1.7409 + 1.3288 1.3288 + -12.0217 -12.0217 + 0.583705 0.583705 + 0.184282 0.184282 + 0.417036 0.417036 + -0.956676 -0.956676 + 0.520674 0.520674 + -0.899164 -0.899164 + 3.50664 3.50664 + -0.636307 -0.636307 + -1.73476 -1.73476 + 1.29043 1.29043 + 0.472702 0.472702 + -1.79694 -1.79694 + -1.04534 -1.04534 + 1.10865 1.10865 + -1.43563 -1.43563 + -1.20749 -1.20749 + 1.26766 1.26766 + 0.673491 0.673491 + -0.934399 -0.934399 + 2.26572 2.26572 + -1.46043 -1.46043 + 2.209 2.209 + -1.00799 -1.00799 + -0.148943 -0.148943 + -1.1825 -1.1825 + -1.21252 -1.21252 + 0.282581 0.282581 + 0.41457 0.41457 + 2.20195 2.20195 + 0.0617071 0.0617071 + -0.550251 -0.550251 + -1.06407 -1.06407 + 0.642124 0.642124 + -2.1808 -2.1808 + 0.495721 0.495721 + 0.932684 0.932684 + 2.11964 2.11964 + 2.48226 2.48226 + 1.99288 1.99288 + -0.40534 -0.40534 + -0.830009 -0.830009 + 0.694545 0.694545 + -1.87861 -1.87861 + 0.656508 0.656508 + 1.45896 1.45896 + 0.831577 0.831577 + 1.56499 1.56499 + 0.220691 0.220691 + -1.53346 -1.53346 + 2.11808 2.11808 + -0.172959 -0.172959 + 2.30349 2.30349 + -0.415921 -0.415921 + -0.412111 -0.412111 + -1.08472 -1.08472 + 0.878408 0.878408 + -1.59121 -1.59121 + 1.27521 1.27521 + 1.49535 1.49535 + 1.01611 1.01611 + 0.826557 0.826557 + 0.413962 0.413962 + -0.418091 -0.418091 + -1.27713 -1.27713 + -1.72911 -1.72911 + -0.680069 -0.680069 + -3.18506 -3.18506 + 0.144236 0.144236 + -1.3908 -1.3908 + -1.37211 -1.37211 + -0.583177 -0.583177 + -2.50307 -2.50307 + 0.214101 0.214101 + -0.742307 -0.742307 + 0.806269 0.806269 + -0.58649 -0.58649 + -1.13406 -1.13406 + 0.589182 0.589182 + -1.41751 -1.41751 + 0.38813 0.38813 + -0.761625 -0.761625 + -1.82823 -1.82823 + -2.76721 -2.76721 + -0.555275 -0.555275 + 2.20053 2.20053 + 0.989203 0.989203 + -1.94756 -1.94756 + -3.27283 -3.27283 + 1.82038 1.82038 + 1.73285 1.73285 + -3.36146 -3.36146 + -0.0434622 -0.0434622 + -3.1662 -3.1662 + -2.27172 -2.27172 + -1.64158 -1.64158 + 1.36217 1.36217 + 0.0614365 0.0614365 + -1.51214 -1.51214 + 0.525372 0.525372 + -2.38711 -2.38711 + 0.91023 0.91023 + 0.642573 0.642573 + 0.787917 0.787917 + -0.165626 -0.165626 + -2.5386 -2.5386 + -0.0847667 -0.0847667 + -1.73854 -1.73854 + -0.629933 -0.629933 + 0.0476356 0.0476356 + -2.96352 -2.96352 + 2.43105 2.43105 + 3.07803 3.07803 + 1.61165 1.61165 + 1.80835 1.80835 + 2.6522 2.6522 + 2.42881 2.42881 + 1.95263 1.95263 + -0.627659 -0.627659 + -2.94554 -2.94554 + -0.584281 -0.584281 + 0.174889 0.174889 + 1.63772 1.63772 + -2.90996 -2.90996 + -1.19518 -1.19518 + -0.30899 -0.30899 + -1.44569 -1.44569 + 0.380009 0.380009 + -1.09017 -1.09017 + 0.401782 0.401782 + -1.01077 -1.01077 + -0.391996 -0.391996 + -0.284267 -0.284267 + -0.281839 -0.281839 + 0.0555518 0.0555518 + 1.16644 1.16644 + -2.17393 -2.17393 + -3.02707 -3.02707 + -0.744337 -0.744337 + 0.614164 0.614164 + -1.26162 -1.26162 + -0.892012 -0.892012 + -1.38321 -1.38321 + -0.690443 -0.690443 + -0.190017 -0.190017 + 0.652475 0.652475 + -0.665589 -0.665589 + -0.958746 -0.958746 + 0.211427 0.211427 + 0.172027 0.172027 + -0.183977 -0.183977 + -4.92239 -4.92239 + 0.234765 0.234765 + -0.128532 -0.128532 + -0.397255 -0.397255 + 0.637882 0.637882 + -0.665182 -0.665182 + 0.989375 0.989375 + 13.9866 13.9866 + 0.507452 0.507452 + 1.82993 1.82993 + -0.661663 -0.661663 + -0.315201 -0.315201 + 0.0298377 0.0298377 + -1.02559 -1.02559 + 1.67329 1.67329 + 0.770169 0.770169 + -0.567072 -0.567072 + -0.822084 -0.822084 + -0.695256 -0.695256 + -0.81899 -0.81899 + 0.00983914 0.00983914 + 3.47498 3.47498 + -0.0516518 -0.0516518 + 1.07953 1.07953 + -0.286307 -0.286307 + -0.901457 -0.901457 + 0.111325 0.111325 + 0.724452 0.724452 + -0.672886 -0.672886 + 0.821217 0.821217 + -3.77372 -3.77372 + 0.461435 0.461435 + -0.231648 -0.231648 + 1.41987 1.41987 + 1.03119 1.03119 + 0.93142 0.93142 + -0.913693 -0.913693 + 2.17759 2.17759 + 1.63487 1.63487 + -0.0814173 -0.0814173 + -1.10167 -1.10167 + 0.479772 0.479772 + 0.542071 0.542071 + 0.226907 0.226907 + 0.484866 0.484866 + -2.79594 -2.79594 + 1.17344 1.17344 + 0.448936 0.448936 + -0.425432 -0.425432 + -0.816027 -0.816027 + 1.11008 1.11008 + 2.16526 2.16526 + 0.574023 0.574023 + 3.34678 3.34678 + 3.05037 3.05037 + -1.49271 -1.49271 + -1.14698 -1.14698 + 0.556459 0.556459 + -1.01214 -1.01214 + -0.309133 -0.309133 + -0.536957 -0.536957 + 1.62871 1.62871 + -2.88469 -2.88469 + -0.0979174 -0.0979174 + -3.37567 -3.37567 + 2.46043 2.46043 + -0.730684 -0.730684 + -0.900807 -0.900807 + -2.45852 -2.45852 + -0.108725 -0.108725 + -1.01572 -1.01572 + -3.68841 -3.68841 + 0.290852 0.290852 + 1.68984 1.68984 + 3.35964 3.35964 + -1.31673 -1.31673 + 0.321869 0.321869 + 2.34154 2.34154 + -1.67666 -1.67666 + -0.035619 -0.035619 + -2.13567 -2.13567 + 1.3469 1.3469 + 0.153865 0.153865 + 1.14801 1.14801 + -0.713299 -0.713299 + -0.412727 -0.412727 + -1.01501 -1.01501 + -0.578133 -0.578133 + 0.684314 0.684314 + 3.48687 3.48687 + 1.21238 1.21238 + -1.95149 -1.95149 + -3.36448 -3.36448 + -0.819665 -0.819665 + 0.277849 0.277849 + 1.29952 1.29952 + -0.717767 -0.717767 + -0.0224495 -0.0224495 + -0.95711 -0.95711 + -2.57496 -2.57496 + 0.856373 0.856373 + 0.938171 0.938171 + -0.492205 -0.492205 + -1.45455 -1.45455 + 1.95963 1.95963 + -0.169591 -0.169591 + 1.93841 1.93841 + -0.0293303 -0.0293303 + -1.06165 -1.06165 + 0.0963018 0.0963018 + -0.0253168 -0.0253168 + -0.615891 -0.615891 + -1.84572 -1.84572 + 2.04114 2.04114 + 0.709038 0.709038 + -11.8503 -11.8503 + 1.30469 1.30469 + 1.29953 1.29953 + -0.0864412 -0.0864412 + -2.55483 -2.55483 + 0.0868521 0.0868521 + -0.472505 -0.472505 + -2.22982 -2.22982 + 0.675526 0.675526 + 0.903289 0.903289 + 2.1813 2.1813 + -0.731082 -0.731082 + 1.23144 1.23144 + 0.211691 0.211691 + -0.447575 -0.447575 + -0.266834 -0.266834 + 0.777422 0.777422 + 2.13011 2.13011 + 1.14053 1.14053 + -1.64879 -1.64879 + 0.253671 0.253671 + 1.37881 1.37881 + -3.47648 -3.47648 + 0.120876 0.120876 + 0.706163 0.706163 + -0.408708 -0.408708 + 1.63921 1.63921 + -0.0897068 -0.0897068 + -0.36319 -0.36319 + -2.72738 -2.72738 + 0.778532 0.778532 + -1.44104 -1.44104 + -0.477193 -0.477193 + 2.99272 2.99272 + -1.93478 -1.93478 + 0.7366 0.7366 + -0.461307 -0.461307 + 1.34756 1.34756 + 0.575798 0.575798 + 0.243039 0.243039 + 0.00184323 0.00184323 + -4.65195 -4.65195 + -0.0233881 -0.0233881 + 2.27189 2.27189 + -0.454542 -0.454542 + -0.621287 -0.621287 + -1.95901 -1.95901 + -1.59841 -1.59841 + -1.11109 -1.11109 + 0.276918 0.276918 + 0.822635 0.822635 + 0.399364 0.399364 + -3.59 -3.59 + 0.63147 0.63147 + -0.122051 -0.122051 + -2.99397 -2.99397 + -0.199531 -0.199531 + 1.70567 1.70567 + -0.802968 -0.802968 + 1.54216 1.54216 + 5.01761 5.01761 + 0.593876 0.593876 + 1.4851 1.4851 + -0.16419 -0.16419 + -2.67396 -2.67396 + -1.4692 -1.4692 + 3.53628 3.53628 + 0.842369 0.842369 + -1.10273 -1.10273 + 1.38417 1.38417 + -0.654059 -0.654059 + 3.03409 3.03409 + -0.599392 -0.599392 + -2.90447 -2.90447 + -1.42843 -1.42843 + 1.0605 1.0605 + 0.648235 0.648235 + -1.72383 -1.72383 + -0.356193 -0.356193 + -0.538367 -0.538367 + -3.79779 -3.79779 + 3.63493 3.63493 + -0.464008 -0.464008 + 0.347418 0.347418 + -0.467898 -0.467898 + 0.672538 0.672538 + -0.160574 -0.160574 + -3.60299 -3.60299 + 2.72608 2.72608 + 0.337715 0.337715 + -0.859081 -0.859081 + 4.38695 4.38695 + 2.18067 2.18067 + -1.61338 -1.61338 + -0.448975 -0.448975 + -1.84397 -1.84397 + 0.0152013 0.0152013 + 0.667211 0.667211 + -0.0943334 -0.0943334 + 0.860503 0.860503 + -0.115682 -0.115682 + -0.284645 -0.284645 + 3.97725 3.97725 + 1.98285 1.98285 + 0.576249 0.576249 + -1.60365 -1.60365 + 6.83394 6.83394 + 3.17895 3.17895 + 2.29932 2.29932 + 0.077594 0.077594 + -0.862484 -0.862484 + -0.455567 -0.455567 + 1.0918 1.0918 + -1.05352 -1.05352 + -0.503157 -0.503157 + -1.28519 -1.28519 + -2.55219 -2.55219 + -0.511736 -0.511736 + -0.601855 -0.601855 + 1.38829 1.38829 + 1.72822 1.72822 + 2.44367 2.44367 + -0.876967 -0.876967 + 1.32143 1.32143 + -0.345428 -0.345428 + 0.158257 0.158257 + -0.502065 -0.502065 + 0.602091 0.602091 + 0.546799 0.546799 + 0.144201 0.144201 + 1.28249 1.28249 + -0.766982 -0.766982 + -2.91678 -2.91678 + -0.420817 -0.420817 + 0.282388 0.282388 + -2.35922 -2.35922 + -0.959736 -0.959736 + 0.970576 0.970576 + 1.28214 1.28214 + -1.38422 -1.38422 + -1.28912 -1.28912 + -1.55508 -1.55508 + 2.60786 2.60786 + -0.174129 -0.174129 + -4.97013 -4.97013 + -3.13894 -3.13894 + -1.0113 -1.0113 + 0.29012 0.29012 + 0.301534 0.301534 + -1.36587 -1.36587 + 0.592223 0.592223 + -0.393313 -0.393313 + 1.54523 1.54523 + 0.489536 0.489536 + 0.431823 0.431823 + -0.698724 -0.698724 + 0.597885 0.597885 + -1.28348 -1.28348 + 9.20118 9.20118 + 0.523845 0.523845 + 1.64891 1.64891 + 1.26397 1.26397 + -0.0614047 -0.0614047 + 0.0327276 0.0327276 + -0.4776 -0.4776 + -2.24761 -2.24761 + 1.49423 1.49423 + -1.70954 -1.70954 + -0.174842 -0.174842 + 0.35175 0.35175 + 0.549776 0.549776 + -1.97708 -1.97708 + -0.797726 -0.797726 + -0.880683 -0.880683 + 1.92272 1.92272 + -1.05489 -1.05489 + -2.33809 -2.33809 + -1.59936 -1.59936 + -0.840616 -0.840616 + 2.6777 2.6777 + 0.86962 0.86962 + -1.01078 -1.01078 + -0.936281 -0.936281 + -1.24079 -1.24079 + -2.68717 -2.68717 + 0.22939 0.22939 + 0.75135 0.75135 + 1.07189 1.07189 + -0.0904491 -0.0904491 + -2.72094 -2.72094 + 0.259405 0.259405 + 0.801323 0.801323 + 1.79821 1.79821 + 0.13453 0.13453 + -0.294662 -0.294662 + 1.39746 1.39746 + -0.876874 -0.876874 + 0.411162 0.411162 + -1.2239 -1.2239 + -0.85717 -0.85717 + 0.50512 0.50512 + 1.59327 1.59327 + -1.76924 -1.76924 + 2.85557 2.85557 + -2.69548 -2.69548 + -1.5875 -1.5875 + -0.796946 -0.796946 + -0.335648 -0.335648 + -0.512417 -0.512417 + 2.13108 2.13108 + -1.85765 -1.85765 + -2.89643 -2.89643 + 1.76352 1.76352 + 0.575773 0.575773 + 1.59789 1.59789 + 1.42267 1.42267 + 0.497862 0.497862 + -0.567461 -0.567461 + 1.36287 1.36287 + 2.10155 2.10155 + -1.18077 -1.18077 + 2.49273 2.49273 + -1.94928 -1.94928 + 0.324793 0.324793 + 0.140937 0.140937 + 2.33415 2.33415 + -0.0985955 -0.0985955 + 1.71064 1.71064 + -0.274678 -0.274678 + 0.82142 0.82142 + 1.97483 1.97483 + 1.33147 1.33147 + 1.25382 1.25382 + -1.92684 -1.92684 + 0.633816 0.633816 + -1.74887 -1.74887 + -0.935251 -0.935251 + 0.406894 0.406894 + 0.630239 0.630239 + 1.49081 1.49081 + -1.7665 -1.7665 + 0.468253 0.468253 + -0.829623 -0.829623 + 0.683713 0.683713 + 0.343987 0.343987 + 1.87329 1.87329 + -0.46613 -0.46613 + 2.83798 2.83798 + 1.64433 1.64433 + 0.303556 0.303556 + -0.810505 -0.810505 + -0.548167 -0.548167 + 1.77963 1.77963 + 0.0793537 0.0793537 + -3.29027 -3.29027 + -0.183904 -0.183904 + -1.61751 -1.61751 + 0.860805 0.860805 + -0.722543 -0.722543 + 0.0263393 0.0263393 + -1.53128 -1.53128 + -0.242737 -0.242737 + 0.319961 0.319961 + 2.17752 2.17752 + 0.845624 0.845624 + 1.91994 1.91994 + 0.258224 0.258224 + 0.256254 0.256254 + -0.64679 -0.64679 + 1.51706 1.51706 + 1.2567 1.2567 + -0.0595054 -0.0595054 + 3.78386 3.78386 + 0.0964969 0.0964969 + 0.822442 0.822442 + 2.36122 2.36122 + 1.19663 1.19663 + 1.45557 1.45557 + -0.487415 -0.487415 + 2.80363 2.80363 + 0.354397 0.354397 + 2.064 2.064 + -1.60732 -1.60732 + 0.152285 0.152285 + -1.06767 -1.06767 + -2.89226 -2.89226 + -0.923556 -0.923556 + 0.129633 0.129633 + 0.414714 0.414714 + 0.313557 0.313557 + -0.63598 -0.63598 + -1.24145 -1.24145 + 0.445823 0.445823 + -1.18621 -1.18621 + -0.824399 -0.824399 + 0.0348621 0.0348621 + 1.72359 1.72359 + 3.23846 3.23846 + -0.982701 -0.982701 + -2.8631 -2.8631 + -0.247987 -0.247987 + 1.10508 1.10508 + -0.335101 -0.335101 + 1.59819 1.59819 + 0.363252 0.363252 + -1.22112 -1.22112 + 2.40608 2.40608 + 2.14135 2.14135 + 0.926073 0.926073 + -0.616568 -0.616568 + 0.829974 0.829974 + 4.0788 4.0788 + 1.97497 1.97497 + -0.943884 -0.943884 + 0.770483 0.770483 + 1.58248 1.58248 + 2.10963 2.10963 + -3.33186 -3.33186 + 1.52246 1.52246 + -1.38448 -1.38448 + 0.178628 0.178628 + 2.35971 2.35971 + 1.35345 1.35345 + 2.2217 2.2217 + -1.6579 -1.6579 + 0.752211 0.752211 + 1.60611 1.60611 + 0.224136 0.224136 + 0.8147 0.8147 + -6.54244 -6.54244 + 2.03158 2.03158 + 0.334614 0.334614 + -0.849204 -0.849204 + 0.573662 0.573662 + 0.834273 0.834273 + 1.98059 1.98059 + 4.59366 4.59366 + -0.0696824 -0.0696824 + -1.12577 -1.12577 + -0.392295 -0.392295 + 0.608493 0.608493 + 1.57322 1.57322 + 0.724246 0.724246 + -2.54194 -2.54194 + -0.61659 -0.61659 + 1.52939 1.52939 + 0.628167 0.628167 + 0.153008 0.153008 + 0.729473 0.729473 + 1.80753 1.80753 + -0.257529 -0.257529 + 2.30209 2.30209 + -1.07183 -1.07183 + -2.34549 -2.34549 + 1.7575 1.7575 + -1.09788 -1.09788 + -0.733157 -0.733157 + -1.15178 -1.15178 + -1.38334 -1.38334 + -0.282802 -0.282802 + -1.64302 -1.64302 + -8.14652 -8.14652 + 0.230736 0.230736 + 1.49834 1.49834 + -3.92113 -3.92113 + 1.26241 1.26241 + -0.867555 -0.867555 + 0.385275 0.385275 + -1.52048 -1.52048 + -3.58632 -3.58632 + 0.216633 0.216633 + -0.626523 -0.626523 + 0.0498179 0.0498179 + -0.736738 -0.736738 + -1.96551 -1.96551 + -0.886975 -0.886975 + -4.19118 -4.19118 + 4.69049 4.69049 + 1.24464 1.24464 + 0.559333 0.559333 + 0.365196 0.365196 + 2.05931 2.05931 + -1.61866 -1.61866 + -0.937082 -0.937082 + -2.28156 -2.28156 + 0.480478 0.480478 + 0.0375252 0.0375252 + 1.16423 1.16423 + 1.06467 1.06467 + -1.51476 -1.51476 + 0.639421 0.639421 + 0.835906 0.835906 + 0.0678173 0.0678173 + 0.911222 0.911222 + 0.178712 0.178712 + 0.16706 0.16706 + -0.338281 -0.338281 + 1.13458 1.13458 + 2.369 2.369 + 0.104902 0.104902 + 3.74878 3.74878 + -0.66965 -0.66965 + -0.539803 -0.539803 + 8.81148 8.81148 + 2.01163 2.01163 + -2.44142 -2.44142 + 1.71428 1.71428 + 0.195195 0.195195 + 0.886484 0.886484 + -1.61999 -1.61999 + 1.38916 1.38916 + -0.182804 -0.182804 + -2.12762 -2.12762 + 1.32879 1.32879 + 2.93628 2.93628 + 0.585458 0.585458 + 0.759949 0.759949 + 0.200884 0.200884 + 0.434392 0.434392 + 0.592809 0.592809 + -0.972697 -0.972697 + -0.30606 -0.30606 + 2.75682 2.75682 + 1.06405 1.06405 + 0.535445 0.535445 + 0.245727 0.245727 + -0.881514 -0.881514 + -0.90398 -0.90398 + 1.34242 1.34242 + 0.974607 0.974607 + -0.23557 -0.23557 + -2.36657 -2.36657 + 2.20975 2.20975 + 0.372629 0.372629 + -0.7674 -0.7674 + 0.503398 0.503398 + -0.39968 -0.39968 + 4.73687 4.73687 + 1.83971 1.83971 + -0.59006 -0.59006 + -3.12939 -3.12939 + 1.67781 1.67781 + -0.346719 -0.346719 + -1.20046 -1.20046 + -0.543011 -0.543011 + 2.00497 2.00497 + -2.29529 -2.29529 + 1.46115 1.46115 + -1.93647 -1.93647 + 0.346798 0.346798 + -0.781626 -0.781626 + -3.73517 -3.73517 + 0.403249 0.403249 + 0.147234 0.147234 + 0.581328 0.581328 + -1.56541 -1.56541 + -0.0382613 -0.0382613 + -2.69999 -2.69999 + 2.53761 2.53761 + 0.485891 0.485891 + 2.04661 2.04661 + -1.34907 -1.34907 + -2.77599 -2.77599 + -1.9941 -1.9941 + 2.26706 2.26706 + -0.841907 -0.841907 + 2.89779 2.89779 + 1.34194 1.34194 + 1.72031 1.72031 + 0.35742 0.35742 + -0.673142 -0.673142 + -1.97963 -1.97963 + 2.99822 2.99822 + 0.84135 0.84135 + 0.467833 0.467833 + -1.6184 -1.6184 + 0.545393 0.545393 + 0.0865121 0.0865121 + 1.38347 1.38347 + 2.3421 2.3421 + 2.6215 2.6215 + -1.48313 -1.48313 + 2.29728 2.29728 + -0.629032 -0.629032 + -0.053791 -0.053791 + 0.827546 0.827546 + -2.01859 -2.01859 + -0.151077 -0.151077 + -4.74001 -4.74001 + -2.07772 -2.07772 + 0.964427 0.964427 + 1.76579 1.76579 + -2.11515 -2.11515 + 2.99605 2.99605 + 1.38556 1.38556 + -2.60586 -2.60586 + -3.05364 -3.05364 + -1.22284 -1.22284 + 0.499165 0.499165 + 1.22502 1.22502 + -0.887611 -0.887611 + 1.1518 1.1518 + -2.7606 -2.7606 + 0.106263 0.106263 + 0.92196 0.92196 + -0.542352 -0.542352 + 0.772226 0.772226 + -0.409383 -0.409383 + -0.480159 -0.480159 + 0.434618 0.434618 + -1.93489 -1.93489 + 1.0037 1.0037 + 0.760285 0.760285 + 0.380404 0.380404 + 2.45566 2.45566 + -1.03099 -1.03099 + 0.652107 0.652107 + -1.64881 -1.64881 + -1.38051 -1.38051 + 1.43146 1.43146 + -0.366772 -0.366772 + 1.06121 1.06121 + 0.20026 0.20026 + 0.646821 0.646821 + 0.361296 0.361296 + 0.547448 0.547448 + 1.46293 1.46293 + 1.10425 1.10425 + -1.98927 -1.98927 + 1.33293 1.33293 + 0.702344 0.702344 + -0.294478 -0.294478 + 2.57426 2.57426 + 0.972237 0.972237 + -1.8532 -1.8532 + -1.52489 -1.52489 + -1.92059 -1.92059 + 0.4566 0.4566 + 0.0146066 0.0146066 + 1.15567 1.15567 + -2.15427 -2.15427 + -0.896258 -0.896258 + -0.579245 -0.579245 + 4.10418 4.10418 + 0.147275 0.147275 + -2.60135 -2.60135 + 1.13637 1.13637 + 4.39171 4.39171 + -3.93541 -3.93541 + -2.25406 -2.25406 + -0.766558 -0.766558 + 1.32684 1.32684 + 0.701722 0.701722 + 1.07565 1.07565 + -0.881368 -0.881368 + -1.79475 -1.79475 + -0.12303 -0.12303 + 0.939797 0.939797 + -1.63244 -1.63244 + -0.561795 -0.561795 + 0.668111 0.668111 + 2.28615 2.28615 + 1.6027 1.6027 + 0.210892 0.210892 + 1.10496 1.10496 + -3.5233 -3.5233 + 0.315028 0.315028 + -2.40724 -2.40724 + -0.865286 -0.865286 + -0.178629 -0.178629 + 4.42947 4.42947 + 2.0842 2.0842 + -1.17052 -1.17052 + -0.573684 -0.573684 + -0.230861 -0.230861 + -0.00536986 -0.00536986 + 1.18437 1.18437 + 0.954435 0.954435 + 0.475723 0.475723 + -1.45097 -1.45097 + -0.946094 -0.946094 + -0.371655 -0.371655 + -1.73344 -1.73344 + 0.25137 0.25137 + -0.904498 -0.904498 + 1.72233 1.72233 + -0.0630691 -0.0630691 + 1.20918 1.20918 + -1.58876 -1.58876 + -3.08633 -3.08633 + 0.469343 0.469343 + 0.334824 0.334824 + -0.420134 -0.420134 + -1.39986 -1.39986 + -0.65878 -0.65878 + -0.680645 -0.680645 + -1.46235 -1.46235 + -1.7763 -1.7763 + 0.061792 0.061792 + 2.10605 2.10605 + -0.198492 -0.198492 + 0.437371 0.437371 + -1.539 -1.539 + -0.615466 -0.615466 + 0.739653 0.739653 + -1.62828 -1.62828 + -3.99554 -3.99554 + -0.0523548 -0.0523548 + 1.77712 1.77712 + -0.756514 -0.756514 + 1.43585 1.43585 + -0.730324 -0.730324 + -0.714939 -0.714939 + -0.208195 -0.208195 + 3.00343 3.00343 + -0.780319 -0.780319 + -1.67158 -1.67158 + -4.72578 -4.72578 + -0.409396 -0.409396 + -0.182756 -0.182756 + 0.131891 0.131891 + -0.0622689 -0.0622689 + 0.504642 0.504642 + 1.05712 1.05712 + 0.603898 0.603898 + -1.08295 -1.08295 + 3.50826 3.50826 + -1.21709 -1.21709 + 2.62074 2.62074 + 2.27406 2.27406 + -0.444599 -0.444599 + -1.23079 -1.23079 + 2.01351 2.01351 + -0.771506 -0.771506 + -1.25143 -1.25143 + -2.42808 -2.42808 + 0.618042 0.618042 + -0.139276 -0.139276 + -1.15122 -1.15122 + -1.00705 -1.00705 + -0.513695 -0.513695 + 1.49666 1.49666 + 2.08012 2.08012 + -0.0331592 -0.0331592 + -1.27368 -1.27368 + 0.237436 0.237436 + -1.92915 -1.92915 + -3.38558 -3.38558 + 0.656076 0.656076 + 4.89326 4.89326 + -0.696932 -0.696932 + 1.81671 1.81671 + -0.163389 -0.163389 + -0.640608 -0.640608 + -0.466348 -0.466348 + 0.790762 0.790762 + -0.15507 -0.15507 + -0.387485 -0.387485 + 3.22853 3.22853 + -0.361262 -0.361262 + 0.838794 0.838794 + -1.80534 -1.80534 + -0.514814 -0.514814 + -1.96885 -1.96885 + -1.32784 -1.32784 + 0.420068 0.420068 + -0.720913 -0.720913 + -0.340982 -0.340982 + 0.496067 0.496067 + -0.921807 -0.921807 + -1.92175 -1.92175 + 0.59455 0.59455 + 0.827755 0.827755 + -1.90727 -1.90727 + 1.51996 1.51996 + 1.79573 1.79573 + -2.10208 -2.10208 + 4.78008 4.78008 + 0.767953 0.767953 + 0.240847 0.240847 + 1.97272 1.97272 + -1.59962 -1.59962 + -1.47237 -1.47237 + -0.580724 -0.580724 + 1.75388 1.75388 + -1.67413 -1.67413 + -0.636017 -0.636017 + -1.35096 -1.35096 + 0.722983 0.722983 + -0.585923 -0.585923 + -0.529864 -0.529864 + -0.91566 -0.91566 + 0.242429 0.242429 + 0.236836 0.236836 + -1.78323 -1.78323 + 0.716162 0.716162 + 0.166974 0.166974 + -0.00891157 -0.00891157 + 0.936227 0.936227 + 0.992144 0.992144 + 0.00694193 0.00694193 + -3.58995 -3.58995 + -2.90908 -2.90908 + 2.30937 2.30937 + -0.504803 -0.504803 + 0.870025 0.870025 + 1.0472 1.0472 + -0.165569 -0.165569 + -0.110656 -0.110656 + -0.240613 -0.240613 + 1.47692 1.47692 + 0.153659 0.153659 + 0.237772 0.237772 + -0.427641 -0.427641 + -1.6352 -1.6352 + -0.329495 -0.329495 + 1.89567 1.89567 + 0.500036 0.500036 + 2.60672 2.60672 + 1.08705 1.08705 + 0.234073 0.234073 + 1.80469 1.80469 + 0.859386 0.859386 + 1.59161 1.59161 + -1.25096 -1.25096 + 0.693032 0.693032 + -0.208542 -0.208542 + 2.9791 2.9791 + 0.0637262 0.0637262 + -0.329288 -0.329288 + 0.0765908 0.0765908 + -0.975783 -0.975783 + 0.44728 0.44728 + 0.158878 0.158878 + -1.23836 -1.23836 + 0.670039 0.670039 + -0.0279593 -0.0279593 + -1.08503 -1.08503 + -1.61416 -1.61416 + -1.68594 -1.68594 + 0.91421 0.91421 + 0.164412 0.164412 + -0.452083 -0.452083 + -3.85434 -3.85434 + -1.17754 -1.17754 + -0.824006 -0.824006 + 2.34023 2.34023 + -2.09985 -2.09985 + 0.881987 0.881987 + -1.37035 -1.37035 + 0.642559 0.642559 + -0.17144 -0.17144 + 1.61998 1.61998 + -0.621223 -0.621223 + -3.74722 -3.74722 + 1.14967 1.14967 + 0.718755 0.718755 + 0.675293 0.675293 + -1.53459 -1.53459 + 0.521371 0.521371 + -0.0142472 -0.0142472 + -1.10933 -1.10933 + -1.60977 -1.60977 + -0.152616 -0.152616 + 0.402871 0.402871 + 0.929587 0.929587 + 1.69781 1.69781 + -1.8187 -1.8187 + 2.04684 2.04684 + -1.29619 -1.29619 + -0.212775 -0.212775 + 1.52843 1.52843 + -1.69828 -1.69828 + -1.05251 -1.05251 + 3.80258 3.80258 + 1.77294 1.77294 + 1.20703 1.20703 + 0.81561 0.81561 + 0.748058 0.748058 + 1.45736 1.45736 + 1.33881 1.33881 + -0.86026 -0.86026 + -0.00694993 -0.00694993 + -1.67171 -1.67171 + -2.64284 -2.64284 + -0.617916 -0.617916 + -0.658362 -0.658362 + -0.302764 -0.302764 + -0.218727 -0.218727 + 2.68447 2.68447 + 1.91834 1.91834 + -0.630305 -0.630305 + -0.483942 -0.483942 + -0.205622 -0.205622 + 0.230539 0.230539 + -2.42183 -2.42183 + 0.000632207 0.000632207 + 1.22722 1.22722 + 0.499288 0.499288 + -0.461535 -0.461535 + -2.41778 -2.41778 + -6.43109 -6.43109 + 1.87164 1.87164 + -0.128426 -0.128426 + 1.70185 1.70185 + 0.321244 0.321244 + -2.8774 -2.8774 + -0.973839 -0.973839 + -1.78429 -1.78429 + -1.42296 -1.42296 + -2.18451 -2.18451 + -0.997205 -0.997205 + -2.09179 -2.09179 + -0.418012 -0.418012 + -3.86836 -3.86836 + -1.08119 -1.08119 + -2.73167 -2.73167 + -1.83625 -1.83625 + 0.048819 0.048819 + 1.82894 1.82894 + 0.828514 0.828514 + 0.392871 0.392871 + 1.13363 1.13363 + -1.79231 -1.79231 + 0.163824 0.163824 + -1.27652 -1.27652 + 2.06888 2.06888 + -0.63393 -0.63393 + 0.684518 0.684518 + -0.906544 -0.906544 + -0.987848 -0.987848 + -0.478984 -0.478984 + -1.2593 -1.2593 + 2.07364 2.07364 + 0.162523 0.162523 + -0.799031 -0.799031 + -0.299117 -0.299117 + 0.0988399 0.0988399 + -0.238386 -0.238386 + 3.0895 3.0895 + 0.415509 0.415509 + 1.50283 1.50283 + -0.561321 -0.561321 + 2.24325 2.24325 + -0.490123 -0.490123 + -1.27889 -1.27889 + 0.10889 0.10889 + 0.581722 0.581722 + -0.657735 -0.657735 + -0.722435 -0.722435 + -1.3811 -1.3811 + 1.5928 1.5928 + -9.72189 -9.72189 + -1.07349 -1.07349 + -1.25083 -1.25083 + 0.919328 0.919328 + -1.9068 -1.9068 + -1.23178 -1.23178 + -0.782403 -0.782403 + -0.689571 -0.689571 + -2.07093 -2.07093 + -0.0676932 -0.0676932 + -1.47628 -1.47628 + -1.79765 -1.79765 + 1.79051 1.79051 + 0.410585 0.410585 + -1.41706 -1.41706 + 1.64883 1.64883 + -0.528457 -0.528457 + 0.193134 0.193134 + -0.989156 -0.989156 + 3.3787 3.3787 + 0.734755 0.734755 + 1.20919 1.20919 + 0.142022 0.142022 + 0.950151 0.950151 + -2.14197 -2.14197 + 2.91323 2.91323 + -0.216118 -0.216118 + 1.18072 1.18072 + -5.00813 -5.00813 + 1.22234 1.22234 + -2.40273 -2.40273 + -1.93511 -1.93511 + 1.02775 1.02775 + -0.407362 -0.407362 + -0.033977 -0.033977 + -0.673432 -0.673432 + -0.395374 -0.395374 + -0.0686072 -0.0686072 + 2.26745 2.26745 + -4.0298 -4.0298 + 0.30017 0.30017 + 0.576736 0.576736 + 1.84158 1.84158 + -0.427059 -0.427059 + -3.47072 -3.47072 + -0.207896 -0.207896 + 1.07928 1.07928 + -0.424236 -0.424236 + -0.809626 -0.809626 + 2.48986 2.48986 + -0.808405 -0.808405 + -0.0510026 -0.0510026 + -0.919838 -0.919838 + 1.44427 1.44427 + 2.71946 2.71946 + 0.0683191 0.0683191 + -1.08351 -1.08351 + -1.56937 -1.56937 + 3.01879 3.01879 + -1.56172 -1.56172 + 2.06467 2.06467 + 2.03327 2.03327 + -0.81693 -0.81693 + 0.922913 0.922913 + 0.95788 0.95788 + -0.778203 -0.778203 + 1.08222 1.08222 + 1.44635 1.44635 + -1.75929 -1.75929 + 0.112181 0.112181 + 0.681292 0.681292 + -0.432171 -0.432171 + 1.06316 1.06316 + -1.7268 -1.7268 + -3.37779 -3.37779 + -2.89964 -2.89964 + 0.901763 0.901763 + -1.2757 -1.2757 + -12.5488 -12.5488 + 2.55937 2.55937 + -1.79624 -1.79624 + -2.36511 -2.36511 + 0.755598 0.755598 + -1.59443 -1.59443 + -3.05103 -3.05103 + -1.29157 -1.29157 + 0.254057 0.254057 + -0.628874 -0.628874 + -0.119316 -0.119316 + 4.96649 4.96649 + -1.36958 -1.36958 + -0.258365 -0.258365 + 0.207744 0.207744 + -0.166113 -0.166113 + -1.22071 -1.22071 + -3.68573 -3.68573 + -3.16103 -3.16103 + -1.89154 -1.89154 + 0.98744 0.98744 + -0.585739 -0.585739 + -1.99954 -1.99954 + 2.03071 2.03071 + 0.471045 0.471045 + -1.93214 -1.93214 + -0.825462 -0.825462 + 0.497266 0.497266 + -0.563979 -0.563979 + 1.69555 1.69555 + 3.04967 3.04967 + -0.302795 -0.302795 + 1.60455 1.60455 + 2.28473 2.28473 + -1.28218 -1.28218 + -0.559678 -0.559678 + -0.759997 -0.759997 + 0.809902 0.809902 + -0.451467 -0.451467 + -1.21982 -1.21982 + 0.325713 0.325713 + -3.28885 -3.28885 + -0.124363 -0.124363 + -0.698127 -0.698127 + -0.213705 -0.213705 + -2.03161 -2.03161 + 1.82803 1.82803 + 0.644781 0.644781 + 0.20483 0.20483 + 4.57695 4.57695 + -2.63731 -2.63731 + -2.18961 -2.18961 + -2.48184 -2.48184 + 0.0614645 0.0614645 + 0.377704 0.377704 + 2.34104 2.34104 + 0.657159 0.657159 + -1.25309 -1.25309 + -2.45413 -2.45413 + 1.88619 1.88619 + 1.74094 1.74094 + 2.3351 2.3351 + 0.212613 0.212613 + -1.16212 -1.16212 + 1.02027 1.02027 + 0.0929927 0.0929927 + 0.260815 0.260815 + -1.58322 -1.58322 + -1.5689 -1.5689 + 1.90734 1.90734 + -0.398199 -0.398199 + 1.1045 1.1045 + -1.90117 -1.90117 + -1.35457 -1.35457 + -0.904438 -0.904438 + -2.4261 -2.4261 + 2.24749 2.24749 + 0.293511 0.293511 + -1.96554 -1.96554 + -0.310607 -0.310607 + 0.399931 0.399931 + 1.06214 1.06214 + 1.14559 1.14559 + -1.29189 -1.29189 + 1.25441 1.25441 + 2.97262 2.97262 + -1.02313 -1.02313 + 1.05321 1.05321 + -0.804898 -0.804898 + 3.63862 3.63862 + 0.527294 0.527294 + -0.681347 -0.681347 + 1.11035 1.11035 + 2.40787 2.40787 + 1.19095 1.19095 + -0.380581 -0.380581 + 0.0582491 0.0582491 + 0.264974 0.264974 + 0.169384 0.169384 + 1.4731 1.4731 + 2.23289 2.23289 + 0.191901 0.191901 + -2.60952 -2.60952 + -2.62244 -2.62244 + 0.784122 0.784122 + 2.74349 2.74349 + -0.155218 -0.155218 + 0.967558 0.967558 + 0.186159 0.186159 + -0.204567 -0.204567 + -0.606278 -0.606278 + -0.011704 -0.011704 + 1.08489 1.08489 + -0.277437 -0.277437 + 0.0090858 0.0090858 + 0.258057 0.258057 + -0.373131 -0.373131 + -1.44058 -1.44058 + -1.01993 -1.01993 + -2.4798 -2.4798 + 0.271599 0.271599 + -1.84189 -1.84189 + -0.490777 -0.490777 + -0.414488 -0.414488 + -1.07757 -1.07757 + 1.23712 1.23712 + -2.02542 -2.02542 + -2.07667 -2.07667 + 2.89555 2.89555 + -2.61436 -2.61436 + -0.469252 -0.469252 + -1.29202 -1.29202 + -0.646527 -0.646527 + 0.00517958 0.00517958 + 0.5503 0.5503 + 1.06083 1.06083 + -0.324516 -0.324516 + -1.84483 -1.84483 + -0.574072 -0.574072 + -0.211102 -0.211102 + -0.585931 -0.585931 + 0.854918 0.854918 + -1.33179 -1.33179 + -1.87131 -1.87131 + 1.63465 1.63465 + 0.904734 0.904734 + -1.27273 -1.27273 + -2.41944 -2.41944 + 0.626935 0.626935 + -1.74987 -1.74987 + -3.06298 -3.06298 + -0.714794 -0.714794 + -1.00437 -1.00437 + 0.197116 0.197116 + 0.276584 0.276584 + 0.163489 0.163489 + 1.74588 1.74588 + -1.48997 -1.48997 + 0.561946 0.561946 + -0.164713 -0.164713 + -1.79587 -1.79587 + 0.681301 0.681301 + -0.303574 -0.303574 + 0.695659 0.695659 + 2.46479 2.46479 + 2.98348 2.98348 + 1.50751 1.50751 + 0.531878 0.531878 + -1.00327 -1.00327 + 1.42955 1.42955 + 1.41544 1.41544 + 0.631301 0.631301 + 0.887182 0.887182 + -0.689095 -0.689095 + 1.0004 1.0004 + 1.73016 1.73016 + 3.52498 3.52498 + 2.04091 2.04091 + 1.25617 1.25617 + -0.769574 -0.769574 + -0.439135 -0.439135 + -0.320473 -0.320473 + 1.88776 1.88776 + -3.08042 -3.08042 + 0.427227 0.427227 + 1.32274 1.32274 + -2.11399 -2.11399 + 3.3764 3.3764 + -0.943091 -0.943091 + 2.07374 2.07374 + -1.59418 -1.59418 + -4.37112 -4.37112 + 0.620227 0.620227 + -0.322782 -0.322782 + 0.447005 0.447005 + 0.280967 0.280967 + 1.68907 1.68907 + -1.60102 -1.60102 + 2.9697 2.9697 + -0.0128504 -0.0128504 + -0.782678 -0.782678 + 0.397837 0.397837 + -0.159312 -0.159312 + 1.14526 1.14526 + 0.0428197 0.0428197 + 1.77001 1.77001 + 1.08919 1.08919 + 0.55684 0.55684 + 0.398568 0.398568 + 1.27518 1.27518 + -1.85676 -1.85676 + -2.56394 -2.56394 + -0.357036 -0.357036 + 2.1851 2.1851 + -1.03051 -1.03051 + 0.922326 0.922326 + 0.654941 0.654941 + -0.902982 -0.902982 + -0.90051 -0.90051 + -0.127011 -0.127011 + 1.76007 1.76007 + 0.424547 0.424547 + -1.13683 -1.13683 + 1.83884 1.83884 + -0.918215 -0.918215 0.658109 0.658109 - 2.37478 2.37478 - -6.70679 -6.70679 - 6.08307 6.08307 - -29.6624 -29.6624 - 1.55578 1.55578 - 5.31311 5.31311 - -5.40681 -5.40681 - 1.80228 1.80228 - 4.50431 4.50431 - 7.25673 7.25673 - 5.89811 5.89811 - -2.92888 -2.92888 - 7.48853 7.48853 - -1.67318 -1.67318 - 0.974302 0.974302 - -8.10178 -8.10178 - 3.29435 3.29435 - -1.64519 -1.64519 - -7.08854 -7.08854 - 6.68891 6.68891 - -5.69927 -5.69927 - -3.51768 -3.51768 - 11.2895 11.2895 - -0.828568 -0.828568 - 5.53562 5.53562 - -0.358066 -0.358066 - -5.92559 -5.92559 - 4.39224 4.39224 - -5.1225 -5.1225 - -9.51174 -9.51174 - 9.80076 9.80076 - -1.85858 -1.85858 - 6.95181 6.95181 - -1.71297 -1.71297 - -0.275297 -0.275297 - -0.860135 -0.860135 - -0.484906 -0.484906 - 5.71425 5.71425 - 2.74639 2.74639 - -8.40417 -8.40417 - -1.84935 -1.84935 - 2.94526 2.94526 - 10.708 10.708 - 0.892511 0.892511 - -1.36773 -1.36773 - -7.25911 -7.25911 - 3.91428 3.91428 - -0.776027 -0.776027 - 3.44102 3.44102 - -4.87806 -4.87806 - 3.65101 3.65101 - -3.01077 -3.01077 - 1.17918 1.17918 - 5.82266 5.82266 - 8.52564 8.52564 - 4.35296 4.35296 - -2.94897 -2.94897 - -4.19366 -4.19366 - -4.7939 -4.7939 - 3.44038 3.44038 - -7.87089 -7.87089 - -3.18931 -3.18931 - -6.65708 -6.65708 - 1.09687 1.09687 - -4.36662 -4.36662 - 2.90783 2.90783 - 4.66889 4.66889 - -1.26146 -1.26146 - -2.01469 -2.01469 - -2.44566 -2.44566 - -2.15098 -2.15098 - 3.4006 3.4006 - 0.0396139 0.0396139 - 2.29469 2.29469 - -7.62709 -7.62709 - 7.18738 7.18738 - 1.45481 1.45481 - 2.37791 2.37791 - -5.37208 -5.37208 - -0.0612415 -0.0612415 - -1.46115 -1.46115 - 4.29624 4.29624 - 3.25993 3.25993 - 2.42986 2.42986 - 6.56133 6.56133 - -2.07349 -2.07349 - 5.61643 5.61643 - 5.48251 5.48251 - -0.703666 -0.703666 - -5.09456 -5.09456 - 0.57249 0.57249 - 4.28577 4.28577 - 2.468 2.468 - -10.013 -10.013 - -3.26046 -3.26046 - -7.91038 -7.91038 - -2.03302 -2.03302 - 3.49234 3.49234 - -1.2481 -1.2481 - -1.87417 -1.87417 - -1.93016 -1.93016 - 2.14307 2.14307 - -9.0722 -9.0722 - 2.03124 2.03124 - -0.938906 -0.938906 - 0.817464 0.817464 - 2.23636 2.23636 - 1.3076 1.3076 - 4.90629 4.90629 - 2.16603 2.16603 - 5.84398 5.84398 - -6.56748 -6.56748 - 7.22968 7.22968 - 0.664381 0.664381 - 11.2001 11.2001 - -4.98902 -4.98902 - 0.841822 0.841822 - -1.35522 -1.35522 - -2.43996 -2.43996 - 5.14732 5.14732 - -7.50974 -7.50974 - 5.73113 5.73113 - -2.72015 -2.72015 - -5.04474 -5.04474 - -13.1 -13.1 - 0.0777815 0.0777815 - 7.85631 7.85631 - -0.323243 -0.323243 - -2.97974 -2.97974 - 0.925187 0.925187 - 5.77219 5.77219 - 4.39868 4.39868 - 2.22326 2.22326 - 1.79052 1.79052 - -3.37507 -3.37507 - -4.08645 -4.08645 - 5.59349 5.59349 - 11.879 11.879 - -0.8099 -0.8099 - 16.6866 16.6866 - 2.85772 2.85772 - 3.73902 3.73902 - -0.406009 -0.406009 - 7.49033 7.49033 - -1.01733 -1.01733 - 4.03678 4.03678 - 4.91574 4.91574 - 14.6191 14.6191 - -1.18215 -1.18215 - -2.79895 -2.79895 - -5.16604 -5.16604 - -2.24596 -2.24596 - 1.83945 1.83945 - 1.72673 1.72673 - -23.2963 -23.2963 - -0.623748 -0.623748 - -2.8419 -2.8419 - 6.56374 6.56374 - 10.3431 10.3431 - 5.28302 5.28302 - 3.12716 3.12716 - 8.41242 8.41242 - 0.416003 0.416003 - -2.43236 -2.43236 - -1.63284 -1.63284 - 5.3806 5.3806 - 9.39975 9.39975 - 4.44496 4.44496 - -3.01441 -3.01441 - -1.33538 -1.33538 - 2.23541 2.23541 - -4.30131 -4.30131 - -1.20324 -1.20324 - 4.79406 4.79406 - 0.692551 0.692551 - -2.20403 -2.20403 - 0.12931 0.12931 - 0.842875 0.842875 - 0.29791 0.29791 - 6.59639 6.59639 - 8.6591 8.6591 - 2.07311 2.07311 - -6.48842 -6.48842 - 2.70007 2.70007 - -0.143695 -0.143695 - 3.99651 3.99651 - 6.86089 6.86089 - -2.54281 -2.54281 - -5.085 -5.085 - 3.61747 3.61747 - 2.09466 2.09466 - 3.35667 3.35667 - 7.38405 7.38405 - 0.816999 0.816999 - -0.564258 -0.564258 - 2.46281 2.46281 - -0.081471 -0.081471 - 12.0933 12.0933 - 9.45364 9.45364 - 0.303564 0.303564 - -2.20687 -2.20687 - 1.90101 1.90101 - -2.65606 -2.65606 - -11.3589 -11.3589 - -1.68249 -1.68249 - -1.25813 -1.25813 - -0.96125 -0.96125 - -2.84666 -2.84666 - 1.18914 1.18914 - 0.211945 0.211945 - -4.8988 -4.8988 - 0.894798 0.894798 - 3.9685 3.9685 - -0.852608 -0.852608 - 3.37537 3.37537 - -0.847579 -0.847579 - -4.37006 -4.37006 - -4.12787 -4.12787 - 4.37155 4.37155 - -7.86631 -7.86631 - -3.59755 -3.59755 - -2.55397 -2.55397 - 4.25921 4.25921 - 2.21721 2.21721 - 5.72299 5.72299 - 8.32362 8.32362 - 14.4057 14.4057 - 1.49376 1.49376 - 3.108 3.108 - -1.34388 -1.34388 - 3.77816 3.77816 - 5.69761 5.69761 - 0.255491 0.255491 - 4.15979 4.15979 - -14.6016 -14.6016 - 3.1475 3.1475 - 2.86732 2.86732 - -2.7875 -2.7875 - -8.78827 -8.78827 - -1.38068 -1.38068 - -2.74156 -2.74156 - -4.82257 -4.82257 - -4.64984 -4.64984 - -0.462036 -0.462036 - 2.36274 2.36274 - 2.73927 2.73927 - -4.01583 -4.01583 - -4.20256 -4.20256 - 7.33455 7.33455 - 7.53557 7.53557 - 3.2532 3.2532 - -0.556551 -0.556551 - 4.39618 4.39618 - 2.92025 2.92025 - -49.4395 -49.4395 - 1.84066 1.84066 - -6.03682 -6.03682 - 9.70956 9.70956 - 12.18 12.18 - -0.134471 -0.134471 - 0.388477 0.388477 - -4.30526 -4.30526 - 3.98614 3.98614 - -3.20351 -3.20351 - 3.81764 3.81764 - 5.34853 5.34853 - 0.382215 0.382215 - -0.473372 -0.473372 - -4.4073 -4.4073 - -10.1129 -10.1129 - -6.82482 -6.82482 - 5.39935 5.39935 - -0.664077 -0.664077 - 7.75577 7.75577 - -5.565 -5.565 - -2.28518 -2.28518 - -3.09472 -3.09472 - 6.0196 6.0196 - -1.32035 -1.32035 - 2.5721 2.5721 - -9.0201 -9.0201 - 6.87621 6.87621 - 7.57662 7.57662 - -2.42131 -2.42131 - -7.11 -7.11 - 1.5457 1.5457 - 1.38686 1.38686 - -1.67077 -1.67077 - 5.34357 5.34357 - -5.22992 -5.22992 - -5.50112 -5.50112 - -0.820436 -0.820436 - -6.85987 -6.85987 - 4.36935 4.36935 - 8.27737 8.27737 - 7.16613 7.16613 - 7.21538 7.21538 - 0.0297893 0.0297893 - -3.30991 -3.30991 - 1.18508 1.18508 - -0.745072 -0.745072 - -1.31153 -1.31153 - -2.57184 -2.57184 - -0.187369 -0.187369 - 6.79233 6.79233 - 8.04294 8.04294 - 3.06986 3.06986 - -5.13761 -5.13761 - 0.539648 0.539648 - 5.02007 5.02007 - 2.67737 2.67737 - -6.69984 -6.69984 - 6.76321 6.76321 - 6.25102 6.25102 - 3.80545 3.80545 - -2.16059 -2.16059 - 2.81803 2.81803 - 0.447194 0.447194 - 1.84756 1.84756 - -6.42528 -6.42528 - -2.23379 -2.23379 - -2.61151 -2.61151 - -2.86143 -2.86143 - -2.94039 -2.94039 - -3.38503 -3.38503 - 0.474985 0.474985 - -9.66389 -9.66389 - 4.96293 4.96293 - -5.6718 -5.6718 - 7.06422 7.06422 - -8.36354 -8.36354 - 0.0182466 0.0182466 - 9.20883 9.20883 - 8.23981 8.23981 - -1.41968 -1.41968 - -1.36057 -1.36057 - -3.99568 -3.99568 - 2.51484 2.51484 - 5.41846 5.41846 - -10.8511 -10.8511 - -8.41267 -8.41267 - 2.04668 2.04668 - -5.61525 -5.61525 - -9.73507 -9.73507 - -0.497102 -0.497102 - 4.29467 4.29467 - -1.61424 -1.61424 - -0.818494 -0.818494 - -7.02135 -7.02135 - 13.4836 13.4836 - -4.10115 -4.10115 - -8.11914 -8.11914 - -2.79895 -2.79895 + -0.365433 -0.365433 + -2.18908 -2.18908 + -1.93223 -1.93223 + 0.408743 0.408743 + -1.92079 -1.92079 + 0.0976185 0.0976185 + 4.67569 4.67569 + -1.49021 -1.49021 + 0.176745 0.176745 + -1.03095 -1.03095 + -2.1574 -2.1574 + -0.593898 -0.593898 + 1.01641 1.01641 + -0.28545 -0.28545 + -2.17595 -2.17595 + 1.00962 1.00962 + 0.592646 0.592646 + -0.343818 -0.343818 + -0.440034 -0.440034 + 1.38711 1.38711 + -1.43068 -1.43068 + 0.681063 0.681063 + -1.94052 -1.94052 + -2.38281 -2.38281 + -0.789486 -0.789486 + -1.35709 -1.35709 + -1.69084 -1.69084 + -1.62863 -1.62863 + -0.284129 -0.284129 + -2.83546 -2.83546 + -0.915304 -0.915304 + 0.257879 0.257879 + 0.745803 0.745803 + -0.134261 -0.134261 + -0.00297398 -0.00297398 + -0.452233 -0.452233 + -2.36279 -2.36279 + 1.21461 1.21461 + -1.13373 -1.13373 + -0.329279 -0.329279 + -1.64147 -1.64147 + -0.579568 -0.579568 + -0.783701 -0.783701 + 0.606064 0.606064 + 0.249187 0.249187 + -0.266901 -0.266901 + 5.9043 5.9043 + -0.630335 -0.630335 + -0.37167 -0.37167 + 2.66605 2.66605 + -1.36786 -1.36786 + 0.452445 0.452445 + -1.28971 -1.28971 + 2.37668 2.37668 + 2.52488 2.52488 + 0.142467 0.142467 + 1.68463 1.68463 + 1.30605 1.30605 + -3.62793 -3.62793 + 2.48566 2.48566 + -1.13421 -1.13421 + 0.0926649 0.0926649 + -3.63194 -3.63194 + 0.108222 0.108222 + -1.92097 -1.92097 + -0.609085 -0.609085 + -2.23882 -2.23882 + 1.30409 1.30409 + 0.251002 0.251002 + 0.820516 0.820516 + 1.14979 1.14979 + 4.24577 4.24577 + 0.195344 0.195344 + -0.322737 -0.322737 + 0.93393 0.93393 + 2.81175 2.81175 + 0.446439 0.446439 + -1.38935 -1.38935 + 0.386624 0.386624 + 0.464916 0.464916 + -1.58662 -1.58662 + -1.12035 -1.12035 + 1.79443 1.79443 + 1.03373 1.03373 + 1.24398 1.24398 + -0.343199 -0.343199 + 0.778172 0.778172 + 3.7689 3.7689 + 1.15726 1.15726 + 0.0548858 0.0548858 + 1.46606 1.46606 + -0.0390753 -0.0390753 + 0.525417 0.525417 + -0.399366 -0.399366 + -0.389525 -0.389525 + 1.53548 1.53548 + -0.660596 -0.660596 + 1.02893 1.02893 + 5.36388 5.36388 + -0.158376 -0.158376 + -0.00158735 -0.00158735 + 1.48366 1.48366 + 1.58287 1.58287 + -1.32445 -1.32445 + 1.18222 1.18222 + 0.859016 0.859016 + -0.366681 -0.366681 + -1.97608 -1.97608 + -1.60289 -1.60289 + 0.801427 0.801427 + 0.734912 0.734912 + -0.655545 -0.655545 + -3.4244 -3.4244 + 1.79295 1.79295 + 1.64987 1.64987 + -0.372111 -0.372111 + -0.818163 -0.818163 + -1.73224 -1.73224 + 1.07321 1.07321 + -1.09122 -1.09122 + -2.31512 -2.31512 + -1.51542 -1.51542 + -0.366266 -0.366266 + 2.78864 2.78864 + -1.79099 -1.79099 + -0.527644 -0.527644 + 0.617675 0.617675 + 0.817341 0.817341 + -1.59739 -1.59739 + -0.942853 -0.942853 + -0.024404 -0.024404 + 0.838706 0.838706 + 2.69883 2.69883 + -1.84305 -1.84305 + 0.197916 0.197916 + 1.40902 1.40902 + 0.780807 0.780807 + -0.447145 -0.447145 + 0.39841 0.39841 + -0.568937 -0.568937 + 3.35843 3.35843 + 0.244485 0.244485 + -0.409614 -0.409614 + 1.79979 1.79979 + -0.114665 -0.114665 + 2.12529 2.12529 + -1.1328 -1.1328 + -1.32568 -1.32568 + -1.33014 -1.33014 + 0.61469 0.61469 + 0.23576 0.23576 + -1.58669 -1.58669 + 0.882018 0.882018 + -0.199537 -0.199537 + 0.0213817 0.0213817 + -0.0526741 -0.0526741 + -1.90557 -1.90557 + -0.861437 -0.861437 + -1.73529 -1.73529 + -1.20074 -1.20074 + 1.52766 1.52766 + 0.229589 0.229589 + 0.371437 0.371437 + -2.27666 -2.27666 + 0.500965 0.500965 + 0.342153 0.342153 + 1.54886 1.54886 + -2.35151 -2.35151 + 0.369683 0.369683 + 0.537004 0.537004 + 0.462267 0.462267 + 0.104486 0.104486 + 1.76568 1.76568 + 0.66308 0.66308 + -0.610417 -0.610417 + -1.19933 -1.19933 + 0.458763 0.458763 + 1.12317 1.12317 + 0.639384 0.639384 + 0.420247 0.420247 + -0.516877 -0.516877 + 1.85253 1.85253 + -1.28899 -1.28899 + -1.2008 -1.2008 + 0.55433 0.55433 + 0.209396 0.209396 + -1.13624 -1.13624 + -1.29488 -1.29488 + -0.990851 -0.990851 + -1.82754 -1.82754 + -1.10815 -1.10815 + 0.268715 0.268715 + -1.8783 -1.8783 + 1.46547 1.46547 + 2.47695 2.47695 + -2.0124 -2.0124 + -2.23708 -2.23708 + 2.61164 2.61164 + 0.260522 0.260522 + 0.570767 0.570767 + 2.82558 2.82558 + 1.21898 1.21898 + -2.1909 -2.1909 + 1.95577 1.95577 + -0.334229 -0.334229 + 0.490568 0.490568 + 0.476326 0.476326 + -2.85789 -2.85789 + -1.67129 -1.67129 + -3.44224 -3.44224 + 1.39022 1.39022 + 1.13294 1.13294 + 1.12937 1.12937 + 1.55696 1.55696 + 0.641922 0.641922 + 3.02277 3.02277 + 0.727429 0.727429 + 2.29113 2.29113 + -0.345511 -0.345511 + -0.963097 -0.963097 + 2.04846 2.04846 + 1.82306 1.82306 + -0.10738 -0.10738 + 1.47278 1.47278 + 0.44212 0.44212 + -0.664988 -0.664988 + -0.522717 -0.522717 + 0.015111 0.015111 + -1.17036 -1.17036 + 0.533411 0.533411 + -0.755567 -0.755567 + 1.3441 1.3441 + 2.34532 2.34532 + 1.54432 1.54432 + 0.346052 0.346052 + -0.317292 -0.317292 + -0.419647 -0.419647 + 0.778187 0.778187 + 2.52819 2.52819 + 1.71307 1.71307 + 0.108402 0.108402 + -0.286356 -0.286356 + -0.184754 -0.184754 + -2.0023 -2.0023 + -1.88134 -1.88134 + -0.401823 -0.401823 + -0.103521 -0.103521 + 0.753005 0.753005 + -0.803658 -0.803658 + 2.44367 2.44367 + 1.58612 1.58612 + -0.94574 -0.94574 + 0.269329 0.269329 + 2.60832 2.60832 + -0.113991 -0.113991 + 2.23152 2.23152 + -1.14749 -1.14749 + -1.86227 -1.86227 + -2.06391 -2.06391 + -0.960271 -0.960271 + -0.953305 -0.953305 + -0.162597 -0.162597 + 0.405977 0.405977 + 1.18975 1.18975 + -2.68021 -2.68021 + 0.557723 0.557723 + 0.1416 0.1416 + -2.27623 -2.27623 + -0.91957 -0.91957 + 1.4132 1.4132 + 2.40127 2.40127 + 0.364167 0.364167 + 1.67016 1.67016 + 0.345176 0.345176 + -0.885645 -0.885645 + -1.92508 -1.92508 + -0.527327 -0.527327 + 0.59694 0.59694 + -0.928777 -0.928777 + -2.45285 -2.45285 + 0.093849 0.093849 + 0.875405 0.875405 + 0.734934 0.734934 + -1.58165 -1.58165 + -0.43735 -0.43735 + 0.103331 0.103331 + 1.57271 1.57271 + -0.670804 -0.670804 + 3.11503 3.11503 + 2.03147 2.03147 + -1.12063 -1.12063 + 1.76581 1.76581 + -1.91044 -1.91044 + -0.334989 -0.334989 + -0.115597 -0.115597 + 2.82 2.82 + 2.8363 2.8363 + -2.12753 -2.12753 + 0.551957 0.551957 + 0.778678 0.778678 + 0.26151 0.26151 + 0.527825 0.527825 + -1.24336 -1.24336 + -1.28699 -1.28699 + -0.227357 -0.227357 + 4.99396 4.99396 + -1.90422 -1.90422 + -0.883407 -0.883407 + 3.10147 3.10147 + 0.33776 0.33776 + 0.12595 0.12595 + -0.147078 -0.147078 + 0.40136 0.40136 + 0.092439 0.092439 + -0.903678 -0.903678 + 0.786246 0.786246 + 1.43764 1.43764 -4.39428 -4.39428 - -0.737467 -0.737467 - 1.37013 1.37013 - 9.56244 9.56244 - 2.92491 2.92491 - -7.13393 -7.13393 - -0.179291 -0.179291 - -6.00313 -6.00313 - 7.27104 7.27104 - -1.7103 -1.7103 - -7.84843 -7.84843 - 13.7304 13.7304 - 2.40973 2.40973 - -7.07755 -7.07755 - 1.31745 1.31745 - -9.99271 -9.99271 - -15.4753 -15.4753 - 4.38711 4.38711 - -5.41127 -5.41127 - -1.06491 -1.06491 - 1.09245 1.09245 - -1.33961 -1.33961 - -4.42681 -4.42681 - -4.44164 -4.44164 - -1.80772 -1.80772 - -5.06035 -5.06035 - 0.197369 0.197369 - 7.27798 7.27798 - -6.88382 -6.88382 - 3.21319 3.21319 - 8.04111 8.04111 - -3.94107 -3.94107 - 1.79716 1.79716 - -0.2134 -0.2134 - 1.36955 1.36955 - 13.7009 13.7009 - -7.3497 -7.3497 - 1.80078 1.80078 - 4.25352 4.25352 - -2.80092 -2.80092 - -3.81295 -3.81295 - -4.92036 -4.92036 - 0.856001 0.856001 - -1.26696 -1.26696 - 2.65207 2.65207 - -1.01876 -1.01876 - 1.50837 1.50837 - -11.5335 -11.5335 - 5.80989 5.80989 - 2.45606 2.45606 - 1.64394 1.64394 - 2.73651 2.73651 - -11.1653 -11.1653 - -1.66359 -1.66359 - -0.0317267 -0.0317267 - 0.115458 0.115458 - 4.43585 4.43585 - 1.24902 1.24902 - 7.30894 7.30894 - 16.7814 16.7814 - -0.456154 -0.456154 - -3.94033 -3.94033 - -4.4947 -4.4947 - -2.52048 -2.52048 - 0.0890704 0.0890704 - -4.66338 -4.66338 - 3.88142 3.88142 - 2.35984 2.35984 - 4.84037 4.84037 - 6.95444 6.95444 - 2.74408 2.74408 - -3.23958 -3.23958 - -0.467292 -0.467292 - 6.26367 6.26367 - -1.50588 -1.50588 - 4.13389 4.13389 - -2.53819 -2.53819 - -4.4987 -4.4987 - -10.3487 -10.3487 - -14.8297 -14.8297 - -8.48112 -8.48112 - 3.95155 3.95155 - 1.2289 1.2289 - -4.38025 -4.38025 - -0.61687 -0.61687 - 10.8511 10.8511 - 1.15556 1.15556 - -2.19768 -2.19768 - -7.66931 -7.66931 - 4.72919 4.72919 - -7.6738 -7.6738 - -0.688528 -0.688528 - 4.74928 4.74928 - 4.92126 4.92126 - 0.897546 0.897546 - 3.85735 3.85735 - 0.201364 0.201364 - -5.62425 -5.62425 + 1.17994 1.17994 + 0.485781 0.485781 + 1.63216 1.63216 + -1.36006 -1.36006 + 0.773674 0.773674 + 0.618503 0.618503 + -0.0300672 -0.0300672 + 0.484393 0.484393 + -0.874462 -0.874462 + -2.0849 -2.0849 + -0.576099 -0.576099 + 0.765223 0.765223 -3.83117 -3.83117 - 4.05866 4.05866 - 3.10063 3.10063 - 2.5224 2.5224 - -1.51274 -1.51274 - -0.683338 -0.683338 - -3.23147 -3.23147 - -4.21268 -4.21268 - -2.21401 -2.21401 - 1.57887 1.57887 - 0.848257 0.848257 - -5.83704 -5.83704 - -7.00011 -7.00011 - 3.16884 3.16884 - -4.44161 -4.44161 - -7.62482 -7.62482 - -0.266943 -0.266943 - 0.41761 0.41761 - -7.45144 -7.45144 - -0.211132 -0.211132 - 0.276707 0.276707 - 16.7781 16.7781 - 0.689757 0.689757 - -3.04049 -3.04049 - 2.91684 2.91684 - 1.97161 1.97161 - 3.7721 3.7721 - -1.60698 -1.60698 - -4.18868 -4.18868 - 7.66491 7.66491 - -0.64664 -0.64664 - -0.660623 -0.660623 - 8.68174 8.68174 - 0.282074 0.282074 - -2.85266 -2.85266 - -1.91293 -1.91293 - 7.18736 7.18736 - -10.3875 -10.3875 - -1.91603 -1.91603 - 6.29739 6.29739 - -0.0375388 -0.0375388 - -1.60576 -1.60576 - -3.22148 -3.22148 - -4.24549 -4.24549 - 1.30822 1.30822 - 2.52307 2.52307 - 0.403345 0.403345 - -0.744478 -0.744478 - 2.41241 2.41241 - -4.58098 -4.58098 - -0.791842 -0.791842 - 3.73626 3.73626 - -1.43002 -1.43002 - 4.30716 4.30716 - 3.30255 3.30255 - -4.08011 -4.08011 - -5.07282 -5.07282 - -1.54759 -1.54759 - -2.2305 -2.2305 - 6.8791 6.8791 - 9.7396 9.7396 - -6.50395 -6.50395 - 3.57178 3.57178 - 7.08987 7.08987 - 6.2669 6.2669 - 5.87329 5.87329 - 2.36823 2.36823 - -6.16 -6.16 - 1.96238 1.96238 - 7.31651 7.31651 - -1.5257 -1.5257 - -2.89061 -2.89061 - 0.407546 0.407546 - 5.10645 5.10645 - 11.0716 11.0716 - 4.7443 4.7443 - -8.77353 -8.77353 - -0.631177 -0.631177 - -4.36973 -4.36973 - 1.48666 1.48666 - 7.7678 7.7678 - -2.65407 -2.65407 - 4.56869 4.56869 - -0.541163 -0.541163 - 2.89543 2.89543 - 5.39424 5.39424 - -3.62954 -3.62954 - 3.77547 3.77547 - -5.96886 -5.96886 - -4.38947 -4.38947 - -2.96756 -2.96756 - 2.28222 2.28222 - -1.08489 -1.08489 - 1.74726 1.74726 - -3.46088 -3.46088 - 11.9371 11.9371 - -5.02359 -5.02359 - 2.51632 2.51632 - -0.0297022 -0.0297022 - -2.60011 -2.60011 - 0.254202 0.254202 - 9.7949 9.7949 - 3.64937 3.64937 - 10.0857 10.0857 - -5.36637 -5.36637 - 4.11127 4.11127 - 8.90571 8.90571 - -5.97219 -5.97219 - -7.21379 -7.21379 - -5.01561 -5.01561 - 2.98616 2.98616 - 1.99064 1.99064 - 0.16465 0.16465 - -4.07902 -4.07902 - 4.34018 4.34018 - -2.13528 -2.13528 - 2.39903 2.39903 - 4.00804 4.00804 - -1.85741 -1.85741 - -7.73083 -7.73083 - -4.21139 -4.21139 - 4.65743 4.65743 - 0.963549 0.963549 - 0.29506 0.29506 - 6.05798 6.05798 - 12.4428 12.4428 - -0.398651 -0.398651 - -0.584559 -0.584559 - 2.75445 2.75445 - -0.207975 -0.207975 - 6.11926 6.11926 - -8.66125 -8.66125 - 3.07568 3.07568 - -3.19358 -3.19358 - -2.53024 -2.53024 - 14.1187 14.1187 - -0.412049 -0.412049 - 12.5809 12.5809 - 6.26236 6.26236 - 5.23037 5.23037 - -0.11356 -0.11356 - -6.62321 -6.62321 - -1.29651 -1.29651 - -1.48734 -1.48734 - 13.0753 13.0753 - 4.21767 4.21767 - -2.4425 -2.4425 - -0.0901323 -0.0901323 - 9.79684 9.79684 - 4.74522 4.74522 - -3.34804 -3.34804 - 7.37816 7.37816 - 2.57938 2.57938 - 1.92968 1.92968 - 3.75166 3.75166 - 5.0617 5.0617 - 8.74324 8.74324 - -0.93703 -0.93703 - -1.36031 -1.36031 - -2.5439 -2.5439 - 1.56784 1.56784 - 2.56237 2.56237 - -1.02578 -1.02578 - 6.62085 6.62085 - 7.69745 7.69745 - 6.26864 6.26864 - -4.20046 -4.20046 - -2.30926 -2.30926 - 2.74598 2.74598 - 4.11078 4.11078 - 2.8455 2.8455 - -3.45407 -3.45407 - 2.82327 2.82327 - -1.00356 -1.00356 - 8.85974 8.85974 - 6.35864 6.35864 - -1.59146 -1.59146 - -0.361996 -0.361996 - -1.25198 -1.25198 - 8.2867 8.2867 - 0.981644 0.981644 - 2.68003 2.68003 - 1.10236 1.10236 - -1.63423 -1.63423 - -2.79552 -2.79552 - -6.5718 -6.5718 - -0.257779 -0.257779 - -4.49325 -4.49325 - 5.0455 5.0455 - 14.4508 14.4508 - 3.60407 3.60407 - 3.09003 3.09003 - -8.32962 -8.32962 - -1.41178 -1.41178 - 12.5777 12.5777 - -2.01342 -2.01342 - -1.48205 -1.48205 - 0.967158 0.967158 - -0.532548 -0.532548 - -5.23274 -5.23274 - -1.49702 -1.49702 - 0.739607 0.739607 - 3.49171 3.49171 - -1.0507 -1.0507 - -7.48299 -7.48299 - 7.57395 7.57395 - -3.04813 -3.04813 - 16.322 16.322 - 7.81441 7.81441 - -3.41529 -3.41529 - 2.05401 2.05401 - 1.08232 1.08232 - 12.5735 12.5735 - 0.126572 0.126572 - -6.92158 -6.92158 - -1.4651 -1.4651 - -3.19425 -3.19425 - -1.44093 -1.44093 - -3.82056 -3.82056 - 6.72914 6.72914 - -5.46583 -5.46583 - -1.43396 -1.43396 - 7.42164 7.42164 - 1.00438 1.00438 - -0.41415 -0.41415 - -2.54987 -2.54987 - 6.88491 6.88491 - 3.84807 3.84807 - -5.62245 -5.62245 - 5.24133 5.24133 - 7.99514 7.99514 - -2.51593 -2.51593 - 8.19568 8.19568 - 0.854985 0.854985 - -6.20478 -6.20478 - -2.58235 -2.58235 - -6.51346 -6.51346 - 12.8877 12.8877 - 8.6194 8.6194 - -6.82669 -6.82669 - -4.67379 -4.67379 - 8.13137 8.13137 - 0.733511 0.733511 - 5.66079 5.66079 - -2.94337 -2.94337 - -3.29462 -3.29462 - -6.3809 -6.3809 - -1.85613 -1.85613 - 0.635069 0.635069 - 0.432626 0.432626 - -14.6426 -14.6426 - 8.05825 8.05825 - 6.50637 6.50637 - 1.44014 1.44014 - -4.60602 -4.60602 - -6.49137 -6.49137 - 6.33163 6.33163 - -1.97616 -1.97616 - 0.573379 0.573379 - -2.78039 -2.78039 - -0.140087 -0.140087 - 1.52619 1.52619 - 6.83379 6.83379 - -0.197981 -0.197981 - -3.00849 -3.00849 - -2.09725 -2.09725 - -2.06883 -2.06883 - -0.328198 -0.328198 - -0.212338 -0.212338 - 5.4425 5.4425 - 6.48574 6.48574 - 2.00073 2.00073 - -3.15642 -3.15642 - -0.0673389 -0.0673389 - -4.19911 -4.19911 - 4.5466 4.5466 - 3.73221 3.73221 - -1.01059 -1.01059 - -4.29015 -4.29015 - 4.9909 4.9909 - 3.22397 3.22397 - -1.27984 -1.27984 - 2.83358 2.83358 - 2.25695 2.25695 - 7.2879 7.2879 - -1.47955 -1.47955 - 12.7627 12.7627 - -3.72449 -3.72449 - 3.97719 3.97719 - 14.2197 14.2197 - -1.24031 -1.24031 - -7.41824 -7.41824 - 1.90207 1.90207 - 1.10939 1.10939 - -7.47202 -7.47202 - 3.85738 3.85738 - -4.12085 -4.12085 - 1.12097 1.12097 - -0.545646 -0.545646 - 3.04129 3.04129 - 1.05043 1.05043 - 0.993448 0.993448 - -5.78424 -5.78424 - -1.97199 -1.97199 - -5.74806 -5.74806 - 2.70835 2.70835 - -8.09729 -8.09729 - -6.36035 -6.36035 - -1.24361 -1.24361 - -2.44813 -2.44813 - 7.48353 7.48353 - 2.0202 2.0202 - 3.04366 3.04366 - -3.98778 -3.98778 - 4.80106 4.80106 - 0.926552 0.926552 - 3.35253 3.35253 - -4.10577 -4.10577 - -3.57853 -3.57853 - 4.03372 4.03372 - -2.38792 -2.38792 - 0.12177 0.12177 - -0.761671 -0.761671 - -4.25652 -4.25652 - 7.27933 7.27933 - 0.165182 0.165182 - 1.34367 1.34367 - -7.36923 -7.36923 - 2.38548 2.38548 - 0.117217 0.117217 - 2.02002 2.02002 - -4.60023 -4.60023 - 2.78 2.78 - -1.34604 -1.34604 - 4.7234 4.7234 - 7.37673 7.37673 - 2.07986 2.07986 - -5.72573 -5.72573 - -6.66143 -6.66143 - 2.43072 2.43072 - 1.34782 1.34782 - -0.114238 -0.114238 - 2.32103 2.32103 - 1.84042 1.84042 - 1.07005 1.07005 - 3.88182 3.88182 - -0.752264 -0.752264 - -2.43517 -2.43517 - -5.29216 -5.29216 - -0.13527 -0.13527 - 1.40188 1.40188 - -5.87815 -5.87815 - -1.90167 -1.90167 - 2.88562 2.88562 - -2.29028 -2.29028 - 2.35477 2.35477 - -3.50731 -3.50731 - 6.0621 6.0621 - 3.2011 3.2011 - 2.19115 2.19115 - -3.03557 -3.03557 - -8.49394 -8.49394 - 0.936501 0.936501 - 7.19188 7.19188 - 4.50162 4.50162 - 0.341394 0.341394 - 2.54484 2.54484 - 1.67305 1.67305 - 3.05008 3.05008 - -2.0266 -2.0266 - 7.28431 7.28431 - -7.70924 -7.70924 - 2.60851 2.60851 - 6.8054 6.8054 - 1.8878 1.8878 - 1.87624 1.87624 - -5.13611 -5.13611 - -3.23698 -3.23698 - 4.03201 4.03201 - -5.27165 -5.27165 - -4.95817 -4.95817 - -0.200461 -0.200461 - 4.27259 4.27259 - 0.449661 0.449661 - 7.49752 7.49752 - -5.47923 -5.47923 - -2.40934 -2.40934 - 25.0066 25.0066 - -3.14511 -3.14511 - -1.62587 -1.62587 - -1.67652 -1.67652 - -2.17888 -2.17888 - 2.37296 2.37296 - -4.41408 -4.41408 - 0.65204 0.65204 - 10.849 10.849 - -2.3021 -2.3021 - 2.20417 2.20417 - 10.0579 10.0579 - -4.03489 -4.03489 - 7.60982 7.60982 - -5.74951 -5.74951 - -2.97582 -2.97582 - -8.61382 -8.61382 - -1.90903 -1.90903 - -3.64556 -3.64556 - -16.2304 -16.2304 - -15.9793 -15.9793 - -4.59448 -4.59448 - -2.67688 -2.67688 - -1.67148 -1.67148 - 5.57026 5.57026 - 0.846445 0.846445 - -7.54149 -7.54149 - -3.61401 -3.61401 - 4.03723 4.03723 - 0.711821 0.711821 - 8.99009 8.99009 - -6.15866 -6.15866 - -1.36865 -1.36865 - -4.31058 -4.31058 - 6.31659 6.31659 - -6.23773 -6.23773 - 0.857388 0.857388 - 3.6152 3.6152 - -1.28774 -1.28774 - -4.92094 -4.92094 - 3.08527 3.08527 - -5.74582 -5.74582 - -4.20897 -4.20897 - -5.19406 -5.19406 - -4.06851 -4.06851 - 5.73867 5.73867 - 3.32767 3.32767 - -11.2588 -11.2588 - -7.94126 -7.94126 - 5.38746 5.38746 - -0.0253579 -0.0253579 - -1.7856 -1.7856 - -1.31209 -1.31209 - 6.85519 6.85519 - 2.71496 2.71496 - -2.58838 -2.58838 - -6.86996 -6.86996 - 1.01204 1.01204 - 3.43433 3.43433 - -0.249192 -0.249192 - 7.96322 7.96322 - 14.3414 14.3414 - 2.44774 2.44774 - 4.73731 4.73731 - -9.14288 -9.14288 - 2.70325 2.70325 - 6.48202 6.48202 - -2.58391 -2.58391 - -4.52079 -4.52079 - -0.64105 -0.64105 - -3.75531 -3.75531 - -3.93321 -3.93321 - -2.5879 -2.5879 - 2.34697 2.34697 - -3.89721 -3.89721 - -1.60712 -1.60712 - -7.49452 -7.49452 - -0.518596 -0.518596 - 0.996693 0.996693 - 2.83468 2.83468 - -6.19363 -6.19363 - -7.25683 -7.25683 - 0.391546 0.391546 - -7.52756 -7.52756 - -0.810817 -0.810817 - -2.64942 -2.64942 - -2.95081 -2.95081 - -6.34989 -6.34989 - 3.9961 3.9961 - 1.36755 1.36755 - -0.335808 -0.335808 - -11.7919 -11.7919 - 1.16904 1.16904 - 6.26031 6.26031 - -4.68064 -4.68064 - 5.55008 5.55008 - 3.65873 3.65873 - -3.95177 -3.95177 - 7.62708 7.62708 - -2.4932 -2.4932 - -0.713266 -0.713266 - 6.76214 6.76214 - -0.802523 -0.802523 - -0.327543 -0.327543 - -6.9053 -6.9053 - -2.69604 -2.69604 - 9.729 9.729 - -7.61691 -7.61691 - -0.658653 -0.658653 - 1.62531 1.62531 - 0.532107 0.532107 - 1.71729 1.71729 - -10.1795 -10.1795 - 5.54208 5.54208 - 4.02502 4.02502 - -1.47596 -1.47596 - 11.818 11.818 - 4.40414 4.40414 - 5.64827 5.64827 - 5.89386 5.89386 - -6.19187 -6.19187 - 4.77889 4.77889 - -0.261731 -0.261731 - -0.570525 -0.570525 - 3.80941 3.80941 - -3.95414 -3.95414 - 0.642971 0.642971 - -7.23493 -7.23493 - 0.744423 0.744423 - 11.5682 11.5682 - -3.17145 -3.17145 - 9.02877 9.02877 - 10.5452 10.5452 - -7.05642 -7.05642 - -6.01952 -6.01952 - -5.61355 -5.61355 - 1.28759 1.28759 - 3.44186 3.44186 - -2.52363 -2.52363 - 8.95712 8.95712 - -1.33999 -1.33999 - -3.25858 -3.25858 - 2.33509 2.33509 - 2.16314 2.16314 - 14.4002 14.4002 - -5.22345 -5.22345 - -5.6232 -5.6232 - -4.20801 -4.20801 - 0.677359 0.677359 - 1.92688 1.92688 - 2.4265 2.4265 - -3.47901 -3.47901 - -3.35004 -3.35004 - -5.32445 -5.32445 - 0.817822 0.817822 - 5.9241 5.9241 - 2.13342 2.13342 - 9.30726 9.30726 - -6.00328 -6.00328 - 5.10125 5.10125 - 16.6941 16.6941 - -1.41774 -1.41774 - 0.843709 0.843709 - 3.71326 3.71326 - -12.7315 -12.7315 - -1.58947 -1.58947 - 2.7713 2.7713 - -5.89993 -5.89993 - -10.1427 -10.1427 - -1.60823 -1.60823 - -4.98621 -4.98621 - -10.6258 -10.6258 - 0.255858 0.255858 - 5.87781 5.87781 - 0.549239 0.549239 - -0.361649 -0.361649 - 2.89543 2.89543 - -1.56252 -1.56252 - -7.04269 -7.04269 - 0.360599 0.360599 - -0.80318 -0.80318 - -8.15537 -8.15537 - 7.86106 7.86106 - 4.25906 4.25906 - 1.78474 1.78474 - 4.15764 4.15764 - -1.8884 -1.8884 - -7.16959 -7.16959 - 2.84539 2.84539 - -3.33161 -3.33161 - 4.89863 4.89863 - -3.36503 -3.36503 - -4.68013 -4.68013 - 5.18058 5.18058 - -9.69276 -9.69276 - -1.56116 -1.56116 - -3.58275 -3.58275 - -2.73766 -2.73766 - 6.64492 6.64492 - -3.78966 -3.78966 - 2.63467 2.63467 - -12.4868 -12.4868 - -3.4241 -3.4241 - 3.2898 3.2898 - 2.20265 2.20265 - -1.36672 -1.36672 - 2.71448 2.71448 - 5.87839 5.87839 - 0.160837 0.160837 - -2.64458 -2.64458 - -3.8078 -3.8078 - 5.08743 5.08743 - -14.014 -14.014 - 4.44746 4.44746 - 6.61584 6.61584 - -0.916513 -0.916513 - -8.08277 -8.08277 - -8.088 -8.088 - -5.14152 -5.14152 - -4.30739 -4.30739 - -8.76727 -8.76727 - -4.53313 -4.53313 - 11.0356 11.0356 - -2.37348 -2.37348 - -8.71711 -8.71711 - -2.22971 -2.22971 - 8.19346 8.19346 - -0.330962 -0.330962 - 1.10067 1.10067 - 1.01878 1.01878 - -10.2666 -10.2666 - 8.15909 8.15909 - 9.09316 9.09316 - -0.862864 -0.862864 - -7.54443 -7.54443 - -3.44703 -3.44703 - 5.21819 5.21819 - -2.06834 -2.06834 - 9.55442 9.55442 - -1.89649 -1.89649 - -5.57892 -5.57892 - 4.22421 4.22421 - -4.06375 -4.06375 - 3.81452 3.81452 - 3.09071 3.09071 - -7.34297 -7.34297 - -1.67899 -1.67899 - 0.58489 0.58489 - -5.33824 -5.33824 - 2.82705 2.82705 - -3.70864 -3.70864 - 4.21641 4.21641 - 3.82508 3.82508 - -4.04356 -4.04356 - 20.0249 20.0249 - -13.1531 -13.1531 - 2.98603 2.98603 - 5.54713 5.54713 - -1.39722 -1.39722 - 2.13016 2.13016 - -2.40215 -2.40215 - 0.168123 0.168123 - 2.77021 2.77021 - -2.32327 -2.32327 - -1.06731 -1.06731 - 2.53877 2.53877 - -1.94325 -1.94325 - 1.47106 1.47106 - 0.294436 0.294436 - -0.547055 -0.547055 - 0.116016 0.116016 - 1.56148 1.56148 - 3.21789 3.21789 - -2.89007 -2.89007 - -4.33765 -4.33765 - 0.566163 0.566163 - 0.402729 0.402729 - -7.80674 -7.80674 - 4.72058 4.72058 - 3.97584 3.97584 - 1.91646 1.91646 - 2.09298 2.09298 - 1.88552 1.88552 - -2.37581 -2.37581 - -18.2615 -18.2615 - 2.68651 2.68651 - 5.5 5.5 - 0.355051 0.355051 - 5.6052 5.6052 - 7.74854 7.74854 - -0.512378 -0.512378 - 1.60299 1.60299 - -5.49563 -5.49563 - -1.96455 -1.96455 - -16.3228 -16.3228 - -6.87737 -6.87737 - -4.60755 -4.60755 - -1.32116 -1.32116 - 2.87263 2.87263 - -2.09541 -2.09541 - 3.43595 3.43595 - 3.63528 3.63528 - 3.52056 3.52056 - -3.59484 -3.59484 - 1.03764 1.03764 - -7.14947 -7.14947 - -5.80634 -5.80634 - 4.71397 4.71397 - 0.720588 0.720588 - -2.24074 -2.24074 - 5.82418 5.82418 - -3.22013 -3.22013 - 3.68858 3.68858 - -1.43166 -1.43166 - 4.47978 4.47978 - -4.83356 -4.83356 - -3.96257 -3.96257 - -5.95512 -5.95512 - 0.496691 0.496691 - -7.58825 -7.58825 - -6.47331 -6.47331 - -1.14446 -1.14446 - 3.91615 3.91615 - -0.588841 -0.588841 - 6.56683 6.56683 - 3.97252 3.97252 - -4.3126 -4.3126 - -8.20913 -8.20913 - 0.310182 0.310182 - -7.3006 -7.3006 - 7.92805 7.92805 - 2.1756 2.1756 - 1.06404 1.06404 - 1.14471 1.14471 - -1.50242 -1.50242 - 0.00723557 0.00723557 - 5.76841 5.76841 - -1.96707 -1.96707 - 8.87243 8.87243 - -3.23281 -3.23281 - 12.3087 12.3087 - 3.3245 3.3245 - 3.00334 3.00334 - -5.74048 -5.74048 - 7.43939 7.43939 - -0.906001 -0.906001 - 2.24067 2.24067 - -6.23989 -6.23989 - 2.81483 2.81483 - -1.62648 -1.62648 - -7.26368 -7.26368 - 1.69171 1.69171 - -11.2631 -11.2631 - -2.32992 -2.32992 - -6.07361 -6.07361 - -7.56822 -7.56822 - -7.56737 -7.56737 - 5.97037 5.97037 - 6.74398 6.74398 - -2.24599 -2.24599 - 2.95213 2.95213 - -12.7864 -12.7864 - 0.680035 0.680035 - -1.39988 -1.39988 - -4.74028 -4.74028 - 3.01887 3.01887 - 1.89636 1.89636 - 4.46014 4.46014 - -4.38308 -4.38308 - 11.7633 11.7633 - -3.54671 -3.54671 - -3.47584 -3.47584 - 3.80037 3.80037 - 7.77849 7.77849 - -7.00006 -7.00006 - -4.87665 -4.87665 - -4.54736 -4.54736 - -7.81752 -7.81752 - -0.0654465 -0.0654465 - -3.70587 -3.70587 - -2.24231 -2.24231 - 5.58005 5.58005 - -3.09415 -3.09415 - -5.55063 -5.55063 - -4.19666 -4.19666 - -6.83328 -6.83328 - -6.9216 -6.9216 - -3.72782 -3.72782 - -2.18574 -2.18574 - 1.28076 1.28076 - -3.40691 -3.40691 - 0.486964 0.486964 - -2.11025 -2.11025 - -1.42349 -1.42349 - 6.06854 6.06854 - -1.37534 -1.37534 - 9.47832 9.47832 - -0.567045 -0.567045 - -6.98328 -6.98328 - 6.73139 6.73139 - -1.56812 -1.56812 - 0.141683 0.141683 - 1.78697 1.78697 - -2.03874 -2.03874 - 1.28356 1.28356 - 6.9912 6.9912 - -3.8858 -3.8858 - -1.38808 -1.38808 - -2.16632 -2.16632 - 3.57955 3.57955 - 2.73506 2.73506 - -3.03108 -3.03108 - -3.44677 -3.44677 - 1.37111 1.37111 - -10.0008 -10.0008 - -3.61651 -3.61651 - 1.97313 1.97313 - 2.11298 2.11298 - 0.174957 0.174957 - -0.131546 -0.131546 - 7.58484 7.58484 - 4.27907 4.27907 - 0.855439 0.855439 - 4.44153 4.44153 - -1.04577 -1.04577 - -7.49625 -7.49625 - 2.1572 2.1572 - 13.0815 13.0815 - 4.57025 4.57025 - 0.704658 0.704658 - 3.25079 3.25079 - -0.682139 -0.682139 - -4.17209 -4.17209 - -1.38547 -1.38547 - 5.52688 5.52688 - -4.90717 -4.90717 - 2.56402 2.56402 - -1.37164 -1.37164 - -6.05044 -6.05044 - 8.3158 8.3158 - -0.640461 -0.640461 - -2.40145 -2.40145 - -1.02959 -1.02959 - -6.75028 -6.75028 - 4.20206 4.20206 - 0.615412 0.615412 - -0.389435 -0.389435 - -5.07439 -5.07439 - -5.34136 -5.34136 - -1.88522 -1.88522 - -4.82628 -4.82628 - 0.54435 0.54435 - -3.28948 -3.28948 - 5.0051 5.0051 - -8.5501 -8.5501 - 7.31448 7.31448 - 0.145651 0.145651 - 3.28586 3.28586 - -1.8624 -1.8624 - -8.9235 -8.9235 - 3.15894 3.15894 - -9.9459 -9.9459 - 0.517233 0.517233 - -4.59899 -4.59899 - 0.641116 0.641116 - 10.3809 10.3809 - 2.39935 2.39935 - -0.378496 -0.378496 - 0.680329 0.680329 - 2.35584 2.35584 - -2.24714 -2.24714 - -4.8742 -4.8742 - -3.96429 -3.96429 - 1.29263 1.29263 - 0.618875 0.618875 - -0.611961 -0.611961 - 1.06612 1.06612 - -3.39289 -3.39289 - -0.226022 -0.226022 - 4.24418 4.24418 - 0.884239 0.884239 - 8.25747 8.25747 - -3.23019 -3.23019 - -9.99374 -9.99374 - 8.54414 8.54414 - -6.06374 -6.06374 - -4.92601 -4.92601 - 7.22101 7.22101 - 11.5756 11.5756 - 13.436 13.436 - 4.13522 4.13522 - 9.67412 9.67412 - -3.13805 -3.13805 - 7.50856 7.50856 - -7.98069 -7.98069 - 4.92059 4.92059 - -6.72969 -6.72969 - -4.48762 -4.48762 - -3.60328 -3.60328 - -1.75053 -1.75053 - 1.5638 1.5638 - 4.74213 4.74213 - 5.16046 5.16046 - -1.9857 -1.9857 - -6.34885 -6.34885 - -3.58963 -3.58963 - 4.96795 4.96795 - 1.44405 1.44405 - -2.74682 -2.74682 - -0.545296 -0.545296 - -10.7507 -10.7507 - -0.117477 -0.117477 - -0.436907 -0.436907 - -1.11656 -1.11656 - 1.64789 1.64789 - -4.08799 -4.08799 - -1.04262 -1.04262 - 6.06007 6.06007 - -6.68208 -6.68208 - 6.81976 6.81976 - -6.89836 -6.89836 - -0.555115 -0.555115 - -2.85307 -2.85307 - -7.76567 -7.76567 - -5.65104 -5.65104 - 8.93521 8.93521 - -5.0663 -5.0663 - 2.52214 2.52214 - 0.382824 0.382824 - -0.398468 -0.398468 - 5.05183 5.05183 - 4.134 4.134 - 1.42909 1.42909 - 2.99357 2.99357 - 10.7821 10.7821 - -4.54764 -4.54764 - -0.0440308 -0.0440308 - 0.647161 0.647161 - 3.27569 3.27569 - -32.9478 -32.9478 - 6.92399 6.92399 - -3.05953 -3.05953 - -2.29742 -2.29742 - -0.41863 -0.41863 - 2.99125 2.99125 - 3.40805 3.40805 - -1.36651 -1.36651 - -3.25561 -3.25561 - 5.11504 5.11504 - -0.532291 -0.532291 - 9.93341 9.93341 - -2.2806 -2.2806 - 10.9617 10.9617 - -2.53642 -2.53642 - 0.995763 0.995763 - -1.28898 -1.28898 - -2.99921 -2.99921 - -2.46773 -2.46773 - -11.0849 -11.0849 - -11.64 -11.64 - -3.73617 -3.73617 - 2.74223 2.74223 - -0.976817 -0.976817 - -0.384814 -0.384814 - -3.38815 -3.38815 - 2.27591 2.27591 - -5.25732 -5.25732 - -1.65764 -1.65764 - -5.8501 -5.8501 - -4.85863 -4.85863 - 2.78987 2.78987 - 5.3324 5.3324 - -9.16758 -9.16758 - 7.90047 7.90047 - 5.68696 5.68696 - 7.2668 7.2668 - -0.857072 -0.857072 - 0.0834347 0.0834347 - 1.11833 1.11833 - 0.88212 0.88212 - -4.40785 -4.40785 - 5.25846 5.25846 - 7.46283 7.46283 - 6.26981 6.26981 - -10.8935 -10.8935 - -0.226332 -0.226332 - -1.64568 -1.64568 - -0.389003 -0.389003 - -0.854872 -0.854872 - -3.38063 -3.38063 - -4.74874 -4.74874 - -1.81717 -1.81717 - -6.03338 -6.03338 - 9.41153 9.41153 - -2.75636 -2.75636 - -4.03638 -4.03638 - -2.82527 -2.82527 - 0.641039 0.641039 - -3.08939 -3.08939 - -1.04523 -1.04523 - -4.17379 -4.17379 - 0.453503 0.453503 - 5.64541 5.64541 - 2.72225 2.72225 - -1.67354 -1.67354 - -6.68729 -6.68729 - -1.20785 -1.20785 - 3.51562 3.51562 - 2.38257 2.38257 - 2.75735 2.75735 - -4.62925 -4.62925 - 7.98247 7.98247 - 6.254 6.254 - 3.85448 3.85448 - -4.40298 -4.40298 - -8.28751 -8.28751 - -7.28055 -7.28055 - 7.31675 7.31675 - 3.53957 3.53957 - 2.94378 2.94378 - 1.41268 1.41268 - 5.2878 5.2878 - -0.807317 -0.807317 - -13.141 -13.141 - 5.71505 5.71505 - -3.86739 -3.86739 - 0.922435 0.922435 - -4.52167 -4.52167 - 0.82741 0.82741 - 4.1254 4.1254 - -3.64229 -3.64229 - -4.34879 -4.34879 - -5.69361 -5.69361 - 10.0503 10.0503 - -6.20878 -6.20878 - -5.70531 -5.70531 - -0.265037 -0.265037 - 4.91217 4.91217 - -9.85839 -9.85839 - 9.14639 9.14639 - 0.78426 0.78426 - -6.03581 -6.03581 - -1.225 -1.225 - -1.82514 -1.82514 - -4.38257 -4.38257 - -4.14898 -4.14898 - 1.30056 1.30056 - -4.04361 -4.04361 - -10.7862 -10.7862 - -1.71033 -1.71033 - -5.3235 -5.3235 - -5.05158 -5.05158 - 2.03088 2.03088 - -4.639 -4.639 - -8.90379 -8.90379 - -1.46286 -1.46286 - 4.78737 4.78737 - 2.84292 2.84292 - -4.60125 -4.60125 - -0.454598 -0.454598 - -3.54703 -3.54703 - -3.15574 -3.15574 - -5.66794 -5.66794 - -0.499733 -0.499733 - 4.80394 4.80394 - 7.0018 7.0018 - -12.2494 -12.2494 - -0.705371 -0.705371 - 0.0740021 0.0740021 - -2.66987 -2.66987 - 2.48263 2.48263 - -9.06332 -9.06332 - -1.01261 -1.01261 - 3.84118 3.84118 - 4.21216 4.21216 - -1.18673 -1.18673 - -11.0005 -11.0005 - -9.71638 -9.71638 - 1.76212 1.76212 - -2.83766 -2.83766 - -9.13768 -9.13768 - -1.05015 -1.05015 - 2.53008 2.53008 - 0.379504 0.379504 - 5.28803 5.28803 - -6.17221 -6.17221 - 5.75619 5.75619 - 2.3737 2.3737 - -9.0974 -9.0974 - -7.85433 -7.85433 - -10.9094 -10.9094 - 1.20756 1.20756 - 2.61486 2.61486 - 1.23359 1.23359 - 43.6151 43.6151 - -1.72859 -1.72859 - -0.965831 -0.965831 - -0.482239 -0.482239 - -1.82159 -1.82159 - 1.661 1.661 - 1.93636 1.93636 - -11.9999 -11.9999 - 0.104367 0.104367 - -1.70555 -1.70555 - -9.81074 -9.81074 - 12.7941 12.7941 - -3.36221 -3.36221 - -6.06523 -6.06523 - 0.47411 0.47411 - -6.64475 -6.64475 - -0.763006 -0.763006 - -3.9763 -3.9763 - -2.86732 -2.86732 - -20.6937 -20.6937 - 1.84418 1.84418 - 5.65243 5.65243 - 10.7255 10.7255 - -1.21293 -1.21293 - 3.15057 3.15057 - 8.96094 8.96094 - -0.205015 -0.205015 - 8.44579 8.44579 - 2.01362 2.01362 - 2.36648 2.36648 - 11.6752 11.6752 - 2.19072 2.19072 - -13.9182 -13.9182 - 3.3257 3.3257 - -6.60627 -6.60627 - 1.62083 1.62083 - -2.00847 -2.00847 - 11.6978 11.6978 - 5.93254 5.93254 - 4.93134 4.93134 - -2.50847 -2.50847 - -5.92846 -5.92846 - 1.16717 1.16717 - 6.9673 6.9673 - -1.21182 -1.21182 - 7.25413 7.25413 - -4.24031 -4.24031 - -3.12368 -3.12368 - 1.73734 1.73734 - -2.6551 -2.6551 - 5.01063 5.01063 - 10.9923 10.9923 - 3.08502 3.08502 - -1.67866 -1.67866 - 10.7003 10.7003 - -0.982895 -0.982895 - 1.97681 1.97681 - -1.29045 -1.29045 - 1.64227 1.64227 - 3.21157 3.21157 - -4.63376 -4.63376 - 4.47725 4.47725 - 7.77208 7.77208 - 0.332548 0.332548 - 2.82084 2.82084 - 0.958649 0.958649 - 1.21302 1.21302 - -3.16936 -3.16936 - 0.0672417 0.0672417 - 0.563038 0.563038 - -1.87542 -1.87542 - -3.01753 -3.01753 - 2.73107 2.73107 - -3.68276 -3.68276 - 4.64376 4.64376 - -12.4341 -12.4341 - 4.43429 4.43429 - 5.72878 5.72878 - 2.39332 2.39332 - 1.91106 1.91106 - 2.50458 2.50458 - 0.942479 0.942479 - -0.489758 -0.489758 - 0.311101 0.311101 - -2.74953 -2.74953 - 4.95959 4.95959 - 1.26862 1.26862 - 10.3622 10.3622 - 3.61213 3.61213 - -2.19285 -2.19285 - 1.28587 1.28587 - -1.85274 -1.85274 - -1.62541 -1.62541 - 2.00382 2.00382 - -5.8959 -5.8959 - -0.918042 -0.918042 - 6.43711 6.43711 - 0.419441 0.419441 - -2.61133 -2.61133 - -0.0277654 -0.0277654 - 2.77443 2.77443 - 3.83764 3.83764 - -1.44486 -1.44486 - -0.611288 -0.611288 - -4.30436 -4.30436 - 5.29466 5.29466 - 1.56058 1.56058 - 1.88962 1.88962 - 0.761408 0.761408 - 1.76505 1.76505 - 1.18453 1.18453 - 1.71559 1.71559 - -3.14851 -3.14851 - 2.73145 2.73145 - -1.23904 -1.23904 - 0.00672958 0.00672958 - 3.40979 3.40979 - -1.77498 -1.77498 - -7.12266 -7.12266 - -9.24697 -9.24697 - -4.12038 -4.12038 - -2.77817 -2.77817 - 8.23453 8.23453 - -1.29818 -1.29818 - -7.02203 -7.02203 - -5.8994 -5.8994 - 8.20499 8.20499 - 0.356509 0.356509 - -0.515947 -0.515947 - -6.23904 -6.23904 - 5.59801 5.59801 - -4.44281 -4.44281 - -2.28591 -2.28591 - -3.31819 -3.31819 - 2.39253 2.39253 - 3.18355 3.18355 - -2.73303 -2.73303 - -0.0346074 -0.0346074 - -10.2692 -10.2692 - 6.74308 6.74308 - 5.72055 5.72055 - -4.49033 -4.49033 - 1.99176 1.99176 - 6.10782 6.10782 - 2.65759 2.65759 - 1.97884 1.97884 - 0.927606 0.927606 - 1.25006 1.25006 - 9.3695 9.3695 - -2.75726 -2.75726 - -0.580415 -0.580415 - 2.92463 2.92463 - -4.49535 -4.49535 - -1.61397 -1.61397 - 3.26733 3.26733 - -3.61505 -3.61505 - -2.46453 -2.46453 - 2.42436 2.42436 - 5.68683 5.68683 - 6.07494 6.07494 - 4.35205 4.35205 - -5.29467 -5.29467 - -3.90039 -3.90039 - -1.70776 -1.70776 - -6.3172 -6.3172 - 4.03858 4.03858 - -2.58786 -2.58786 - -1.1514 -1.1514 - -0.632569 -0.632569 - -0.343314 -0.343314 - -12.2115 -12.2115 - 0.405742 0.405742 - -6.46017 -6.46017 - -2.30808 -2.30808 + 1.46556 1.46556 + 1.88278 1.88278 + 3.49482 3.49482 + 2.57193 2.57193 + -3.45378 -3.45378 + 0.101699 0.101699 + -0.67995 -0.67995 + -1.76783 -1.76783 + 1.16307 1.16307 + -0.258105 -0.258105 + 1.91705 1.91705 + -2.29152 -2.29152 + 0.348873 0.348873 + -0.74269 -0.74269 + 0.158894 0.158894 + -0.454844 -0.454844 + 2.23608 2.23608 + -0.210747 -0.210747 + -0.544031 -0.544031 + 0.656622 0.656622 + -1.47574 -1.47574 + 0.0299864 0.0299864 + 1.77173 1.77173 + -2.23061 -2.23061 + -1.00651 -1.00651 + -0.0842836 -0.0842836 + -1.17457 -1.17457 + 3.36571 3.36571 + 0.656163 0.656163 + -0.0818359 -0.0818359 + 2.36086 2.36086 + -2.1491 -2.1491 + 1.45799 1.45799 + -1.09112 -1.09112 1.1336 1.1336 - 1.47556 1.47556 - 1.98494 1.98494 - 2.24865 2.24865 - -1.65786 -1.65786 - -4.62769 -4.62769 - 4.43717 4.43717 - 8.75249 8.75249 - 4.29167 4.29167 - -3.96876 -3.96876 - -3.52244 -3.52244 - 0.161164 0.161164 - -4.13202 -4.13202 - 1.42269 1.42269 - -3.05155 -3.05155 - 1.81371 1.81371 - -1.03765 -1.03765 - 0.696656 0.696656 - 2.95359 2.95359 - -4.74837 -4.74837 - -9.03481 -9.03481 - 4.8852 4.8852 - 9.47173 9.47173 - 11.3037 11.3037 - -3.88084 -3.88084 - -5.99356 -5.99356 - 7.81639 7.81639 - -6.51949 -6.51949 - 7.801 7.801 - -0.795429 -0.795429 - -0.801046 -0.801046 - 2.70658 2.70658 - 5.51012 5.51012 - 1.8181 1.8181 - -0.452854 -0.452854 - -10.1558 -10.1558 - 1.95877 1.95877 - -3.88197 -3.88197 - 1.72033 1.72033 - -1.8939 -1.8939 - -1.64082 -1.64082 - -0.409815 -0.409815 - 9.98658 9.98658 - -0.115277 -0.115277 - 1.49827 1.49827 - 1.6696 1.6696 - 2.29297 2.29297 - -2.14941 -2.14941 - 2.43318 2.43318 - 3.59845 3.59845 - -4.58877 -4.58877 - -9.25371 -9.25371 - 2.03609 2.03609 - 5.5921 5.5921 - -0.532859 -0.532859 - 4.34937 4.34937 - 1.57036 1.57036 - 2.30747 2.30747 - 7.5055 7.5055 - 3.41771 3.41771 - 0.589402 0.589402 - 1.55834 1.55834 - 5.12407 5.12407 - -1.41727 -1.41727 - 1.03223 1.03223 - -2.06257 -2.06257 - 3.11532 3.11532 - 1.90042 1.90042 - 8.66814 8.66814 - 5.36716 5.36716 - 2.38085 2.38085 - 5.72834 5.72834 - -6.5998 -6.5998 - 0.852569 0.852569 - -7.5648 -7.5648 - 2.98063 2.98063 - 7.81573 7.81573 - 1.82276 1.82276 - -1.81083 -1.81083 - 5.48043 5.48043 - -1.85315 -1.85315 - -1.62277 -1.62277 - -10.4951 -10.4951 - 5.34799 5.34799 - -1.77515 -1.77515 - 5.88005 5.88005 - 0.0799242 0.0799242 - 1.23264 1.23264 - -11.835 -11.835 - 3.56828 3.56828 - 7.53741 7.53741 - -5.24051 -5.24051 - -0.206917 -0.206917 - 4.36865 4.36865 - -4.10348 -4.10348 - 0.857712 0.857712 - -5.09677 -5.09677 - 7.37208 7.37208 - -3.14614 -3.14614 - 12.061 12.061 - 4.80096 4.80096 - 2.82421 2.82421 - -4.97446 -4.97446 - -11.0289 -11.0289 - -8.33282 -8.33282 - 0.69922 0.69922 - 5.08771 5.08771 - 2.65174 2.65174 - -3.30182 -3.30182 - 5.21741 5.21741 - 8.85373 8.85373 - 8.36416 8.36416 - 2.54295 2.54295 - -1.61657 -1.61657 - 1.12017 1.12017 - -7.33205 -7.33205 - 3.82582 3.82582 - -0.858026 -0.858026 - 1.40304 1.40304 - 1.35079 1.35079 - 4.19532 4.19532 - -1.77923 -1.77923 - -10.5119 -10.5119 - 10.8061 10.8061 - -3.49603 -3.49603 - 3.12404 3.12404 - -3.93328 -3.93328 - -6.73356 -6.73356 - 1.80532 1.80532 - -0.368024 -0.368024 - -3.47875 -3.47875 - -4.22893 -4.22893 - 2.52519 2.52519 - -3.54943 -3.54943 - -2.39869 -2.39869 - 4.22126 4.22126 - -0.253856 -0.253856 - 7.51866 7.51866 - -4.54093 -4.54093 - 3.44497 3.44497 - 4.77417 4.77417 - 4.49646 4.49646 - -5.78678 -5.78678 - 0.745013 0.745013 - 1.69763 1.69763 - -2.64759 -2.64759 - 1.66108 1.66108 - -4.68276 -4.68276 - 5.31823 5.31823 - 3.52288 3.52288 - 4.9695 4.9695 - 12.2016 12.2016 - 2.46849 2.46849 - -7.60038 -7.60038 - 8.21628 8.21628 - 5.99856 5.99856 - -6.80947 -6.80947 - 7.22522 7.22522 - -2.00065 -2.00065 - -8.24049 -8.24049 - -0.0804049 -0.0804049 - -2.06638 -2.06638 - -2.82884 -2.82884 - -4.25891 -4.25891 - -5.20258 -5.20258 - -3.19396 -3.19396 - -5.14527 -5.14527 - -4.28244 -4.28244 - 4.70805 4.70805 - -3.08065 -3.08065 - -4.86906 -4.86906 - -29.0266 -29.0266 - -1.22941 -1.22941 - -1.30928 -1.30928 - -6.35234 -6.35234 - 1.87904 1.87904 - 8.37797 8.37797 - -5.8821 -5.8821 - 3.10138 3.10138 - -3.27553 -3.27553 - -0.208451 -0.208451 - -2.28999 -2.28999 - 12.2896 12.2896 - -1.27394 -1.27394 - -3.41924 -3.41924 - -0.289592 -0.289592 - 1.79867 1.79867 - 1.98504 1.98504 - 1.55159 1.55159 - 1.10858 1.10858 - 0.352842 0.352842 - -0.309044 -0.309044 - -0.165336 -0.165336 - 1.15822 1.15822 - -1.39342 -1.39342 - -0.162562 -0.162562 - -8.06055 -8.06055 - -5.02776 -5.02776 - -8.66927 -8.66927 - 1.14576 1.14576 - -1.52122 -1.52122 - -1.29436 -1.29436 - 3.26421 3.26421 - 7.55561 7.55561 - 7.7265 7.7265 - -0.48821 -0.48821 - 12.439 12.439 - 7.0264 7.0264 - -11.9855 -11.9855 - -3.74151 -3.74151 - -0.200302 -0.200302 - 5.39515 5.39515 - -4.3468 -4.3468 - 9.25599 9.25599 - 3.37455 3.37455 - -6.15424 -6.15424 - -6.6271 -6.6271 - 0.000272481 0.000272481 - -5.48117 -5.48117 - -0.493191 -0.493191 - -3.46473 -3.46473 - 2.33812 2.33812 - 0.885965 0.885965 - 4.74926 4.74926 - 1.51959 1.51959 - 2.50956 2.50956 - -0.728024 -0.728024 - 1.0381 1.0381 - 5.48121 5.48121 - -1.68033 -1.68033 - -5.05915 -5.05915 - -0.646233 -0.646233 - 0.614062 0.614062 - 4.54219 4.54219 - -1.63006 -1.63006 - -3.10589 -3.10589 - -3.12801 -3.12801 - -5.98177 -5.98177 - -3.59188 -3.59188 - 1.7066 1.7066 - -7.43935 -7.43935 - 10.6141 10.6141 - 12.6478 12.6478 - -1.7222 -1.7222 - -2.1519 -2.1519 - -7.16573 -7.16573 - 0.887314 0.887314 - -8.59735 -8.59735 - -1.3609 -1.3609 - 4.47651 4.47651 - 0.900892 0.900892 - 7.81857 7.81857 - 6.19857 6.19857 - 2.12844 2.12844 - 3.08551 3.08551 - 4.15866 4.15866 - 2.09657 2.09657 - -2.27786 -2.27786 - 1.33571 1.33571 - 4.46899 4.46899 - 4.46674 4.46674 - 3.20736 3.20736 - 5.68287 5.68287 - 10.1058 10.1058 - 5.1894 5.1894 - 3.5452 3.5452 - 10.06 10.06 - 7.02935 7.02935 - -1.06066 -1.06066 - 10.32 10.32 - -0.860463 -0.860463 - 5.95992 5.95992 - -6.30137 -6.30137 - -5.01947 -5.01947 - 5.75187 5.75187 - -1.10079 -1.10079 - -1.91783 -1.91783 - 0.815744 0.815744 - -0.958663 -0.958663 - -3.28825 -3.28825 - -6.37854 -6.37854 - 6.91577 6.91577 - 2.54565 2.54565 - 1.39487 1.39487 - 1.59679 1.59679 - 4.72347 4.72347 - 2.49221 2.49221 - 1.29896 1.29896 - -4.08232 -4.08232 - -0.648436 -0.648436 - -6.43531 -6.43531 - -0.556197 -0.556197 - -1.40304 -1.40304 - 0.699818 0.699818 - -5.29777 -5.29777 - -3.44335 -3.44335 - 7.35309 7.35309 - 8.846 8.846 - 8.39833 8.39833 - -2.71436 -2.71436 - 3.37063 3.37063 - -3.18723 -3.18723 - 1.32256 1.32256 - -3.09485 -3.09485 - 8.78146 8.78146 - -1.30004 -1.30004 - 3.03526 3.03526 - -1.4592 -1.4592 - 3.90288 3.90288 - -13.5124 -13.5124 - 1.35105 1.35105 - 3.37337 3.37337 - 2.5171 2.5171 - -4.22085 -4.22085 - 13.1858 13.1858 - -6.02839 -6.02839 - 5.75692 5.75692 - 2.46171 2.46171 - -0.950315 -0.950315 - -3.63255 -3.63255 - 1.88 1.88 - 5.48758 5.48758 - 4.96786 4.96786 - -6.17199 -6.17199 - -0.284244 -0.284244 - -1.80256 -1.80256 - 3.03221 3.03221 - -8.90171 -8.90171 - -8.66084 -8.66084 - -9.06366 -9.06366 - -3.02007 -3.02007 - -8.2276 -8.2276 - 8.10032 8.10032 - -4.11364 -4.11364 - -3.39291 -3.39291 - 3.64208 3.64208 - -0.739833 -0.739833 - -2.84156 -2.84156 - -0.843081 -0.843081 - -0.249744 -0.249744 - 7.05075 7.05075 - 0.369632 0.369632 - -1.90893 -1.90893 - 9.79465 9.79465 - 3.52356 3.52356 - 4.14091 4.14091 - 1.66568 1.66568 - -10.7162 -10.7162 - -7.64522 -7.64522 - 1.54688 1.54688 - 7.84479 7.84479 - 0.466458 0.466458 - 4.03315 4.03315 - 0.472926 0.472926 - 1.73319 1.73319 - 1.79317 1.79317 - 1.46234 1.46234 - -8.45267 -8.45267 - 7.30327 7.30327 - 3.08869 3.08869 - 5.27442 5.27442 - 2.92876 2.92876 - -1.6673 -1.6673 - 14.4442 14.4442 - 13.4055 13.4055 - -1.47522 -1.47522 - -3.57821 -3.57821 - 9.00659 9.00659 - -9.6723 -9.6723 - 2.8818 2.8818 - -2.61898 -2.61898 - 1.17927 1.17927 - -3.15135 -3.15135 - -0.976968 -0.976968 - 1.45062 1.45062 - 4.66687 4.66687 - 4.94346 4.94346 - -2.20375 -2.20375 - 2.93643 2.93643 - 7.51365 7.51365 - 6.50034 6.50034 - 1.74088 1.74088 - -4.43403 -4.43403 - 0.796894 0.796894 - -1.23803 -1.23803 - 5.33941 5.33941 - 4.90517 4.90517 - 0.569053 0.569053 - -0.609673 -0.609673 - 5.091 5.091 - 1.76184 1.76184 - -3.81174 -3.81174 - -5.39095 -5.39095 - -3.09718 -3.09718 - -1.87868 -1.87868 - -4.85278 -4.85278 - -1.05327 -1.05327 - -1.11892 -1.11892 - -3.52006 -3.52006 - -2.8466 -2.8466 - 3.03494 3.03494 - 3.7605 3.7605 - -1.8123 -1.8123 - -5.10186 -5.10186 - 2.85973 2.85973 - -3.6241 -3.6241 - 1.78302 1.78302 - -12.3108 -12.3108 - 0.378043 0.378043 - -1.70182 -1.70182 - -0.91773 -0.91773 - -5.37355 -5.37355 + 0.414929 0.414929 + 1.73283 1.73283 + 0.333332 0.333332 + 0.612909 0.612909 + 2.27707 2.27707 + -17.3366 -17.3366 + -0.706549 -0.706549 + 0.569747 0.569747 + -2.88274 -2.88274 + -0.39159 -0.39159 + -1.272 -1.272 + 1.8522 1.8522 + 2.84343 2.84343 + -0.497109 -0.497109 + 0.712927 0.712927 + -0.454025 -0.454025 + 0.26183 0.26183 + 1.70567 1.70567 + 2.65185 2.65185 + 0.0631512 0.0631512 + 1.55346 1.55346 + 1.88528 1.88528 + 0.550265 0.550265 + -0.663944 -0.663944 + 0.796127 0.796127 + 1.5298 1.5298 + -0.619118 -0.619118 + 0.718367 0.718367 + -1.1251 -1.1251 + -0.162817 -0.162817 + 0.158565 0.158565 + -0.479435 -0.479435 + 0.39778 0.39778 + -0.0668922 -0.0668922 + -0.554059 -0.554059 + -0.112442 -0.112442 + 0.723476 0.723476 + 0.585585 0.585585 + -23.8354 -23.8354 + 2.23005 2.23005 + 3.19716 3.19716 + 0.878244 0.878244 + 1.24065 1.24065 + 2.02788 2.02788 + 1.33889 1.33889 + -0.0959704 -0.0959704 + 0.819301 0.819301 + 1.13624 1.13624 + -3.73236 -3.73236 + -0.228297 -0.228297 + 0.300764 0.300764 + 2.44121 2.44121 + 1.64459 1.64459 + -2.9741 -2.9741 + -0.0767092 -0.0767092 + 1.18346 1.18346 + 0.766311 0.766311 + -2.34982 -2.34982 + -1.19871 -1.19871 + -1.12327 -1.12327 + -1.51106 -1.51106 + -0.954545 -0.954545 + 1.09042 1.09042 + 1.78073 1.78073 + -1.6752 -1.6752 + 0.032185 0.032185 + -0.67565 -0.67565 + -0.318844 -0.318844 + -2.8893 -2.8893 + -0.0509838 -0.0509838 + -1.21507 -1.21507 + 2.30929 2.30929 + 0.216374 0.216374 + 1.31134 1.31134 + -0.0998794 -0.0998794 + 1.20712 1.20712 + -0.650083 -0.650083 + 0.399312 0.399312 + -0.983942 -0.983942 + -1.13459 -1.13459 + -1.05734 -1.05734 + -1.5015 -1.5015 + -1.77513 -1.77513 + -1.40363 -1.40363 + 1.33786 1.33786 + -0.826259 -0.826259 + 1.47633 1.47633 + -0.943677 -0.943677 + -1.46965 -1.46965 + -0.526938 -0.526938 + 0.191906 0.191906 + -0.886405 -0.886405 + 0.449917 0.449917 + 0.515804 0.515804 + -1.39774 -1.39774 + -0.654256 -0.654256 + -3.35005 -3.35005 + 2.06083 2.06083 + -0.704538 -0.704538 + -0.462466 -0.462466 + -2.67366 -2.67366 + -2.18423 -2.18423 + -2.64621 -2.64621 + 0.763978 0.763978 + -2.66418 -2.66418 + -1.60911 -1.60911 + 2.16377 2.16377 + 0.0968811 0.0968811 + 1.1338 1.1338 + 1.00793 1.00793 + 1.46003 1.46003 + -1.83402 -1.83402 + 0.91668 0.91668 + 0.614564 0.614564 + 0.635087 0.635087 + -1.34331 -1.34331 + -0.972611 -0.972611 + 0.674227 0.674227 + -0.667037 -0.667037 + -2.53443 -2.53443 + -1.53923 -1.53923 + -0.503973 -0.503973 + -3.74964 -3.74964 + -2.319 -2.319 + 0.634011 0.634011 + -0.232782 -0.232782 + 0.494076 0.494076 + -1.22967 -1.22967 + 2.0448 2.0448 + -2.49856 -2.49856 + 0.234543 0.234543 + 0.926016 0.926016 + 0.556234 0.556234 + 1.40825 1.40825 + -0.0225873 -0.0225873 + 0.584709 0.584709 + 0.750481 0.750481 + 0.548451 0.548451 + -0.50962 -0.50962 + 1.37736 1.37736 + -0.205268 -0.205268 + -2.18919 -2.18919 + -1.09548 -1.09548 + 2.88677 2.88677 + 0.36593 0.36593 + 1.59942 1.59942 + 2.36871 2.36871 + -0.360048 -0.360048 + 0.462762 0.462762 + -2.94594 -2.94594 + -1.04354 -1.04354 + 0.866209 0.866209 + 1.5211 1.5211 + 1.06147 1.06147 + 0.900548 0.900548 + -2.17463 -2.17463 + 2.6977 2.6977 + -0.854358 -0.854358 + 1.45336 1.45336 + 0.700988 0.700988 + -1.89129 -1.89129 + 0.346364 0.346364 + 1.09519 1.09519 + 1.19629 1.19629 + -0.193888 -0.193888 + 3.33354 3.33354 + -0.986782 -0.986782 + -0.328471 -0.328471 + 1.29318 1.29318 + 0.463316 0.463316 + -1.3053 -1.3053 + -0.247824 -0.247824 + 0.770688 0.770688 + -0.471133 -0.471133 + 0.613311 0.613311 + -0.394463 -0.394463 + -1.93519 -1.93519 + -1.45159 -1.45159 + 0.868683 0.868683 + 2.57851 2.57851 + -2.39077 -2.39077 + 1.63547 1.63547 + -0.0639398 -0.0639398 + 1.19782 1.19782 + -1.83348 -1.83348 + -0.473202 -0.473202 + 0.124168 0.124168 + 1.16798 1.16798 + 1.21296 1.21296 + -0.527453 -0.527453 + -1.20688 -1.20688 + -0.713952 -0.713952 + 0.262652 0.262652 + -2.18811 -2.18811 + 4.102 4.102 + 0.494994 0.494994 + 2.77783 2.77783 + 1.07731 1.07731 + 0.198336 0.198336 + 9.57229 9.57229 + 0.349448 0.349448 + -2.29562 -2.29562 + -0.644837 -0.644837 + -0.217495 -0.217495 + -1.45496 -1.45496 + -2.09866 -2.09866 + -0.0384889 -0.0384889 + 0.745561 0.745561 + 0.14508 0.14508 + -1.51823 -1.51823 + -3.26751 -3.26751 + 0.0523259 0.0523259 + 1.55301 1.55301 + -3.45208 -3.45208 + 0.572187 0.572187 + -2.49679 -2.49679 + 2.37328 2.37328 + 0.765015 0.765015 + -0.27352 -0.27352 + 0.0308149 0.0308149 + -0.593623 -0.593623 + -1.2373 -1.2373 + -0.656835 -0.656835 + -2.13627 -2.13627 + 0.352383 0.352383 + -0.406136 -0.406136 + 1.4126 1.4126 + 1.98515 1.98515 + 0.606976 0.606976 + 0.788446 0.788446 + 0.16519 0.16519 + 0.596765 0.596765 + -0.873611 -0.873611 + -1.43965 -1.43965 + 0.0971001 0.0971001 + 3.33849 3.33849 + 4.49406 4.49406 + 1.91075 1.91075 + 2.4465 2.4465 + 2.35879 2.35879 + -2.22234 -2.22234 + 2.1624 2.1624 + 0.037742 0.037742 + -1.28695 -1.28695 + 0.341673 0.341673 + 1.11316 1.11316 + 1.57437 1.57437 + -0.778719 -0.778719 + -2.35793 -2.35793 + -0.95091 -0.95091 + -1.59879 -1.59879 + -1.65049 -1.65049 + 1.05626 1.05626 + 1.14966 1.14966 + -2.68582 -2.68582 + -1.96644 -1.96644 + 0.950004 0.950004 + 3.03491 3.03491 + -1.64747 -1.64747 + -2.14413 -2.14413 + -0.804942 -0.804942 + 2.19513 2.19513 + 0.789969 0.789969 + -0.899571 -0.899571 + -0.190668 -0.190668 + 0.600747 0.600747 + 1.16949 1.16949 + -1.37309 -1.37309 + -1.43457 -1.43457 + 0.462702 0.462702 + 1.40818 1.40818 + -0.665746 -0.665746 + -2.68812 -2.68812 + -0.0352648 -0.0352648 + -1.20922 -1.20922 + -0.577064 -0.577064 + -1.71517 -1.71517 + 1.35121 1.35121 + 1.4034 1.4034 + -4.70048 -4.70048 + -4.44732 -4.44732 + 0.136732 0.136732 + 2.41848 2.41848 + -4.30087 -4.30087 + -2.46922 -2.46922 + 0.376919 0.376919 + 1.9388 1.9388 + 1.19049 1.19049 + 1.97452 1.97452 + -0.172176 -0.172176 + -2.31517 -2.31517 + 7.36023 7.36023 + -0.738473 -0.738473 + 0.0146472 0.0146472 + 0.14857 0.14857 + 2.47231 2.47231 + 2.18251 2.18251 + -2.32425 -2.32425 + -2.17348 -2.17348 + 7.06046 7.06046 + 1.42047 1.42047 + -1.54418 -1.54418 + -0.469984 -0.469984 + 1.63989 1.63989 + 0.346456 0.346456 + 1.15453 1.15453 + -0.441885 -0.441885 + 0.183807 0.183807 + 3.1971 3.1971 + -0.647559 -0.647559 + 0.00128156 0.00128156 + -0.288304 -0.288304 + 2.59287 2.59287 + 1.19027 1.19027 + 0.797827 0.797827 + 1.09197 1.09197 + 0.43136 0.43136 + 0.380913 0.380913 + -1.74969 -1.74969 + 1.14465 1.14465 + -0.386246 -0.386246 + 0.566953 0.566953 + 0.00563241 0.00563241 + 0.841304 0.841304 + 1.16119 1.16119 + -0.473218 -0.473218 + 0.567447 0.567447 + -2.51315 -2.51315 + 0.857171 0.857171 + -1.81643 -1.81643 + 1.81088 1.81088 + 1.49892 1.49892 + -0.900041 -0.900041 + -0.0992927 -0.0992927 + 0.635308 0.635308 + 0.0198956 0.0198956 + 2.37949 2.37949 + -1.77301 -1.77301 + -0.716384 -0.716384 + 0.226013 0.226013 + -3.16016 -3.16016 + 0.470524 0.470524 + 0.11595 0.11595 + -2.40913 -2.40913 + -0.394387 -0.394387 + -0.712704 -0.712704 + 0.539429 0.539429 + 2.49816 2.49816 + -0.611206 -0.611206 + -0.562752 -0.562752 + -0.835585 -0.835585 + 3.25424 3.25424 + -0.480187 -0.480187 + 5.29497 5.29497 + -2.8016 -2.8016 + 0.0118055 0.0118055 + 1.86542 1.86542 + 0.101613 0.101613 + -2.00855 -2.00855 + 0.771721 0.771721 + -0.0705836 -0.0705836 + -0.722898 -0.722898 + 0.11892 0.11892 + -1.12107 -1.12107 + 0.23499 0.23499 + -3.2635 -3.2635 + 0.774997 0.774997 + 3.05413 3.05413 + 2.52668 2.52668 + -0.733611 -0.733611 + -0.465726 -0.465726 + 1.61462 1.61462 + -0.297293 -0.297293 + 0.247443 0.247443 + 0.451474 0.451474 + -2.18135 -2.18135 + -0.253176 -0.253176 + -1.77935 -1.77935 + 1.96681 1.96681 + -2.6561 -2.6561 + -0.98605 -0.98605 + 3.74766 3.74766 + -1.86272 -1.86272 + -1.6491 -1.6491 + 1.43297 1.43297 + 3.00856 3.00856 + -2.91703 -2.91703 + 2.31349 2.31349 + 0.612495 0.612495 + 0.303318 0.303318 + 0.96081 0.96081 + 0.401881 0.401881 + -0.130979 -0.130979 + -0.660971 -0.660971 + 0.39712 0.39712 + -1.30811 -1.30811 + 1.66804 1.66804 + -0.362844 -0.362844 + -3.37322 -3.37322 + -0.898031 -0.898031 + -2.35764 -2.35764 + 0.234687 0.234687 + 0.776599 0.776599 + -0.030942 -0.030942 + -3.49072 -3.49072 + -0.354064 -0.354064 + 0.254806 0.254806 + 0.413767 0.413767 + -2.23448 -2.23448 + -3.00986 -3.00986 + -3.03087 -3.03087 + 1.22426 1.22426 + -0.00528415 -0.00528415 + 1.63329 1.63329 + -3.00948 -3.00948 + 0.544208 0.544208 + -2.26614 -2.26614 + 0.202507 0.202507 + 1.10992 1.10992 + -1.14457 -1.14457 + 0.223205 0.223205 + -2.11668 -2.11668 + 0.939281 0.939281 + -1.74119 -1.74119 + 1.36972 1.36972 + -1.02231 -1.02231 + 0.760286 0.760286 + 1.4837 1.4837 + 2.29908 2.29908 + 1.30551 1.30551 + -0.294187 -0.294187 + -1.49058 -1.49058 + 0.292385 0.292385 + -1.13357 -1.13357 + -0.90415 -0.90415 + 0.874832 0.874832 + 1.10395 1.10395 + 2.24293 2.24293 + -1.39277 -1.39277 + -1.87169 -1.87169 + -0.0884342 -0.0884342 + 0.167855 0.167855 + 1.65614 1.65614 + 1.8828 1.8828 + 0.115504 0.115504 + 4.08686 4.08686 + -0.626715 -0.626715 + -0.0646496 -0.0646496 + 0.178995 0.178995 + 1.50054 1.50054 + 1.85495 1.85495 + 1.94429 1.94429 + 0.0459594 0.0459594 + -1.9626 -1.9626 + -1.70859 -1.70859 + -1.05544 -1.05544 + 0.331346 0.331346 + 0.698268 0.698268 + -1.06962 -1.06962 + -2.22214 -2.22214 + 1.438 1.438 + 1.93536 1.93536 + -0.429009 -0.429009 + -1.04787 -1.04787 + -1.8681 -1.8681 + -1.46553 -1.46553 + -0.704678 -0.704678 + -1.80182 -1.80182 + -0.954295 -0.954295 + -2.80916 -2.80916 + -1.25241 -1.25241 + -0.581477 -0.581477 + 1.11401 1.11401 + -0.206182 -0.206182 + -0.0337333 -0.0337333 + -0.192489 -0.192489 + -0.602529 -0.602529 + -0.465761 -0.465761 + 0.886887 0.886887 + -1.47977 -1.47977 + 0.81157 0.81157 + -4.08164 -4.08164 + -0.650256 -0.650256 + -0.239842 -0.239842 + -0.376812 -0.376812 + 0.608583 0.608583 + 3.24043 3.24043 + -2.40365 -2.40365 + -2.12369 -2.12369 + -2.85674 -2.85674 + -1.11819 -1.11819 + -0.183936 -0.183936 + -1.38699 -1.38699 + 0.63657 0.63657 + 3.24116 3.24116 + -0.0696033 -0.0696033 + 1.67356 1.67356 + -0.111105 -0.111105 + 0.928557 0.928557 + -1.21476 -1.21476 + 1.87918 1.87918 + -0.538827 -0.538827 + -1.96541 -1.96541 + -2.25371 -2.25371 + 1.95748 1.95748 + -0.19679 -0.19679 + -0.139908 -0.139908 + 0.621337 0.621337 + -0.019154 -0.019154 + 0.701629 0.701629 + 0.374446 0.374446 + -1.25937 -1.25937 + 1.34433 1.34433 + -0.922861 -0.922861 + 2.90885 2.90885 + -2.56409 -2.56409 + -0.256198 -0.256198 + -0.460552 -0.460552 + 0.178514 0.178514 + 0.182276 0.182276 + 0.646096 0.646096 + -1.0186 -1.0186 + 0.979231 0.979231 + 1.84611 1.84611 + 2.91017 2.91017 + 0.327271 0.327271 + 0.454179 0.454179 + 1.09265 1.09265 + -0.143546 -0.143546 + 0.846766 0.846766 + 0.719947 0.719947 + -1.76553 -1.76553 + 3.30387 3.30387 + 0.321026 0.321026 + 0.201893 0.201893 + 2.04676 2.04676 + 0.304647 0.304647 + -0.753991 -0.753991 + -1.40742 -1.40742 + -0.77855 -0.77855 + 0.29947 0.29947 + -0.920328 -0.920328 + -0.463037 -0.463037 + -1.81857 -1.81857 + -0.305488 -0.305488 + 1.80501 1.80501 + -2.63795 -2.63795 + -1.81647 -1.81647 + -2.82112 -2.82112 + 0.773168 0.773168 + 0.275974 0.275974 + 0.0240539 0.0240539 + -0.764899 -0.764899 + 1.05992 1.05992 + -0.84197 -0.84197 + 1.27771 1.27771 + 0.71536 0.71536 + -0.970194 -0.970194 + -2.22678 -2.22678 + -1.18751 -1.18751 + 2.08652 2.08652 + -1.77634 -1.77634 + 0.330371 0.330371 + -0.459388 -0.459388 + 3.26156 3.26156 + -0.801958 -0.801958 + 0.507587 0.507587 + 0.0705198 0.0705198 + -4.27573 -4.27573 + 1.35474 1.35474 + -0.563817 -0.563817 + -0.036168 -0.036168 + 0.373159 0.373159 + 0.146304 0.146304 + 0.540254 0.540254 + -1.32898 -1.32898 + 0.0550983 0.0550983 + -0.894857 -0.894857 + 0.879887 0.879887 + -1.01703 -1.01703 + 0.262536 0.262536 + 0.605519 0.605519 + -0.682249 -0.682249 + -1.76663 -1.76663 + 0.425809 0.425809 + -1.77275 -1.77275 + -2.40044 -2.40044 + -1.28476 -1.28476 + -1.80539 -1.80539 + 3.42497 3.42497 + -0.737508 -0.737508 + 1.09345 1.09345 + 0.473303 0.473303 + 0.569697 0.569697 + 0.1682 0.1682 + 1.14317 1.14317 + -0.305234 -0.305234 + -0.0553584 -0.0553584 + 0.553831 0.553831 + 2.72079 2.72079 + -1.11628 -1.11628 + -0.897232 -0.897232 + 1.24584 1.24584 + 0.407171 0.407171 + -0.548786 -0.548786 + -0.799434 -0.799434 + -0.337201 -0.337201 + 1.38839 1.38839 + -2.03675 -2.03675 + -0.451153 -0.451153 + -1.51334 -1.51334 + 0.99722 0.99722 + -0.952379 -0.952379 + -1.8534 -1.8534 + 0.427123 0.427123 + -0.813837 -0.813837 + 1.34476 1.34476 + -0.0132453 -0.0132453 + 1.59654 1.59654 + 1.28669 1.28669 + -0.262967 -0.262967 + 0.319661 0.319661 + 0.514152 0.514152 + 8.72845 8.72845 + 0.981584 0.981584 + 2.1889 2.1889 + 2.59352 2.59352 + 4.8238 4.8238 + -1.98665 -1.98665 + -3.87038 -3.87038 + 0.451746 0.451746 + -2.04906 -2.04906 + 0.192938 0.192938 + 1.08472 1.08472 + -0.548332 -0.548332 + -0.871136 -0.871136 + 2.97136 2.97136 + 0.822154 0.822154 + -1.43775 -1.43775 + -1.05529 -1.05529 + 0.332036 0.332036 + -1.01064 -1.01064 + 1.01202 1.01202 + 0.603861 0.603861 + 1.44141 1.44141 + -0.0347525 -0.0347525 + -2.645 -2.645 + -2.77208 -2.77208 + 2.51425 2.51425 + -1.59414 -1.59414 + 2.38812 2.38812 + -0.515258 -0.515258 + -1.70386 -1.70386 + 0.740403 0.740403 + -0.519196 -0.519196 + 3.56872 3.56872 + 2.61736 2.61736 + 0.921465 0.921465 + 0.435321 0.435321 + -0.620441 -0.620441 + 2.59352 2.59352 + -1.05395 -1.05395 + 2.66575 2.66575 + -0.144386 -0.144386 + 1.83629 1.83629 + -0.66284 -0.66284 + 0.456305 0.456305 + 2.47313 2.47313 + -0.947016 -0.947016 + 0.201143 0.201143 + 0.489325 0.489325 + 1.87943 1.87943 + 0.317216 0.317216 + 1.68371 1.68371 + -1.48571 -1.48571 + 1.8859 1.8859 + 0.964108 0.964108 + 3.81978 3.81978 + 1.57192 1.57192 + 3.67288 3.67288 + 0.946176 0.946176 + 1.40396 1.40396 + 0.7311 0.7311 + 1.83264 1.83264 + 1.88672 1.88672 + -1.65002 -1.65002 + -1.27921 -1.27921 + 0.328562 0.328562 + -1.67421 -1.67421 + 1.5893 1.5893 + -0.871527 -0.871527 + 2.71922 2.71922 + 2.10276 2.10276 + -0.53672 -0.53672 + 1.90948 1.90948 + -1.04531 -1.04531 + 0.536989 0.536989 + 0.0668934 0.0668934 + 2.15979 2.15979 + -1.34284 -1.34284 + 2.87173 2.87173 + -1.23698 -1.23698 + 1.12051 1.12051 + -0.241109 -0.241109 + -1.84701 -1.84701 + -1.19951 -1.19951 + -1.08058 -1.08058 + 0.412929 0.412929 + -0.620347 -0.620347 + 3.21218 3.21218 + 0.266479 0.266479 + 0.999923 0.999923 + 0.151579 0.151579 + 0.0457301 0.0457301 + -0.863071 -0.863071 + 0.650015 0.650015 + -0.724167 -0.724167 + 0.943086 0.943086 + -2.67036 -2.67036 + 0.592203 0.592203 + -1.73829 -1.73829 + -0.439297 -0.439297 + 0.35609 0.35609 + -0.415771 -0.415771 + 0.226092 0.226092 + 1.98817 1.98817 + -2.98191 -2.98191 + -0.157906 -0.157906 + -0.0408938 -0.0408938 + 1.32982 1.32982 + -1.98093 -1.98093 + 1.68608 1.68608 + 1.42987 1.42987 + -0.506031 -0.506031 + -3.44969 -3.44969 + -0.670397 -0.670397 + 0.00317784 0.00317784 + -2.44821 -2.44821 + -0.973505 -0.973505 + -2.01111 -2.01111 + 0.0882928 0.0882928 + -1.30306 -1.30306 + 0.606925 0.606925 + -2.90382 -2.90382 + 2.34368 2.34368 + 1.06742 1.06742 + -0.147726 -0.147726 + -0.644699 -0.644699 + 2.38565 2.38565 + 0.423195 0.423195 + -2.39995 -2.39995 + 3.07238 3.07238 + 1.89833 1.89833 + 2.6225 2.6225 + -0.520012 -0.520012 + 2.15609 2.15609 + -3.67348 -3.67348 + -0.0497026 -0.0497026 + 2.84596 2.84596 + 0.620075 0.620075 + -0.554731 -0.554731 + 2.15651 2.15651 + -0.754285 -0.754285 + -0.205464 -0.205464 + -0.530061 -0.530061 + 1.06215 1.06215 + -0.484595 -0.484595 + 0.527476 0.527476 + 2.19013 2.19013 + -2.27807 -2.27807 + -0.523498 -0.523498 + 1.80502 1.80502 + 2.36156 2.36156 + -0.482968 -0.482968 + -1.094 -1.094 + -1.03241 -1.03241 + 1.35803 1.35803 + 2.60347 2.60347 + 0.971085 0.971085 + 0.552662 0.552662 + -0.776221 -0.776221 + 0.828938 0.828938 + 1.69376 1.69376 + -0.0488535 -0.0488535 + 0.585078 0.585078 + -0.783814 -0.783814 + 3.78798 3.78798 + -0.977169 -0.977169 + -0.699405 -0.699405 + -1.00392 -1.00392 + 1.78081 1.78081 + 0.846007 0.846007 + -1.53142 -1.53142 + -0.278166 -0.278166 + -1.7533 -1.7533 + -1.5105 -1.5105 + -0.911813 -0.911813 + -2.28061 -2.28061 + 1.55691 1.55691 + 0.119324 0.119324 + -1.75206 -1.75206 + 1.31127 1.31127 + -1.10018 -1.10018 + 1.50646 1.50646 + 0.671974 0.671974 + 0.408163 0.408163 + -2.99704 -2.99704 + -0.397517 -0.397517 + 1.10579 1.10579 + 0.309227 0.309227 + 0.211714 0.211714 + -1.84012 -1.84012 + -0.727436 -0.727436 + 2.11426 2.11426 + 1.31798 1.31798 + -0.253154 -0.253154 + 1.98683 1.98683 + -0.523705 -0.523705 + 0.645383 0.645383 + 4.59945 4.59945 + -0.753718 -0.753718 + -2.0014 -2.0014 + -2.86775 -2.86775 + -0.51614 -0.51614 + 3.9091 3.9091 + 2.32844 2.32844 + -1.42046 -1.42046 + 2.29523 2.29523 + 0.966474 0.966474 + 0.938259 0.938259 + 0.314803 0.314803 + -0.965298 -0.965298 + 1.36128 1.36128 + 2.32066 2.32066 + 1.15654 1.15654 + 0.208872 0.208872 + -0.40363 -0.40363 + -2.79805 -2.79805 + -1.10689 -1.10689 + 1.06074 1.06074 + 2.46081 2.46081 + -3.64489 -3.64489 + 0.0728799 0.0728799 + 1.73179 1.73179 + -0.529045 -0.529045 + 0.661595 0.661595 + -1.01312 -1.01312 + 1.3391 1.3391 + 0.30467 0.30467 + 2.72219 2.72219 + 4.78742 4.78742 + -3.13914 -3.13914 + 1.88923 1.88923 + 2.43035 2.43035 + 0.547254 0.547254 + 0.406615 0.406615 + -3.88793 -3.88793 + -0.369601 -0.369601 + -0.868166 -0.868166 + -1.47208 -1.47208 + -0.294463 -0.294463 + -1.52052 -1.52052 + -1.18209 -1.18209 + 0.554649 0.554649 + 0.517766 0.517766 + -1.10023 -1.10023 + 1.65443 1.65443 + 2.07565 2.07565 + -0.788139 -0.788139 + 1.2133 1.2133 + -0.364007 -0.364007 + 1.45819 1.45819 + 2.62594 2.62594 + -3.35007 -3.35007 + -0.143461 -0.143461 + 0.21478 0.21478 + 0.800875 0.800875 + -0.329759 -0.329759 + 1.33073 1.33073 + 0.147415 0.147415 + -1.57766 -1.57766 + 0.151905 0.151905 + 1.63538 1.63538 + -0.0241539 -0.0241539 + 0.674812 0.674812 + 0.0364426 0.0364426 + 0.470479 0.470479 + 0.617235 0.617235 + 0.941854 0.941854 + -0.39178 -0.39178 + -3.72198 -3.72198 + 0.249689 0.249689 + 0.790793 0.790793 ```` ### Using postprocessing function @@ -4264,7 +4264,7 @@ msg = aiembed(schema, ```` ```` -DataMessage(Matrix{Float64} of size (4096, 2)) +PromptingTools.DataMessage(Matrix{Float64} of size (4096, 2)) ```` Cosine similarity is then a simple multiplication @@ -4275,8 +4275,8 @@ msg.content' * msg.content[:, 1] ```` 2-element Vector{Float64}: - 0.9999999999999946 - 0.34130017815042357 + 0.9999999999999982 + 0.40796033843072876 ```` --- diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md index 5d06928ea..cb51bdeb8 100644 --- a/docs/src/frequently_asked_questions.md +++ b/docs/src/frequently_asked_questions.md @@ -55,7 +55,7 @@ Resources: If you use a local model (eg, with Ollama), it's free. If you use any commercial APIs (eg, OpenAI), you will likely pay per "token" (a sub-word unit). -For example, a simple request with a simple question and 1 sentence response in return (”Is statement XYZ a positive comment”) will cost you ~$0.0001 (ie, one hundredth of a cent) +For example, a simple request with a simple question and 1 sentence response in return (”Is statement XYZ a positive comment”) will cost you ~$0.0001 (ie, one-hundredth of a cent) **Is it worth paying for?** @@ -90,7 +90,15 @@ A better way: Resources: - [OpenAI Guide](https://platform.openai.com/docs/quickstart?context=python) -Note: In the future, we hope to add `Preferences.jl`-based workflow to set the API key and other preferences. +## Setting the API Key via Preferences.jl + +You can also set the API key in `LocalPreferences.toml`, so it persists across sessions and projects. + +Use: `PromptingTools.set_preferences!("OPENAI_API_KEY"="your-api-key")` + +To double-check, run `PromptingTools.get_preferences("OPENAI_API_KEY")` and you should see your key! + +See more detail in the `?PromptingTools.PREFERENCES` docstring. ## Understanding the API Keyword Arguments in `aigenerate` (`api_kwargs`) diff --git a/examples/working_with_ollama.jl b/examples/working_with_ollama.jl index d1aad3176..a8ca77aa2 100644 --- a/examples/working_with_ollama.jl +++ b/examples/working_with_ollama.jl @@ -7,28 +7,50 @@ using PromptingTools const PT = PromptingTools -# Notice the schema change! If you want this to be the new default, you need to change `PT.PROMPT_SCHEMA` -schema = PT.OllamaManagedSchema() -# You can choose models from https://ollama.ai/library - I prefer `openhermes2.5-mistral` -model = "openhermes2.5-mistral" +# There were are several models from https://ollama.ai/library that we have added to our `PT.MODEL_REGISTRY`, which means you don't need to worry about schema changes: +# Eg, "llama2" or "openhermes2.5-mistral" (see `PT.list_registry()` and `PT.list_aliases()`) +# +# Note: You must download these models prior to using them with `ollama pull ` in your Terminal. # ## Text Generation with aigenerate # ### Simple message -msg = aigenerate(schema, "Say hi!"; model) +# +# TL;DR if you use models in `PT.MODEL_REGISTRY`, you don't need to add `schema` as the first argument: +# +msg = aigenerate("Say hi!"; model = "llama2") # ### Standard string interpolation +model = "openhermes2.5-mistral" + a = 1 -msg = aigenerate(schema, "What is `$a+$a`?"; model) +msg = aigenerate("What is `$a+$a`?"; model) name = "John" -msg = aigenerate(schema, "Say hi to {{name}}."; name, model) +msg = aigenerate("Say hi to {{name}}."; name, model) # ### Advanced Prompts conversation = [ PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), PT.UserMessage("I have feelings for my iPhone. What should I do?")] -msg = aigenerate(schema, conversation; model) +msg = aigenerate(conversation; model) + +# ### Schema Changes / Custom models +# If you're using some model that is not in the registry, you can either add it: +PT.register_model!(; + name = "llama123", + schema = PT.OllamaManagedSchema(), + description = "Some model") +PT.MODEL_ALIASES["l123"] = "llama123" # set an alias you like for it + +# OR define the schema explicitly (to avoid dispatch on global `PT.PROMPT_SCHEMA`): +schema = PT.OllamaManagedSchema() +aigenerate(schema, "Say hi!"; model = "llama2") + +# Note: If you only use Ollama, you can change the default schema to `PT.OllamaManagedSchema()` +# via `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaManagedSchema", "MODEL_CHAT"=>"llama2")` +# +# Restart your session and run `aigenerate("Say hi!")` to test it. # ## Embeddings with aiembed diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 701936fc7..7f5c6bdd1 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -6,7 +6,8 @@ using OpenAI using JSON3 using JSON3: StructTypes using HTTP -using Preferences +import Preferences +using Preferences: @load_preference, @set_preferences! using PrecompileTools # GLOBALS and Preferences are managed by Preferences.jl - see src/preferences.jl for details diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 484c6eb04..283031eb4 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -84,31 +84,32 @@ struct OllamaManagedSchema <: AbstractOllamaManagedSchema end end ## Dispatch into a default schema (can be set by Preferences.jl) +# Since we load it as strings, we need to convert it to a symbol and instantiate it const PROMPT_SCHEMA::AbstractPromptSchema = @load_preference("PROMPT_SCHEMA", - default=OpenAISchema()) + default="OpenAISchema") |> x -> getproperty(@__MODULE__, Symbol(x))() -function aigenerate(prompt; model, kwargs...) +function aigenerate(prompt; model = MODEL_CHAT, kwargs...) global MODEL_REGISTRY # first look up the model schema in the model registry; otherwise, use the default schema PROMPT_SCHEMA schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema aigenerate(schema, prompt; model, kwargs...) end -function aiembed(doc_or_docs, args...; kwargs...) +function aiembed(doc_or_docs, args...; model = MODEL_EMBEDDING, kwargs...) global MODEL_REGISTRY schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema aiembed(schema, doc_or_docs, args...; kwargs...) end -function aiclassify(prompt; kwargs...) +function aiclassify(prompt; model = MODEL_CHAT, kwargs...) global MODEL_REGISTRY schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema aiclassify(schema, prompt; kwargs...) end -function aiextract(prompt; kwargs...) +function aiextract(prompt; model = MODEL_CHAT, kwargs...) global MODEL_REGISTRY schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema aiextract(schema, prompt; kwargs...) end -function aiscan(prompt; kwargs...) +function aiscan(prompt; model = MODEL_CHAT, kwargs...) schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema aiscan(schema, prompt; kwargs...) end \ No newline at end of file diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 35add2ade..431976734 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -106,7 +106,7 @@ end ## User-Facing API """ aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, - model::String = MODEL_CHAT, + api_key::String = "", model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), @@ -181,7 +181,7 @@ Note: Managed Ollama currently supports at most 1 User Message and 1 System Mess """ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = "", model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], @@ -195,7 +195,8 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ if !dry_run time = @elapsed resp = ollama_api(prompt_schema, conv_rendered.prompt; - conv_rendered.system, endpoint = "generate", model, http_kwargs, api_kwargs...) + conv_rendered.system, endpoint = "generate", model_id, http_kwargs, + api_kwargs...) msg = AIMessage(; content = resp.response[:response] |> strip, status = Int(resp.status), tokens = (resp.response[:prompt_eval_count], @@ -223,7 +224,7 @@ end doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, postprocess::F = identity; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = "", model::String = MODEL_EMBEDDING, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, @@ -239,7 +240,7 @@ The `aiembed` function generates embeddings for the given input using a specifie so users should consider implementing an async version with with `Threads.@spawn` - `postprocess::F`: The post-processing function to apply to each embedding. Defaults to the identity function, but could be `LinearAlgebra.normalize`. - `verbose::Bool`: A flag indicating whether to print verbose information. Defaults to `true`. -- `api_key::String`: The API key to use for the OpenAI API. Defaults to `API_KEY`. +- `api_key::String`: The API key to use for the OpenAI API. Defaults to `""`. - `model::String`: The model to use for generating embeddings. Defaults to `MODEL_EMBEDDING`. - `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to empty `NamedTuple`. - `api_kwargs::NamedTuple`: Additional keyword arguments for the Ollama API. Defaults to an empty `NamedTuple`. @@ -295,7 +296,7 @@ msg.content # 4096-element Vector{Float64} function aiembed(prompt_schema::AbstractOllamaManagedSchema, doc::AbstractString, postprocess::F = identity; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = "", model::String = MODEL_EMBEDDING, http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) where {F <: Function} @@ -304,7 +305,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) time = @elapsed resp = ollama_api(prompt_schema, doc; - endpoint = "embeddings", model, http_kwargs, api_kwargs...) + endpoint = "embeddings", model_id, http_kwargs, api_kwargs...) msg = DataMessage(; content = postprocess(resp.response[:embedding]), status = Int(resp.status), @@ -318,7 +319,7 @@ end function aiembed(prompt_schema::AbstractOllamaManagedSchema, docs::Vector{<:AbstractString}, postprocess::F = identity; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = "", model::String = MODEL_EMBEDDING, kwargs...) where {F <: Function} ## diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 8a8fe70b9..b89542d00 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -60,7 +60,7 @@ end """ aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = OPENAI_API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, @@ -129,7 +129,7 @@ msg=aigenerate(conversation) """ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = OPENAI_API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = (retry_non_idempotent = true, @@ -191,7 +191,7 @@ end doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, postprocess::F = identity; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = OPENAI_API_KEY, model::String = MODEL_EMBEDDING, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, @@ -206,7 +206,7 @@ The `aiembed` function generates embeddings for the given input using a specifie - `doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}`: The document or list of documents to generate embeddings for. - `postprocess::F`: The post-processing function to apply to each embedding. Defaults to the identity function. - `verbose::Bool`: A flag indicating whether to print verbose information. Defaults to `true`. -- `api_key::String`: The API key to use for the OpenAI API. Defaults to `API_KEY`. +- `api_key::String`: The API key to use for the OpenAI API. Defaults to `OPENAI_API_KEY`. - `model::String`: The model to use for generating embeddings. Defaults to `MODEL_EMBEDDING`. - `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to `(retry_non_idempotent = true, retries = 5, readtimeout = 120)`. - `api_kwargs::NamedTuple`: Additional keyword arguments for the OpenAI API. Defaults to an empty `NamedTuple`. @@ -242,7 +242,7 @@ msg.content' * msg.content[:, 1] # [1.0, 0.787] function aiembed(prompt_schema::AbstractOpenAISchema, doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, postprocess::F = identity; verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = OPENAI_API_KEY, model::String = MODEL_EMBEDDING, http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, @@ -455,7 +455,7 @@ Note that the error message refers to a giraffe not being a human, function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; return_type::Type, verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = OPENAI_API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], @@ -517,7 +517,7 @@ aiscan([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, image_detail::AbstractString = "auto", attach_to_latest::Bool = true, - verbose::Bool = true, + verbose::Bool = true, api_key::String = OPENAI_API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], @@ -608,7 +608,7 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE image_detail::AbstractString = "auto", attach_to_latest::Bool = true, verbose::Bool = true, - api_key::String = API_KEY, + api_key::String = OPENAI_API_KEY, model::String = MODEL_CHAT, return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], diff --git a/src/precompilation.jl b/src/precompilation.jl index 2e2f4510b..0a540344a 100644 --- a/src/precompilation.jl +++ b/src/precompilation.jl @@ -1,6 +1,10 @@ # Load templates +load_template(joinpath(@__DIR__, "..", "templates", "general", "BlankSystemUser.json")) load_templates!(); +# Preferences +@load_preference("MODEL_CHAT", default="x") + # API Calls prep mock_response = Dict(:choices => [ Dict(:message => Dict(:content => "Hello!", diff --git a/src/user_preferences.jl b/src/user_preferences.jl index a2b6da8f7..ecf136221 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -5,15 +5,17 @@ PREFERENCES You can set preferences for PromptingTools by setting environment variables (for `OPENAI_API_KEY` only) - or by using the `@set_preferences!` macro (see `Preferences.jl`). + or by using the `set_preferences!`. It will create a `LocalPreferences.toml` file in your current directory and will reload your prefences from there. + +Check your preferences by calling `get_preferences(key::String)`. -If you can always check if a preference has been set by `@has_preference("")` or directly get its value by `@load_preference("")`. - -# Available Preferences (for `@set_preferences!`) +# Available Preferences (for `set_preferences!`) - `OPENAI_API_KEY`: The API key for the OpenAI API. See [OpenAI's documentation](https://platform.openai.com/docs/quickstart?context=python) for more information. - `MODEL_CHAT`: The default model to use for aigenerate and most ai* calls. See `MODEL_REGISTRY` for a list of available models or define your own. - `MODEL_EMBEDDING`: The default model to use for aiembed (embedding documents). See `MODEL_REGISTRY` for a list of available models or define your own. +- `PROMPT_SCHEMA`: The default prompt schema to use for aigenerate and most ai* calls (if not specified in `MODEL_REGISTRY`). Set as a string, eg, `"OpenAISchema"`. + See `PROMPT_SCHEMA` for more information. - `MODEL_ALIASES`: A dictionary of model aliases (`alias => full_model_name`). Aliases are used to refer to models by their aliases instead of their full names to make it more convenient to use them. See `MODEL_ALIASES` for more information. @@ -25,23 +27,81 @@ Define your `register_model!()` calls in your `startup.jl` file to make them ava Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. -WARNING: Never ever sync your `LocalPreferences.toml` file! It contains your API key and other sensitive information!!! +WARNING: NEVER EVER sync your `LocalPreferences.toml` file! It contains your API key and other sensitive information!!! """ const PREFERENCES = nothing +""" + set_preferences!(pairs::Pair{String, <:Any}...) + +Set preferences for PromptingTools. See `?PREFERENCES` for more information. + +See also: `get_preferences` + +# Example + +Change your API key and default model: +```julia +PromptingTools.set_preferences!("OPENAI_API_KEY" => "key1", "MODEL_CHAT" => "chat1") +``` +""" +function set_preferences!(pairs::Pair{String, <:Any}...) + allowed_preferences = [ + "OPENAI_API_KEY", + "MODEL_CHAT", + "MODEL_EMBEDDING", + "MODEL_ALIASES", + "PROMPT_SCHEMA", + ] + for (key, value) in pairs + @assert key in allowed_preferences "Unknown preference '$key'! (Allowed preferences: $(join(allowed_preferences,", "))" + @set_preferences!(key=>value) + if key == "MODEL_ALIASES" || key == "PROMPT_SCHEMA" + # cannot change in the same session + continue + else + setproperty!(@__MODULE__, Symbol(key), value) + end + end + @info("Preferences set; restart your Julia session for this change to take effect!") +end +""" + get_preferences(key::String) + +Get preferences for PromptingTools. See `?PREFERENCES` for more information. + +See also: `set_preferences!` + +# Example +```julia +PromptingTools.get_preferences("MODEL_CHAT") +``` +""" +function get_preferences(key::String) + allowed_preferences = [ + "OPENAI_API_KEY", + "MODEL_CHAT", + "MODEL_EMBEDDING", + "MODEL_ALIASES", + "PROMPT_SCHEMA", + ] + @assert key in allowed_preferences "Unknown preference '$key'! (Allowed preferences: $(join(allowed_preferences,", "))" + getproperty(@__MODULE__, Symbol(key)) +end + ## Load up GLOBALS const MODEL_CHAT::String = @load_preference("MODEL_CHAT", default="gpt-3.5-turbo") -const MODEL_EMBEDDING::String = @load_preference("MODEL_CHAT", +const MODEL_EMBEDDING::String = @load_preference("MODEL_EMBEDDING", default="text-embedding-ada-002") # the prompt schema default is defined in llm_interace.jl ! # const PROMPT_SCHEMA = OpenAISchema() # First, load from preferences, then from environment variables -const API_KEY::String = @load_preference("OPENAI_API_KEY", +const OPENAI_API_KEY::String = @load_preference("OPENAI_API_KEY", default=get(ENV, "OPENAI_API_KEY", "")) # Note: Disable this warning by setting OPENAI_API_KEY to anything -isempty(API_KEY) && - @warn "OPENAI_API_KEY environment variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.API_KEY=`!" +isempty(OPENAI_API_KEY) && + @warn "OPENAI_API_KEY variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.OPENAI_API_KEY=`!" ## Model registry # A dictionary of model names and their specs (ie, name, costs per token, etc.) @@ -280,6 +340,11 @@ function Base.delete!(registry::ModelRegistry, key::String) return registry end +"Shows the list of models in the registry. Add more with `register_model!`." +list_registry() = sort(collect(keys(MODEL_REGISTRY.registry))) +"Shows the Dictionary of model aliases in the registry. Add more with `MODEL_ALIASES[alias] = model_name`." +list_aliases() = MODEL_REGISTRY.aliases + """ MODEL_ALIASES diff --git a/test/user_preferences.jl b/test/user_preferences.jl index 0c570ca96..ce3b87b89 100644 --- a/test/user_preferences.jl +++ b/test/user_preferences.jl @@ -1,6 +1,39 @@ using PromptingTools: ModelSpec, register_model!, MODEL_REGISTRY, MODEL_ALIASES, ModelRegistry -using PromptingTools: OpenAISchema, OllamaManagedSchema +using PromptingTools: list_registry, list_aliases +using PromptingTools: OpenAISchema, OllamaManagedSchema, set_preferences!, get_preferences + +@testset "set_preferences!" begin + # Remember old preferences + OLD_MODEL_CHAT = get_preferences("MODEL_CHAT") + OLD_MODEL_EMBEDDING = get_preferences("MODEL_EMBEDDING") + OLD_OPENAI_API_KEY = get_preferences("OPENAI_API_KEY") + + # Test Setting Allowed Preferences + @testset "Allowed Preferences" for pref in [ + "OPENAI_API_KEY", + "MODEL_CHAT", + "MODEL_EMBEDDING", + ] + set_preferences!(pref => "test_value") + @test get_preferences(pref) == "test_value" # Assuming a get_preferences function exists + @test getproperty(PromptingTools, Symbol(pref)) == "test_value" # Check if the module-level variable is updated + end + + # Test Attempting to Set a Disallowed Preference + @test_throws AssertionError set_preferences!("UNKNOWN_PREF" => "value") + @test_throws AssertionError get_preferences("UNKNOWN_PREF") + + # Test Setting Multiple Preferences at Once + set_preferences!("OPENAI_API_KEY" => "key1", "MODEL_CHAT" => "chat1") + @test get_preferences("OPENAI_API_KEY") == "key1" + @test get_preferences("MODEL_CHAT") == "chat1" + + # Return back to previous state + set_preferences!("OPENAI_API_KEY" => OLD_OPENAI_API_KEY, + "MODEL_CHAT" => OLD_MODEL_CHAT, + "MODEL_EMBEDDING" => OLD_MODEL_EMBEDDING) +end @testset "ModelSpec" begin # Test for Correct Initialization @@ -88,4 +121,8 @@ end expected_output = "ModelRegistry with $(length(MODEL_REGISTRY.registry)) models and $(length(MODEL_REGISTRY.aliases)) aliases. See `?MODEL_REGISTRY` for more information." @test output == expected_output + + # list functions + @test list_registry() == sort(collect(keys(MODEL_REGISTRY.registry))) + @test list_aliases() == MODEL_REGISTRY.aliases end From 3635c0f4fc06e79cfe9b6970a46c700803bf4af7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 13:28:36 +0000 Subject: [PATCH 045/251] fix docs --- README.md | 2 +- docs/src/examples/working_with_aitemplates.md | 2 +- docs/src/examples/working_with_ollama.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6182ad3ec..0dbf0978f 100644 --- a/README.md +++ b/README.md @@ -514,7 +514,7 @@ Resources: If you use a local model (eg, with Ollama), it's free. If you use any commercial APIs (eg, OpenAI), you will likely pay per "token" (a sub-word unit). -For example, a simple request with a simple question and 1 sentence response in return (”Is statement XYZ a positive comment”) will cost you ~$0.0001 (ie, one hundredth of a cent) +For example, a simple request with a simple question and 1 sentence response in return (”Is statement XYZ a positive comment”) will cost you ~$0.0001 (ie, one-hundredth of a cent) **Is it worth paying for?** diff --git a/docs/src/examples/working_with_aitemplates.md b/docs/src/examples/working_with_aitemplates.md index 8b8eb76a0..dacfad0c0 100644 --- a/docs/src/examples/working_with_aitemplates.md +++ b/docs/src/examples/working_with_aitemplates.md @@ -1,5 +1,5 @@ ```@meta -EditURL = "/examples/working_with_aitemplates.jl" +EditURL = "../../../examples/working_with_aitemplates.jl" ``` # Using AITemplates diff --git a/docs/src/examples/working_with_ollama.md b/docs/src/examples/working_with_ollama.md index da9f0eb70..4f5f92d3b 100644 --- a/docs/src/examples/working_with_ollama.md +++ b/docs/src/examples/working_with_ollama.md @@ -1,5 +1,5 @@ ```@meta -EditURL = "/examples/working_with_ollama.jl" +EditURL = "../../../examples/working_with_ollama.jl" ``` # Local models with Ollama.ai From 26037e336fbe3aa24a9ac3740150813731e5786d Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 13:43:25 +0000 Subject: [PATCH 046/251] remove abstract type --- src/code_generation.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index daeabe842..ff10030f2 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -114,8 +114,7 @@ function Base.copy(cb::AbstractCodeBlock) AICode(cb.code, cb.expression, cb.stdout, cb.output, cb.success, cb.error) end # equality check for testing, only equal if all fields are equal and type is the same -Base.var"=="(m1::AbstractCodeBlock, m2::AbstractCodeBlock) = false -function Base.var"=="(c1::T, c2::T) where {T <: AbstractCodeBlock} +function Base.var"=="(c1::T, c2::T) where {T <: AICode} all([getproperty(c1, f) == getproperty(c2, f) for f in fieldnames(T)]) end function Base.show(io::IO, cb::AICode) From d7b5cd6836ab6f7960a064d1e7f781e3357f7440 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 14:00:56 +0000 Subject: [PATCH 047/251] add interface tests --- src/llm_interface.jl | 2 +- test/llm_interface.jl | 68 +++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 test/llm_interface.jl diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 283031eb4..990798000 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -85,7 +85,7 @@ end ## Dispatch into a default schema (can be set by Preferences.jl) # Since we load it as strings, we need to convert it to a symbol and instantiate it -const PROMPT_SCHEMA::AbstractPromptSchema = @load_preference("PROMPT_SCHEMA", +global PROMPT_SCHEMA::AbstractPromptSchema = @load_preference("PROMPT_SCHEMA", default="OpenAISchema") |> x -> getproperty(@__MODULE__, Symbol(x))() function aigenerate(prompt; model = MODEL_CHAT, kwargs...) diff --git a/test/llm_interface.jl b/test/llm_interface.jl new file mode 100644 index 000000000..18bd78744 --- /dev/null +++ b/test/llm_interface.jl @@ -0,0 +1,68 @@ +using PromptingTools: TestEchoOpenAISchema, render, OpenAISchema +using PromptingTools: AIMessage, SystemMessage, AbstractMessage +using PromptingTools: UserMessage, UserMessageWithImages, DataMessage + +@testset "ai* default schema" begin + OLD_PROMPT_SCHEMA = PromptingTools.PROMPT_SCHEMA + ### AIGenerate + # corresponds to OpenAI API v1 + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + + schema = TestEchoOpenAISchema(; response, status = 200) + PromptingTools.PROMPT_SCHEMA = schema + msg = aigenerate("Hello World"; model = "xyz") + expected_output = AIMessage(; + content = "Hello!" |> strip, + status = 200, + tokens = (2, 1), + elapsed = msg.elapsed) + @test msg == expected_output + + ### AIClassify + msg = aiclassify("Hello World"; model = "xyz") + expected_output = AIMessage(; + content = "Hello!" |> strip, + status = 200, + tokens = (2, 1), + elapsed = msg.elapsed) + @test msg == expected_output + + ### AIExtract + response1 = Dict(:choices => [ + Dict(:message => Dict(:function_call => Dict(:arguments => "{\"content\": \"x\"}"))), + ], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + response1[:choices][begin][:message][:function_call][:arguments] |> + x -> JSON3.read(x, MyType) + schema = TestEchoOpenAISchema(; response = response1, status = 200) + PromptingTools.PROMPT_SCHEMA = schema + struct MyType + content::String + end + msg = aiextract("Hello World"; model = "xyz", return_type = MyType) + expected_output = DataMessage(; + content = MyType("x"), + status = 200, + tokens = (2, 1), + elapsed = msg.elapsed) + @test msg == expected_output + + # corresponds to OpenAI API v1 + response2 = Dict(:data => [Dict(:embedding => ones(128))], + :usage => Dict(:total_tokens => 2, :prompt_tokens => 2, :completion_tokens => 0)) + + # Real generation API + schema2 = TestEchoOpenAISchema(; response = response2, status = 200) + PromptingTools.PROMPT_SCHEMA = schema2 + msg = aiembed("Hello World"; model = "xyz") + expected_output = DataMessage(; + content = ones(128), + status = 200, + tokens = (2, 0), + elapsed = msg.elapsed) + @test msg == expected_output + + ## Return things to previous + PromptingTools.PROMPT_SCHEMA = OLD_PROMPT_SCHEMA +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index aadb01abe..cc7a6f672 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,6 +13,7 @@ end include("messages.jl") include("extraction.jl") include("user_preferences.jl") + include("llm_interface.jl") include("llm_shared.jl") include("llm_openai.jl") include("templates.jl") From 815e40f06578e27636922e454825628ddfbc394e Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 14:01:20 +0000 Subject: [PATCH 048/251] fix typo --- test/llm_interface.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/llm_interface.jl b/test/llm_interface.jl index 18bd78744..c012b706d 100644 --- a/test/llm_interface.jl +++ b/test/llm_interface.jl @@ -33,8 +33,7 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage Dict(:message => Dict(:function_call => Dict(:arguments => "{\"content\": \"x\"}"))), ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) - response1[:choices][begin][:message][:function_call][:arguments] |> - x -> JSON3.read(x, MyType) + schema = TestEchoOpenAISchema(; response = response1, status = 200) PromptingTools.PROMPT_SCHEMA = schema struct MyType From b659507c8102f6502c0d88217a9566622f9ace8f Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 14:12:38 +0000 Subject: [PATCH 049/251] version number --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index f4ac50d9b..bb1236c5b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.3.0-DEV" +version = "0.3.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From d8a0a50ff86c0034aa6932f96c9209f33f3f8b31 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 10 Dec 2023 14:13:59 +0000 Subject: [PATCH 050/251] update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5a18508..6c60b5cb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.3.0] + ### Added - Introduced a set of utilities for working with generate Julia code (Eg, extract code-fenced Julia code with `PromptingTools.extract_code_blocks` ) or simply apply `AICode` to the AI messages. `AICode` tries to extract, parse and eval Julia code, if it fails both stdout and errors are captured. It is useful for generating Julia code and, in the future, creating self-healing code agents - Introduced ability to have multi-turn conversations. Set keyword argument `return_all=true` and `ai*` functions will return the whole conversation, not just the last message. To continue a previous conversation, you need to provide it to a keyword argument `conversation` From 0dc08e391f0913a0aa29f9caa3c49e82efc192fa Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 11 Dec 2023 21:41:03 +0000 Subject: [PATCH 051/251] catch parsing errors --- src/code_generation.jl | 32 +++++++++++++++++++++++++++++++- test/code_generation.jl | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index ff10030f2..c11b7d4ba 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -129,6 +129,26 @@ function Base.show(io::IO, cb::AICode) "AICode(Success: $success_str, Parsed: $expression_str, Evaluated: $output_str, Error Caught: $error_str, StdOut: $stdout_str, Code: $count_lines Lines)") end +## Parsing error detection +function isparsed(ex::Expr) + parse_error = Meta.isexpr(ex, :toplevel) && !isempty(ex.args) && + Meta.isexpr(ex.args[end], (:error, :incomplete)) + return !parse_error +end +function isparsed(ex::Nothing) + return false +end +function isparseerror(err::Exception) + return err isa Base.Meta.ParseError || + (err isa ErrorException && startswith(err.msg, "syntax:")) +end +function isparseerror(err::Nothing) + return false +end +function isparsed(cb::AICode) + return isparsed(cb.expression) && !isparseerror(cb.error) +end + ## Overload for AIMessage - simply extracts the code blocks and concatenates them function AICode(msg::AIMessage; kwargs...) code = extract_code_blocks(msg.content) |> Base.Fix2(join, "\n") @@ -171,7 +191,7 @@ function detect_missing_packages(imports_required::AbstractVector{<:Symbol}) end "Checks if a given string has a Julia prompt (`julia> `) at the beginning of a line." -has_julia_prompt(s::T) where {T <: AbstractString} = occursin(r"^julia> "m, s) +has_julia_prompt(s::T) where {T <: AbstractString} = occursin(r"(:?^julia> |^> )"m, s) """ remove_julia_prompt(s::T) where {T<:AbstractString} @@ -191,6 +211,10 @@ function remove_julia_prompt(s::T) where {T <: AbstractString} code_line = true # remove the prompt println(io, replace(line, "julia> " => "")) + elseif startswith(line, r"^> ") + code_line = true + # remove the prompt + println(io, replace(line, "> " => "")) elseif code_line && startswith(line, r"^ ") # continuation of the code line println(io, line) @@ -430,6 +454,12 @@ function eval!(cb::AbstractCodeBlock; return cb end end + ## Catch bad code extraction + if isempty(code) + cb.error = ErrorException("Parse Error: No code found!") + cb.success = false + return cb + end ## Parse into an expression try ex = Meta.parseall(code_extra) diff --git a/test/code_generation.jl b/test/code_generation.jl index 38bfb87fc..56691b71d 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -2,6 +2,7 @@ using PromptingTools: extract_julia_imports using PromptingTools: detect_pkg_operation, detect_missing_packages, extract_function_name using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, eval! using PromptingTools: escape_interpolation, find_subsequence_positions +using PromptingTools: AICode, isparsed, isparseerror @testset "extract_imports tests" begin @test extract_julia_imports("using Test, LinearAlgebra") == @@ -32,10 +33,17 @@ end @testset "has_julia_prompt" begin @test has_julia_prompt("julia> a=1") + @test has_julia_prompt("> a=1") @test has_julia_prompt(""" # something else first julia> a=1 """) + @test has_julia_prompt(""" + > a=\"\"\" + hey + there + \"\"\" + """) @test !has_julia_prompt(""" # something # new @@ -45,6 +53,7 @@ end @testset "remove_julia_prompt" begin @test remove_julia_prompt("julia> a=1") == "a=1" + @test remove_julia_prompt("> a=1") == "a=1" @test remove_julia_prompt(""" # something else first julia> a=1 @@ -281,8 +290,13 @@ end @test cb.output == 123 @test a123 == 123 + # Check that empty code is invalid + cb = AICode("") + @test !isvalid(cb) + @test cb.error isa Exception + # Test prefix and suffix - cb = AICode(; code = "") + cb = AICode(; code = "x=1") eval!(cb; prefix = "a=1", suffix = "b=2") @test cb.output.a == 1 @test cb.output.b == 2 @@ -391,4 +405,27 @@ end code1 = AICode("print(\"Hello\")"; safe_eval = true) code2 = AICode("print(\"Hello\")"; safe_eval = false) @test code1 != code2 +end +@testset "isparsed, isparseerror" begin + ## isparsed + @test isparsed(:(x = 1)) == true + # parse an incomplete call + @test isparsed(Meta.parseall("(")) == false + # parse an error call + @test isparsed(Meta.parseall("+-+-+--+")) == false + # nothing + @test isparsed(nothing) == false + # Validate that we don't have false positives with error + @test isparsed(Meta.parseall("error(\"s\")")) == true + + ## isparseerror + @test isparseerror(nothing) == false + @test isparseerror(ErrorException("syntax: unexpected \"(\" in argument list")) == true + @test isparseerror(Base.Meta.ParseError("xyz")) == true + + # AICode + cb = AICode("(") + @test isparsed(cb) == false + cb = AICode("a+1") + @test isparsed(cb) == true end \ No newline at end of file From cfd3c1f6c57858f878c306b939f5d1f1013e43d7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:57:25 +0000 Subject: [PATCH 052/251] improve code parsing --- CHANGELOG.md | 1 + Project.toml | 2 +- src/code_generation.jl | 23 ++++++++++++++++++++++- test/code_generation.jl | 33 ++++++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c60b5cb3..d4821241b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Improved AICode parsing and error handling (eg, capture more REPL prompts, detect parsing errors earlier), including the option to remove unsafe code (eg, `Pkg.add("SomePkg")`) with `AICode(msg; skip_unsafe=true, vebose=true)` ### Fixed diff --git a/Project.toml b/Project.toml index bb1236c5b..5957c08c7 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.3.0" +version = "0.4.0-DEV" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/code_generation.jl b/src/code_generation.jl index c11b7d4ba..578e99685 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -150,8 +150,12 @@ function isparsed(cb::AICode) end ## Overload for AIMessage - simply extracts the code blocks and concatenates them -function AICode(msg::AIMessage; kwargs...) +function AICode(msg::AIMessage; + verbose::Bool = false, + skip_unsafe::Bool = false, + kwargs...) code = extract_code_blocks(msg.content) |> Base.Fix2(join, "\n") + skip_unsafe && (code = remove_unsafe_lines(code; verbose)) return AICode(code; kwargs...) end @@ -181,6 +185,9 @@ end # Utility to pinpoint unavailable dependencies function detect_missing_packages(imports_required::AbstractVector{<:Symbol}) + # shortcut if no packages are required + isempty(imports_required) && return false, Symbol[] + # available_packages = Base.loaded_modules |> values .|> Symbol missing_packages = filter(pkg -> !in(pkg, available_packages), imports_required) if length(missing_packages) > 0 @@ -190,6 +197,20 @@ function detect_missing_packages(imports_required::AbstractVector{<:Symbol}) end end +"Iterates over the lines of a string and removes those that contain a package operation or a missing import." +function remove_unsafe_lines(code::AbstractString; verbose::Bool = false) + io = IOBuffer() + for line in readlines(IOBuffer(code)) + if !detect_pkg_operation(line) && + !detect_missing_packages(extract_julia_imports(line))[1] + println(io, line) + else + verbose && @info "Unsafe line removed: $line" + end + end + return String(take!(io)) +end + "Checks if a given string has a Julia prompt (`julia> `) at the beginning of a line." has_julia_prompt(s::T) where {T <: AbstractString} = occursin(r"(:?^julia> |^> )"m, s) diff --git a/test/code_generation.jl b/test/code_generation.jl index 56691b71d..0e69ef6f5 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -1,5 +1,6 @@ using PromptingTools: extract_julia_imports -using PromptingTools: detect_pkg_operation, detect_missing_packages, extract_function_name +using PromptingTools: detect_pkg_operation, + detect_missing_packages, extract_function_name, remove_unsafe_lines using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, eval! using PromptingTools: escape_interpolation, find_subsequence_positions using PromptingTools: AICode, isparsed, isparseerror @@ -31,6 +32,20 @@ end @test detect_pkg_operation("import Pkg;") == false end +@testset "remove_unsafe_lines" begin + @test remove_unsafe_lines("Pkg.activate(\".\")") == "" + @test remove_unsafe_lines("Pkg.add(\"SomePkg\")") == "" + s = """ + a=1 + Pkg.add("a") + b=2 + Pkg.add("b") + using 12315456NotExisting + """ + @test remove_unsafe_lines(s) == "a=1\nb=2\n" + @test remove_unsafe_lines("Nothing"; verbose = true) == "Nothing\n" +end + @testset "has_julia_prompt" begin @test has_julia_prompt("julia> a=1") @test has_julia_prompt("> a=1") @@ -345,6 +360,22 @@ b=2 @test cb.stdout == "hello\nworld\n" @test cb.output.b == 2 end + # skip_unsafe=true + s = """ + + """ + let msg = AIMessage(""" + ```julia + a=1 + Pkg.add("a") + b=2 + Pkg.add("b") + using 12315456NotExisting + ``` + """) + cb = AICode(msg; skip_unsafe = true) + @test cb.code == "a=1\nb=2\n" + end # Methods - copy let msg = AIMessage(""" From 47f25ea14f4ba1b3d1a52275db6df423292d8b0a Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 12 Dec 2023 21:14:14 +0000 Subject: [PATCH 053/251] update tests --- src/code_generation.jl | 2 +- test/code_generation.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index 578e99685..4a0e96b39 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -119,7 +119,7 @@ function Base.var"=="(c1::T, c2::T) where {T <: AICode} end function Base.show(io::IO, cb::AICode) success_str = cb.success === nothing ? "N/A" : titlecase(string(cb.success)) - expression_str = cb.expression === nothing ? "N/A" : "True" + expression_str = cb.expression === nothing ? "N/A" : titlecase(string(isparsed(cb))) stdout_str = cb.stdout === nothing ? "N/A" : "True" output_str = cb.output === nothing ? "N/A" : "True" error_str = cb.error === nothing ? "N/A" : "True" diff --git a/test/code_generation.jl b/test/code_generation.jl index 0e69ef6f5..6ec54cc44 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -414,7 +414,7 @@ end "AICode(Success: True, Parsed: True, Evaluated: True, Error Caught: N/A, StdOut: True, Code: 1 Lines)" # Test with error - code_block = AICode("error(\"Test Error\"))\nprint(\"\")") + code_block = AICode("error(\"Test Error\")\nprint(\"\")") buffer = IOBuffer() show(buffer, code_block) output = String(take!(buffer)) From 3862b1cbc5572c5627e9cb27cc5f0d350b6c1a26 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:15:14 +0000 Subject: [PATCH 054/251] updated templates --- .../persona-task/JuliaDataExpertCoTTask.json | 22 +++++++++++++++++++ .../persona-task/JuliaExpertTestCode.json | 22 +++++++++++++++++++ templates/persona-task/JuliaRecapCoTTask.json | 22 +++++++++++++++++++ templates/persona-task/JuliaRecapTask.json | 22 +++++++++++++++++++ 4 files changed, 88 insertions(+) create mode 100644 templates/persona-task/JuliaDataExpertCoTTask.json create mode 100644 templates/persona-task/JuliaExpertTestCode.json create mode 100644 templates/persona-task/JuliaRecapCoTTask.json create mode 100644 templates/persona-task/JuliaRecapTask.json diff --git a/templates/persona-task/JuliaDataExpertCoTTask.json b/templates/persona-task/JuliaDataExpertCoTTask.json new file mode 100644 index 000000000..c5d6eb88a --- /dev/null +++ b/templates/persona-task/JuliaDataExpertCoTTask.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`", + "version": "2.0", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer and very systematic in your approach to solving problems. \nYou follow the below approach when writing code. Your communication is brief and concise.\n\nProblem Solving Steps:\n- Think through your approach step by step\n- Write any functions and other code you need\n- Solve the task\n- Check that your solution is correct\n\nYou precisely follow the given Task and use the Data when provided. When Data is not provided, create some examples.\n", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}\n", + "variables": [ + "task", + "data" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertTestCode.json b/templates/persona-task/JuliaExpertTestCode.json new file mode 100644 index 000000000..a05c5512e --- /dev/null +++ b/templates/persona-task/JuliaExpertTestCode.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "For writing Julia-style unit tests. It expects `code` provided as a string (it can be the whole source code of your app). Instructions are a good way to guide the model which functions to test and how. If you don't need the instructions, set `instructions=\"None.\"`. Placeholders: {{code}}, {{instructions}}", + "version": "1", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer and expert in writing unit and integration tests for Julia applications.\n\nYour task is to write tests for the User's code (or a subset of it).\n\nGeneral Guidelines:\n- Your tests must be as compact as possible while comprehensively covering the functionality of the code\n- Testsets are named after the function\n- Include a brief comment explaining the purpose of each test\n- Write multiple test cases using `@test` to validate different aspects of the `add` function. Think about all pathways through the code and test each one.\n\nIf the user provides any Special Instructions, prioritize them over the General Guidelines.\n\n\nExample:\n\"\"\"\n**User's code:**\n\n```julia\nmyadd(a, b) = a + b\n```\n\n**Response:**\n\n```julia\nusing Test\n\n@testset \"myadd\" begin\n \n # \n\n # Test for correct addition of positive numbers\n @test myadd(2, 3) == 5\n\n # Test for correct addition with a negative number\n @test myadd(-1, 3) == 2\n\n # Test for correct addition with zero\n @test myadd(0, 0) == 0\n\n # Test for correct addition of large numbers\n @test myadd(1000, 2000) == 3000\nend\n```\n\"\"\"\n", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# User's Code\n\n{{code}}\n\n\n# Special Instructions\n\n{{instructions}}\n", + "variables": [ + "code", + "instructions" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/JuliaRecapCoTTask.json b/templates/persona-task/JuliaRecapCoTTask.json new file mode 100644 index 000000000..11804f6a0 --- /dev/null +++ b/templates/persona-task/JuliaRecapCoTTask.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "Not all models know Julia syntax well. This template carries an extensive summary of key information about Julia and its syntax. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`", + "version": "1.0", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer and have a very systematic approach to solving problems.\n\nProblem Solving Steps:\n- Recall Julia snippets that will be useful for this Task\n- Solve the Task\n- Double-check that the solution is correct\n\nReminder on Julia Language:\n- Key Syntax: variables `x = 10`, control structures `if-elseif-else`, `isX ? X : Y`, `for`, `while`; functions `function f(x) end`, anonymous `x -> x^2`, arrays `[1, 2, 3]`, slicing `a[1:2]`, tuples `(1, 2)`, namedtuples `(; name=\"Julia\", )`, dictionary `Dict(\"key\" => value)`, `$` for string interpolation. \n- Prefer Julia standard libraries, avoid new packages unless explicitly requested. \n- Use general type annotations like `Number` or `AbstractString` to not be too restrictive. Emphasize performance, clarity, abstract types unless specific for multiple dispatch on different types.\n- Reserved names: `begin`, `end`, `function`. \n- Distinguished from Python with 1-based indexing, multiple dispatch\n\nIf the user provides any Special Instructions, prioritize them over the above guidelines.\n ", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}\n\n\n\n# Special Instructions\n\n{{instructions}}\n", + "variables": [ + "task", + "instructions" + ], + "_type": "usermessage" + } +] \ No newline at end of file diff --git a/templates/persona-task/JuliaRecapTask.json b/templates/persona-task/JuliaRecapTask.json new file mode 100644 index 000000000..0831e0689 --- /dev/null +++ b/templates/persona-task/JuliaRecapTask.json @@ -0,0 +1,22 @@ +[ + { + "content": "Template Metadata", + "description": "Not all models know Julia syntax well. This template carries a small summary of key information about Julia and its syntax and it will always first recall the Julia facts. If you don't need any instructions, set `instructions=\"None.\"`. Placeholders: `task`, `instructions`", + "version": "1.0", + "source": "", + "_type": "metadatamessage" + }, + { + "content": "You are a world-class Julia language programmer and have a very systematic approach to solving problems.\n\nProblem Solving Steps:\n- Recall Julia snippets that will be useful for this Task\n- Solve the Task\n- Double-check that the solution is correct\n\nReminder on Julia Language:\n- Key Syntax: variables `x = 10`, control structures `if-elseif-else`, `isX ? X : Y`, `for`, `while`; functions `function f(x) end`, anonymous `x -> x^2`, arrays `[1, 2, 3]`, slicing `a[1:2]`, tuples `(1, 2)`, namedtuples `(; name=\"Julia\", )`, dictionary `Dict(\"key\" => value)`, `$` for string interpolation. \n- Prefer Julia standard libraries, avoid new packages unless explicitly requested. \n- Use general type annotations like `Number` or `AbstractString` to not be too restrictive. Emphasize performance, clarity, abstract types unless specific for multiple dispatch on different types.\n- Reserved names: `begin`, `end`, `function`. \n- Distinguished from Python with 1-based indexing, multiple dispatch\n\nIf the user provides any Special Instructions, prioritize them over the above guidelines.\n ", + "variables": [], + "_type": "systemmessage" + }, + { + "content": "# Task\n\n{{task}}\n\n\n\n# Special Instructions\n\n{{instructions}}\n", + "variables": [ + "task", + "instructions" + ], + "_type": "usermessage" + } +] \ No newline at end of file From 84b21456421bb9b4e03c7b0fff9d54908d8cec00 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 13 Dec 2023 20:09:04 +0000 Subject: [PATCH 055/251] add templates --- CHANGELOG.md | 3 +- src/code_generation.jl | 131 +++++++++++++++++- .../persona-task/JuliaDataExpertCoTTask.json | 22 --- .../persona-task/JuliaExpertCoTTask.json | 4 +- test/code_generation.jl | 107 +++++++++++++- 5 files changed, 235 insertions(+), 32 deletions(-) delete mode 100644 templates/persona-task/JuliaDataExpertCoTTask.json diff --git a/CHANGELOG.md b/CHANGELOG.md index d4821241b..8532cf5f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Improved AICode parsing and error handling (eg, capture more REPL prompts, detect parsing errors earlier), including the option to remove unsafe code (eg, `Pkg.add("SomePkg")`) with `AICode(msg; skip_unsafe=true, vebose=true)` +- Improved AICode parsing and error handling (eg, capture more REPL prompts, detect parsing errors earlier, parse more code fence types), including the option to remove unsafe code (eg, `Pkg.add("SomePkg")`) with `AICode(msg; skip_unsafe=true, vebose=true)` +- Added new prompt templates: `JuliaRecapTask`, `JuliaRecapCoTTask`, `JuliaExpertTestCode` and updated `JuliaExpertCoTTask` to be more robust against early stopping for smaller OSS models ### Fixed diff --git a/src/code_generation.jl b/src/code_generation.jl index 4a0e96b39..84e13d778 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -149,12 +149,79 @@ function isparsed(cb::AICode) return isparsed(cb.expression) && !isparseerror(cb.error) end +## Parsing Helpers +JULIA_EXPR_HEADS = [ + :block, + :quote, + :call, + :macrocall, + :(=), + :function, + :for, + :if, + :while, + :let, + :try, + :catch, + :finally, + :method, + :tuple, + :array, + :index, + :ref, + :., + :do, + :curly, + :typed_vcat, + :typed_hcat, + :typed_vcat, + :comprehension, + :generator, + :kw, + :where, +] +# Checks if the provided expression `ex` has some hallmarks of Julia code. Very naive! +# Serves as a quick check to avoid trying to eval output cells (```plaintext ... ```) +is_julia_expr(ex::Any) = false +function is_julia_expr(ex::Expr) + ## Expression itself + Meta.isexpr(ex, JULIA_EXPR_HEADS) && return true + ## Its arguments + for arg in ex.args + Meta.isexpr(arg, JULIA_EXPR_HEADS) && return true + end + ## Nothing found... + return false +end + +## Check if a given String seems to be a valid Julia expression (simple heuristics) +function is_julia_code(code::AbstractString) + # Try to parse the expression, return false if parsing fails + expr = try + Meta.parseall(code) + catch + return false + end + + if isparsed(expr) && is_julia_expr(expr) + return true + else + return false + end +end + ## Overload for AIMessage - simply extracts the code blocks and concatenates them function AICode(msg::AIMessage; verbose::Bool = false, skip_unsafe::Bool = false, kwargs...) - code = extract_code_blocks(msg.content) |> Base.Fix2(join, "\n") + code = extract_code_blocks(msg.content) + if isempty(code) + ## Fallback option for generic code fence, we must check if the content is parseable + code = extract_code_blocks_fallback(msg.content) |> + x -> filter(is_julia_code, x) + end + code = join(code, "\n") skip_unsafe && (code = remove_unsafe_lines(code; verbose)) return AICode(code; kwargs...) end @@ -176,8 +243,10 @@ function extract_julia_imports(input::AbstractString) subparts = map(x -> contains(x, ':') ? split(x, ':')[1] : x, split(subparts, ",")) subparts = replace(join(subparts, ' '), ',' => ' ') - packages = filter(!isempty, split(subparts, " ")) .|> Symbol - append!(package_names, packages) + packages = filter(x -> !isempty(x) && !startswith(x, "Base") && + !startswith(x, "Main"), + split(subparts, " ")) + append!(package_names, Symbol.(packages)) end end return package_names @@ -303,6 +372,8 @@ The extracted code blocks are returned as a vector of strings, with each string Note: Only the content within the code fences is extracted, and the code fences themselves are not included in the output. +See also: `extract_code_blocks_fallback` + # Arguments - `markdown_content::String`: A string containing the markdown content from which Julia code blocks are to be extracted. @@ -379,6 +450,60 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString} return reverse(code_blocks) # Reverse to maintain original order end +""" + extract_code_blocks_fallback(markdown_content::String, delim::AbstractString="```") + +Extract Julia code blocks from a markdown string using a fallback method (splitting by arbitrary `delim`-iters). +Much more simplistic than `extract_code_blocks` and does not support nested code blocks. + +It is often used as a fallback for smaller LLMs that forget to code fence ```julia ... ```. + +# Example + +```julia +code = \"\"\" +\`\`\` +println("hello") +\`\`\` + +Some text + +\`\`\` +println("world") +\`\`\` +\"\"\" + +# We extract text between triple backticks and check each blob if it looks like a valid Julia code +code_parsed = extract_code_blocks_fallback(code) |> x -> filter(is_julia_code, x) |> x -> join(x, "\n") +``` +""" +function extract_code_blocks_fallback(markdown_content::T, + delim::AbstractString = "```") where {T <: AbstractString} + # Convert content and delimiters to codeunits + content_units = codeunits(markdown_content) + delim_units = codeunits(delim) + delim_positions = find_subsequence_positions(delim_units, content_units) + + # Extract code blocks + eltype_ = typeof(@view(markdown_content[begin:end])) + code_blocks = Vector{eltype_}() + isempty(delim_positions) && return code_blocks + + # Run the extraction + start_pos = delim_positions[1] + for end_pos in delim_positions + if end_pos > start_pos + code_block = markdown_content[(start_pos + length(delim_units)):(end_pos - 1)] + # Also remove the julia prompt + push!(code_blocks, remove_julia_prompt(strip(code_block))) + # Reset the start + start_pos = end_pos + end + end + + return code_blocks +end + """ extract_function_name(code_block::String) -> Union{String, Nothing} diff --git a/templates/persona-task/JuliaDataExpertCoTTask.json b/templates/persona-task/JuliaDataExpertCoTTask.json deleted file mode 100644 index c5d6eb88a..000000000 --- a/templates/persona-task/JuliaDataExpertCoTTask.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "content": "Template Metadata", - "description": "For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`", - "version": "2.0", - "source": "", - "_type": "metadatamessage" - }, - { - "content": "You are a world-class Julia language programmer and very systematic in your approach to solving problems. \nYou follow the below approach when writing code. Your communication is brief and concise.\n\nProblem Solving Steps:\n- Think through your approach step by step\n- Write any functions and other code you need\n- Solve the task\n- Check that your solution is correct\n\nYou precisely follow the given Task and use the Data when provided. When Data is not provided, create some examples.\n", - "variables": [], - "_type": "systemmessage" - }, - { - "content": "# Task\n\n{{task}}\n\n\n\n# Data\n\n{{data}}\n", - "variables": [ - "task", - "data" - ], - "_type": "usermessage" - } -] \ No newline at end of file diff --git a/templates/persona-task/JuliaExpertCoTTask.json b/templates/persona-task/JuliaExpertCoTTask.json index a45885bc4..2302a8dda 100644 --- a/templates/persona-task/JuliaExpertCoTTask.json +++ b/templates/persona-task/JuliaExpertCoTTask.json @@ -2,12 +2,12 @@ { "content": "Template Metadata", "description": "For small code task in Julia language. It will first describe the approach (CoT = Chain of Thought). Placeholders: `task`, `data`", - "version": "1", + "version": "2.0", "source": "", "_type": "metadatamessage" }, { - "content": "You are a world-class Julia language programmer with the knowledge of the latest syntax. Your communication is brief and concise. You precisely follow the given task and use the data when provided. When no data is provided, create some examples. First, think through your approach step by step. Then implement the solution.", + "content": "You are a world-class Julia language programmer and very systematic in your approach to solving problems. \nYou follow the below approach when writing code. Your communication is brief and concise.\n\nProblem Solving Steps:\n- Think through your approach step by step\n- Write any functions and other code you need\n- Solve the task\n- Check that your solution is correct\n\nYou precisely follow the given Task and use the Data when provided. When Data is not provided, create some examples.\n", "variables": [], "_type": "systemmessage" }, diff --git a/test/code_generation.jl b/test/code_generation.jl index 6ec54cc44..d3adbcd5a 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -1,9 +1,64 @@ using PromptingTools: extract_julia_imports using PromptingTools: detect_pkg_operation, detect_missing_packages, extract_function_name, remove_unsafe_lines -using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, eval! +using PromptingTools: has_julia_prompt, + remove_julia_prompt, extract_code_blocks, extract_code_blocks_fallback, eval! using PromptingTools: escape_interpolation, find_subsequence_positions -using PromptingTools: AICode, isparsed, isparseerror +using PromptingTools: AICode, isparsed, isparseerror, is_julia_code, is_julia_expr + +@testset "is_julia_expr" begin + # Valid Julia Expressions + @test is_julia_expr(:(x = 1)) == true + @test is_julia_expr(:(x === y)) == true + @test is_julia_expr(:(for i in 1:10 + println(i) + end)) == true + @test is_julia_expr(:(function foo() + return 42 + end)) == true + @test is_julia_expr(:(if x > 0 + println("positive") + end)) == true + + # Invalid Expressions + @test is_julia_expr(:(12345)) == false + + # Nested Expressions + @test is_julia_expr(:(begin + x = 1 + y = 2 + end)) == true + + # Non-Expr Types + @test is_julia_expr(42) == false + @test is_julia_expr("string") == false + @test is_julia_expr([1, 2, 3]) == false +end + +@testset "is_julia_code" begin + + # Valid Julia Code + @test is_julia_code("x = 1 + 2") == true + @test is_julia_code("println(\"Hello, world!\")") == true + @test is_julia_code("function foo()\nreturn 42\nend") == true + + # Invalid Julia Code + @test is_julia_code("x ==== y") == false + + # Empty String + @test is_julia_code("") == false + + # Non-Code Strings + @test is_julia_code("This is a plain text, not a code.") == false + + # Complex Julia Expressions + @test is_julia_code("for i in 1:10\nprintln(i)\nend") == true + @test is_julia_code("if x > 0\nprintln(\"positive\")\nelse\nprintln(\"non-positive\")\nend") == + true + + # Invalid Syntax + @test is_julia_code("function foo() return 42") == false # Missing 'end' keyword +end @testset "extract_imports tests" begin @test extract_julia_imports("using Test, LinearAlgebra") == @@ -12,6 +67,8 @@ using PromptingTools: AICode, isparsed, isparseerror Symbol.(["Test", "ABC", "DEF", "GEM"]) @test extract_julia_imports("import PackageA.PackageB: funcA\nimport PackageC") == Symbol.(["PackageA.PackageB", "PackageC"]) + @test extract_julia_imports("using Base.Threads\nusing Main.MyPkg") == + Symbol[] end @testset "detect_missing_packages" begin @@ -193,6 +250,32 @@ end SubString{String}["# Outer Julia code block\n\n# An example of a nested Julia code block in markdown\n\"\"\"\n```julia\nx = 5\nprintln(x)\n```\n\"\"\"\n\ny = 10\nprintln(y)"] end +@testset "extract_code_blocks_fallback" begin + + # Basic Functionality Test + @test extract_code_blocks_fallback("```\ncode block\n```") == ["code block"] + + # No Code Blocks Test + @test isempty(extract_code_blocks_fallback("Some text without code blocks")) + + # Adjacent Code Blocks Test + @test extract_code_blocks_fallback("```code1``` ```code2```") == ["code1", "", "code2"] + + # Special Characters Test + @test extract_code_blocks_fallback("```\n<>&\"'\n```") == ["<>&\"'"] + + # Large Input Test + large_input = "```" * repeat("large code block\n", 10) * "```" + @test extract_code_blocks_fallback(large_input) == + [strip(repeat("large code block\n", 10))] + + # Empty String Test + @test isempty(extract_code_blocks_fallback("")) + + # Different Delimiter Test + @test extract_code_blocks_fallback("~~~\ncode block\n~~~", "~~~") == ["code block"] +end + @testset "extract_function_name" begin # Test 1: Test an explicit function declaration @test extract_function_name("function testFunction1()\nend") == "testFunction1" @@ -352,6 +435,24 @@ Some text println(\"world\") b=2 ``` +""") + cb = AICode(msg) + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test cb.stdout == "hello\nworld\n" + @test cb.output.b == 2 + end + # Fallback extraction method + let msg = AIMessage(""" +``` +println(\"hello\") +``` +Some text +``` +println(\"world\") +b=2 +``` """) cb = AICode(msg) @test !isnothing(cb.expression) @@ -361,9 +462,7 @@ b=2 @test cb.output.b == 2 end # skip_unsafe=true - s = """ - """ let msg = AIMessage(""" ```julia a=1 From 45101906ccf30284f2f3c06e4b5b3b209dadc38b Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:01:18 +0000 Subject: [PATCH 056/251] add mistral support --- CHANGELOG.md | 2 + src/llm_interface.jl | 42 ++++++++++++++++ src/llm_openai.jl | 105 ++++++++++++++++++++++++++++++++++------ src/user_preferences.jl | 31 +++++++++++- test/llm_openai.jl | 36 +++++++++++++- test/runtests.jl | 2 +- 6 files changed, 199 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8532cf5f8..2a027fd32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Improved AICode parsing and error handling (eg, capture more REPL prompts, detect parsing errors earlier, parse more code fence types), including the option to remove unsafe code (eg, `Pkg.add("SomePkg")`) with `AICode(msg; skip_unsafe=true, vebose=true)` - Added new prompt templates: `JuliaRecapTask`, `JuliaRecapCoTTask`, `JuliaExpertTestCode` and updated `JuliaExpertCoTTask` to be more robust against early stopping for smaller OSS models +- Added support for MistralAI API via the MistralOpenAISchema(). All their standard models have been registered, so you should be able to just use `model="mistral-tiny` in your `aigenerate` calls without any further changes. Remember to either provide `api_kwargs.api_key` or ensure you have ENV variable `MISTRALAI_API_KEY` set. +- Added support for any OpenAI-compatible API via `schema=CustomOpenAISchema()`. All you have to do is to provide your `api_key` and `url` (base URL of the API) in the `api_kwargs` keyword argument. This option is useful if you use [Perplexity.ai](https://docs.perplexity.ai/), [Fireworks.ai](https://app.fireworks.ai/), or any other similar services. ### Fixed diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 990798000..3dc8d2f22 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -44,6 +44,48 @@ struct OpenAISchema <: AbstractOpenAISchema end inputs::Any = nothing end +""" + CustomOpenAISchema + +CustomOpenAISchema() allows user to call any OpenAI-compatible API. + +All user needs to do is to pass this schema as the first argument and provide the BASE URL of the API to call (`api_kwargs.url`). + +# Example + +Assumes that we have a local server running at `http://localhost:8081`: + +```julia +api_key = "..." +prompt = "Say hi!" +msg = aigenerate(CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) +``` + +""" +struct CustomOpenAISchema <: AbstractOpenAISchema end + +""" + MistralOpenAISchema + +MistralOpenAISchema() allows user to call MistralAI API known for mistral and mixtral models. + +It's a flavor of CustomOpenAISchema() with a url preset to `https://api.mistral.ai`. + +Most models have been registered, so you don't even have to specify the schema + +# Example + +Let's call `mistral-tiny` model: +```julia +api_key = "..." # can be set via ENV["MISTRAL_API_KEY"] or via our preference system +msg = aigenerate("Say hi!"; model="mistral_tiny", api_key) +``` + +See `?PREFERENCES` for more details on how to set your API key permanently. + +""" +struct MistralOpenAISchema <: AbstractOpenAISchema end + abstract type AbstractChatMLSchema <: AbstractPromptSchema end """ ChatMLSchema is used by many open-source chatbots, by OpenAI models (under the hood) and by several models and inferfaces (eg, Ollama, vLLM) diff --git a/src/llm_openai.jl b/src/llm_openai.jl index b89542d00..dd0a86660 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -56,6 +56,96 @@ function render(schema::AbstractOpenAISchema, return conversation end +## OpenAI.jl back-end +## Types +# "Providers" are a way to use other APIs that are compatible with OpenAI API specs, eg, Azure and mamy more +# Define our sub-type to distinguish it from other OpenAI.jl providers +abstract type AbstractCustomProvider <: OpenAI.AbstractOpenAIProvider end +Base.@kwdef struct CustomProvider <: AbstractCustomProvider + api_key::String = "" + base_url::String = "http://localhost:8080" + api_version::String = "" +end +function OpenAI.build_url(provider::AbstractCustomProvider, api::AbstractString) + string(provider.base_url, "/", api) +end +function OpenAI.auth_header(provider::AbstractCustomProvider, api_key::AbstractString) + OpenAI.auth_header(OpenAI.OpenAIProvider(provider.api_key, + provider.base_url, + provider.api_version), + api_key) +end +## Extend OpenAI create_chat to allow for testing/debugging +# Default passthrough +function OpenAI.create_chat(schema::AbstractOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + kwargs...) + OpenAI.create_chat(api_key, model, conversation; kwargs...) +end + +# Overload for testing/debugging +function OpenAI.create_chat(schema::TestEchoOpenAISchema, api_key::AbstractString, + model::AbstractString, + conversation; kwargs...) + schema.model_id = model + schema.inputs = conversation + return schema +end + +""" + OpenAI.create_chat(schema::CustomOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String="http://localhost:8080", + kwargs...) + +Dispatch to the OpenAI.create_chat function, for any OpenAI-compatible API. + +It expects `url` keyword argument. Provide it to the `aigenerate` function via `api_kwargs=(; url="my-url")` + +It will forward your query to the "chat/completions" endpoint of the base URL that you provided (=`url`). +""" +function OpenAI.create_chat(schema::CustomOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String = "http://localhost:8080", + kwargs...) + # Build the corresponding provider object + # Create chat will automatically pass our data to endpoint `/chat/completions` + provider = CustomProvider(; api_key, base_url = url) + OpenAI.create_chat(provider, model, conversation; kwargs...) +end + +""" + OpenAI.create_chat(schema::MistralOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String="https://api.mistral.ai/v1", + kwargs...) + +Dispatch to the OpenAI.create_chat function, but with the MistralAI API parameters. + +It tries to access the `MISTRALAI_API_KEY` ENV variable, but you can also provide it via the `api_key` keyword argument. +""" +function OpenAI.create_chat(schema::MistralOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String = "https://api.mistral.ai/v1", + kwargs...) + # Build the corresponding provider object + # try to override provided api_key because the default is OpenAI key + provider = CustomProvider(; + api_key = isempty(MISTRALAI_API_KEY) ? api_key : MISTRALAI_API_KEY, + base_url = url) + OpenAI.create_chat(provider, model, conversation; kwargs...) +end + ## User-Facing API """ aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; @@ -170,21 +260,6 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ return output end -# Extend OpenAI create_chat to allow for testing/debugging -function OpenAI.create_chat(schema::AbstractOpenAISchema, - api_key::AbstractString, - model::AbstractString, - conversation; - kwargs...) - OpenAI.create_chat(api_key, model, conversation; kwargs...) -end -function OpenAI.create_chat(schema::TestEchoOpenAISchema, api_key::AbstractString, - model::AbstractString, - conversation; kwargs...) - schema.model_id = model - schema.inputs = conversation - return schema -end """ aiembed(prompt_schema::AbstractOpenAISchema, diff --git a/src/user_preferences.jl b/src/user_preferences.jl index ecf136221..2c33bf563 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -12,6 +12,7 @@ Check your preferences by calling `get_preferences(key::String)`. # Available Preferences (for `set_preferences!`) - `OPENAI_API_KEY`: The API key for the OpenAI API. See [OpenAI's documentation](https://platform.openai.com/docs/quickstart?context=python) for more information. +- `MISTRALAI_API_KEY`: The API key for the Mistral AI API. See [Mistral AI's documentation](https://docs.mistral.ai/) for more information. - `MODEL_CHAT`: The default model to use for aigenerate and most ai* calls. See `MODEL_REGISTRY` for a list of available models or define your own. - `MODEL_EMBEDDING`: The default model to use for aiembed (embedding documents). See `MODEL_REGISTRY` for a list of available models or define your own. - `PROMPT_SCHEMA`: The default prompt schema to use for aigenerate and most ai* calls (if not specified in `MODEL_REGISTRY`). Set as a string, eg, `"OpenAISchema"`. @@ -24,6 +25,7 @@ Define your `register_model!()` calls in your `startup.jl` file to make them ava # Available ENV Variables - `OPENAI_API_KEY`: The API key for the OpenAI API. +- `MISTRALAI_API_KEY`: The API key for the Mistral AI API. Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. @@ -47,6 +49,7 @@ PromptingTools.set_preferences!("OPENAI_API_KEY" => "key1", "MODEL_CHAT" => "cha """ function set_preferences!(pairs::Pair{String, <:Any}...) allowed_preferences = [ + "MISTRALAI_API_KEY", "OPENAI_API_KEY", "MODEL_CHAT", "MODEL_EMBEDDING", @@ -79,6 +82,7 @@ PromptingTools.get_preferences("MODEL_CHAT") """ function get_preferences(key::String) allowed_preferences = [ + "MISTRALAI_API_KEY", "OPENAI_API_KEY", "MODEL_CHAT", "MODEL_EMBEDDING", @@ -98,11 +102,14 @@ const MODEL_EMBEDDING::String = @load_preference("MODEL_EMBEDDING", # First, load from preferences, then from environment variables const OPENAI_API_KEY::String = @load_preference("OPENAI_API_KEY", - default=get(ENV, "OPENAI_API_KEY", "")) + default=get(ENV, "OPENAI_API_KEY", "")); # Note: Disable this warning by setting OPENAI_API_KEY to anything isempty(OPENAI_API_KEY) && @warn "OPENAI_API_KEY variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.OPENAI_API_KEY=`!" +const MISTRALAI_API_KEY::String = @load_preference("MISTRALAI_API_KEY", + default=get(ENV, "MISTRALAI_API_KEY", "")); + ## Model registry # A dictionary of model names and their specs (ie, name, costs per token, etc.) # Model specs are saved in ModelSpec struct (see below) @@ -261,7 +268,27 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", OllamaManagedSchema(), 0.0, 0.0, - "Yi is a 34B parameter model finetuned by X on top of base model from Starling AI.")) + "Yi is a 34B parameter model finetuned by X on top of base model from Starling AI."), + "mistral-tiny" => ModelSpec("mistral-tiny", + MistralOpenAISchema(), + 1.4e-7, + 4.53e-7, + "Mistral AI's hosted version of Mistral-7B-v0.2. Great for simple tasks."), + "mistral-small" => ModelSpec("mistral-small", + MistralOpenAISchema(), + 6.47e-7, + 1.94e-6, + "Mistral AI's hosted version of Mixtral-8x7B-v0.1. Good for more complicated tasks."), + "mistral-medium" => ModelSpec("mistral-medium", + MistralOpenAISchema(), + 2.7e-6, + 8.09e-6, + "Mistral AI's hosted version of their best model available. Details unknown."), + "mistral-embed" => ModelSpec("mistral-embed", + MistralOpenAISchema(), + 1.08e-7, + 0.0, + "Mistral AI's hosted model for embeddings.")) ### Model Registry Structure @kwdef mutable struct ModelRegistry diff --git a/test/llm_openai.jl b/test/llm_openai.jl index 7ac921ad7..d8e54966e 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -1,6 +1,7 @@ using PromptingTools: TestEchoOpenAISchema, render, OpenAISchema using PromptingTools: AIMessage, SystemMessage, AbstractMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage +using PromptingTools: CustomProvider, CustomOpenAISchema, MistralOpenAISchema @testset "render-OpenAI" begin schema = OpenAISchema() @@ -169,6 +170,39 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage nothing end +@testset "OpenAI.build_url,OpenAI.auth_header" begin + provider = CustomProvider(; base_url = "http://localhost:8082", api_version = "xyz") + @test OpenAI.build_url(provider, "endpoint1") == "http://localhost:8082/endpoint1" + @test OpenAI.auth_header(provider, "ABC") == + ["Authorization" => "Bearer ABC", "Content-Type" => "application/json"] +end + +@testset "OpenAI.create_chat" begin + # Test CustomOpenAISchema() with a mock server + echo_server = HTTP.serve!(8081) do req + content = JSON3.read(req.body) + user_msg = last(content[:messages]) + response = Dict(:choices => [Dict(:message => user_msg)], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + return HTTP.Response(200, JSON3.write(response)) + end + + prompt = "Say Hi!" + msg = aigenerate(CustomOpenAISchema(), + prompt; + model = "my_model", + api_kwargs = (; url = "http://localhost:8081"), + return_all = false) + @test msg.content == prompt + @test msg.tokens == (length(prompt), 0) + + # clean up + close(echo_server) +end + @testset "aigenerate-OpenAI" begin # corresponds to OpenAI API v1 response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], @@ -176,7 +210,7 @@ end # Test the monkey patch schema = TestEchoOpenAISchema(; response, status = 200) - msg = PT.OpenAI.create_chat(schema, "", "", "Hello") + msg = OpenAI.create_chat(schema, "", "", "Hello") @test msg isa TestEchoOpenAISchema # Real generation API diff --git a/test/runtests.jl b/test/runtests.jl index cc7a6f672..34b73012a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,5 @@ using PromptingTools -using JSON3 +using OpenAI, HTTP, JSON3 using Test using Aqua const PT = PromptingTools From ef424f4f58ad8293d2c1c33d1f1c8eabb2295c15 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:46:00 +0000 Subject: [PATCH 057/251] add aiembed support + bump version --- CHANGELOG.md | 8 ++- Project.toml | 2 +- README.md | 34 +++++++++++++ docs/src/examples/readme_examples.md | 35 ++++++++++++- docs/src/frequently_asked_questions.md | 2 + src/llm_openai.jl | 70 ++++++++++++++++++++------ test/llm_openai.jl | 29 ++++++++++- 7 files changed, 159 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a027fd32..c86656267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.4.0] + ### Added - Improved AICode parsing and error handling (eg, capture more REPL prompts, detect parsing errors earlier, parse more code fence types), including the option to remove unsafe code (eg, `Pkg.add("SomePkg")`) with `AICode(msg; skip_unsafe=true, vebose=true)` - Added new prompt templates: `JuliaRecapTask`, `JuliaRecapCoTTask`, `JuliaExpertTestCode` and updated `JuliaExpertCoTTask` to be more robust against early stopping for smaller OSS models - Added support for MistralAI API via the MistralOpenAISchema(). All their standard models have been registered, so you should be able to just use `model="mistral-tiny` in your `aigenerate` calls without any further changes. Remember to either provide `api_kwargs.api_key` or ensure you have ENV variable `MISTRALAI_API_KEY` set. - Added support for any OpenAI-compatible API via `schema=CustomOpenAISchema()`. All you have to do is to provide your `api_key` and `url` (base URL of the API) in the `api_kwargs` keyword argument. This option is useful if you use [Perplexity.ai](https://docs.perplexity.ai/), [Fireworks.ai](https://app.fireworks.ai/), or any other similar services. -### Fixed - ## [0.3.0] ### Added diff --git a/Project.toml b/Project.toml index 5957c08c7..355366810 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.4.0-DEV" +version = "0.4.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/README.md b/README.md index 0dbf0978f..e81450288 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ For more practical examples, see the `examples/` folder and the [Advanced Exampl - [Data Extraction](#data-extraction) - [OCR and Image Comprehension](#ocr-and-image-comprehension) - [Using Ollama models](#using-ollama-models) + - [Using MistralAI API and other OpenAI-compatible APIs](#using-mistralai-api-and-other-openai-compatible-apis) - [More Examples](#more-examples) - [Package Interface](#package-interface) - [Frequently Asked Questions](#frequently-asked-questions) @@ -395,6 +396,37 @@ msg.content # 4096×2 Matrix{Float64}: If you're getting errors, check that Ollama is running - see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. +### Using MistralAI API and other OpenAI-compatible APIs + +Mistral models have long been dominating the open-source space. They are now available via their API, so you can use them with PromptingTools.jl! + +```julia +msg = aigenerate("Say hi!"; model="mistral-tiny") +``` + +It all just works, because we have registered the models in the `PromptingTools.MODEL_REGISTRY`! There are currently 4 models available: `mistral-tiny`, `mistral-small`, `mistral-medium`, `mistral-embed`. + +Under the hood, we use a dedicated schema `MistralOpenAISchema` that leverages most of the OpenAI-specific code base, so you can always provide that explicitly as the first argument: + +```julia +msg = aigenerate(MistralOpenAISchema(), "Say Hi!"; model="mistral-tiny", api_key=ENV["MISTRALAI_API_KEY"]) +``` +As you can see, we can load your API key either from the ENV or via the Preferences.jl mechanism (see `?PREFERENCES` for more information). + +But MistralAI are not the only ones! There are many other exciting providers, eg, [Perplexity.ai](https://docs.perplexity.ai/), [Fireworks.ai](https://app.fireworks.ai/). +As long as they are compatible with the OpenAI API (eg, sending `messages` with `role` and `content` keys), you can use them with PromptingTools.jl by using `schema = CustomOpenAISchema()`: + +```julia +# Set your API key and the necessary base URL for the API +api_key = "..." +prompt = "Say hi!" +msg = aigenerate(CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) +``` + +As you can see, it also works for any local models that you might have running on your computer! + +Note: At the moment, we only support `aigenerate` and `aiembed` functions for MistralAI and other OpenAI-compatible APIs. We plan to extend the support in the future. + ### More Examples TBU... @@ -529,6 +561,8 @@ Resources: ### Configuring the Environment Variable for API Key +This is a guide for OpenAI's API key, but it works for any other API key you might need (eg, `MISTRALAI_API_KEY` for MistralAI API). + To use the OpenAI API with PromptingTools.jl, set your API key as an environment variable: ```julia diff --git a/docs/src/examples/readme_examples.md b/docs/src/examples/readme_examples.md index 5e2f35ca3..18afc5167 100644 --- a/docs/src/examples/readme_examples.md +++ b/docs/src/examples/readme_examples.md @@ -286,4 +286,37 @@ msg = aiembed(schema, ["Embed me", "Embed me"]; model="openhermes2.5-mistral") msg.content # 4096×2 Matrix{Float64}: ``` -If you're getting errors, check that Ollama is running - see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. \ No newline at end of file +If you're getting errors, check that Ollama is running - see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. + +## Using MistralAI API and other OpenAI-compatible APIs + +Mistral models have long been dominating the open-source space. They are now available via their API, so you can use them with PromptingTools.jl! + +```julia +msg = aigenerate("Say hi!"; model="mistral-tiny") +# [ Info: Tokens: 114 @ Cost: $0.0 in 0.9 seconds +# AIMessage("Hello there! I'm here to help answer any questions you might have, or assist you with tasks to the best of my abilities. How can I be of service to you today? If you have a specific question, feel free to ask and I'll do my best to provide accurate and helpful information. If you're looking for general assistance, I can help you find resources or information on a variety of topics. Let me know how I can help.") +``` + +It all just works, because we have registered the models in the `PromptingTools.MODEL_REGISTRY`! There are currently 4 models available: `mistral-tiny`, `mistral-small`, `mistral-medium`, `mistral-embed`. + +Under the hood, we use a dedicated schema `MistralOpenAISchema` that leverages most of the OpenAI-specific code base, so you can always provide that explicitly as the first argument: + +```julia +msg = aigenerate(MistralOpenAISchema(), "Say Hi!"; model="mistral-tiny", api_key=ENV["MISTRALAI_API_KEY"]) +``` +As you can see, we can load your API key either from the ENV or via the Preferences.jl mechanism (see `?PREFERENCES` for more information). + +But MistralAI are not the only ones! There are many other exciting providers, eg, [Perplexity.ai](https://docs.perplexity.ai/), [Fireworks.ai](https://app.fireworks.ai/). +As long as they are compatible with the OpenAI API (eg, sending `messages` with `role` and `content` keys), you can use them with PromptingTools.jl by using `schema = CustomOpenAISchema()`: + +```julia +# Set your API key and the necessary base URL for the API +api_key = "..." +prompt = "Say hi!" +msg = aigenerate(CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) +``` + +As you can see, it also works for any local models that you might have running on your computer! + +Note: At the moment, we only support `aigenerate` and `aiembed` functions for MistralAI and other OpenAI-compatible APIs. We plan to extend the support in the future. \ No newline at end of file diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md index cb51bdeb8..d12612c00 100644 --- a/docs/src/frequently_asked_questions.md +++ b/docs/src/frequently_asked_questions.md @@ -70,6 +70,8 @@ Resources: ## Configuring the Environment Variable for API Key +This is a guide for OpenAI's API key, but it works for any other API key you might need (eg, `MISTRALAI_API_KEY` for MistralAI API). + To use the OpenAI API with PromptingTools.jl, set your API key as an environment variable: ```julia diff --git a/src/llm_openai.jl b/src/llm_openai.jl index dd0a86660..f9aaa9538 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -146,6 +146,61 @@ function OpenAI.create_chat(schema::MistralOpenAISchema, OpenAI.create_chat(provider, model, conversation; kwargs...) end +# Extend OpenAI create_embeddings to allow for testing +function OpenAI.create_embeddings(schema::AbstractOpenAISchema, + api_key::AbstractString, + docs, + model::AbstractString; + kwargs...) + OpenAI.create_embeddings(api_key, docs, model; kwargs...) +end +function OpenAI.create_embeddings(schema::TestEchoOpenAISchema, api_key::AbstractString, + docs, + model::AbstractString; kwargs...) + schema.model_id = model + schema.inputs = docs + return schema +end +function OpenAI.create_embeddings(schema::CustomOpenAISchema, + api_key::AbstractString, + docs, + model::AbstractString; + url::String = "http://localhost:8080", + kwargs...) + # Build the corresponding provider object + # Create chat will automatically pass our data to endpoint `/embeddings` + provider = CustomProvider(; api_key, base_url = url) + OpenAI.create_embeddings(provider, docs, model; kwargs...) +end +function OpenAI.create_embeddings(schema::MistralOpenAISchema, + api_key::AbstractString, + docs, + model::AbstractString; + url::String = "https://api.mistral.ai/v1", + kwargs...) + # Build the corresponding provider object + # try to override provided api_key because the default is OpenAI key + provider = CustomProvider(; + api_key = isempty(MISTRALAI_API_KEY) ? api_key : MISTRALAI_API_KEY, + base_url = url) + OpenAI.create_embeddings(provider, docs, model; kwargs...) +end + +## Temporary fix -- it will be moved upstream +function OpenAI.create_embeddings(provider::AbstractCustomProvider, + input, + model_id::String = OpenAI.DEFAULT_EMBEDDING_MODEL_ID; + http_kwargs::NamedTuple = NamedTuple(), + kwargs...) + return OpenAI.openai_request("embeddings", + provider; + method = "POST", + http_kwargs = http_kwargs, + model = model_id, + input, + kwargs...) +end + ## User-Facing API """ aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; @@ -343,21 +398,6 @@ function aiembed(prompt_schema::AbstractOpenAISchema, return msg end -# Extend OpenAI create_embeddings to allow for testing -function OpenAI.create_embeddings(schema::AbstractOpenAISchema, - api_key::AbstractString, - docs, - model::AbstractString; - kwargs...) - OpenAI.create_embeddings(api_key, docs, model; kwargs...) -end -function OpenAI.create_embeddings(schema::TestEchoOpenAISchema, api_key::AbstractString, - docs, - model::AbstractString; kwargs...) - schema.model_id = model - schema.inputs = docs - return schema -end """ aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; diff --git a/test/llm_openai.jl b/test/llm_openai.jl index d8e54966e..95364b847 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -179,7 +179,8 @@ end @testset "OpenAI.create_chat" begin # Test CustomOpenAISchema() with a mock server - echo_server = HTTP.serve!(8081) do req + PORT = rand(1000:2000) + echo_server = HTTP.serve!(PORT) do req content = JSON3.read(req.body) user_msg = last(content[:messages]) response = Dict(:choices => [Dict(:message => user_msg)], @@ -194,7 +195,7 @@ end msg = aigenerate(CustomOpenAISchema(), prompt; model = "my_model", - api_kwargs = (; url = "http://localhost:8081"), + api_kwargs = (; url = "http://localhost:$(PORT)"), return_all = false) @test msg.content == prompt @test msg.tokens == (length(prompt), 0) @@ -202,6 +203,30 @@ end # clean up close(echo_server) end +@testset "OpenAI.create_embeddings" begin + # Test CustomOpenAISchema() with a mock server + PORT = rand(1000:2000) + echo_server = HTTP.serve!(PORT) do req + content = JSON3.read(req.body) + response = Dict(:data => [Dict(:embedding => ones(128))], + :usage => Dict(:total_tokens => length(content[:input]), + :prompt_tokens => length(content[:input]), + :completion_tokens => 0)) + return HTTP.Response(200, JSON3.write(response)) + end + + prompt = "Embed me!!" + msg = aiembed(CustomOpenAISchema(), + prompt; + model = "my_model", + api_kwargs = (; url = "http://localhost:$(PORT)"), + return_all = false) + @test msg.content == ones(128) + @test msg.tokens == (length(prompt), 0) + + # clean up + close(echo_server) +end @testset "aigenerate-OpenAI" begin # corresponds to OpenAI API v1 From 1e18149708ff23ce82926bbf66cfdecb321fd1f6 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 13 Dec 2023 21:46:59 +0000 Subject: [PATCH 058/251] update docs --- README.md | 5 +++-- docs/src/examples/readme_examples.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e81450288..532e260ce 100644 --- a/README.md +++ b/README.md @@ -409,7 +409,8 @@ It all just works, because we have registered the models in the `PromptingTools. Under the hood, we use a dedicated schema `MistralOpenAISchema` that leverages most of the OpenAI-specific code base, so you can always provide that explicitly as the first argument: ```julia -msg = aigenerate(MistralOpenAISchema(), "Say Hi!"; model="mistral-tiny", api_key=ENV["MISTRALAI_API_KEY"]) +const PT = PromptingTools +msg = aigenerate(PT.MistralOpenAISchema(), "Say Hi!"; model="mistral-tiny", api_key=ENV["MISTRALAI_API_KEY"]) ``` As you can see, we can load your API key either from the ENV or via the Preferences.jl mechanism (see `?PREFERENCES` for more information). @@ -420,7 +421,7 @@ As long as they are compatible with the OpenAI API (eg, sending `messages` with # Set your API key and the necessary base URL for the API api_key = "..." prompt = "Say hi!" -msg = aigenerate(CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) +msg = aigenerate(PT.CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) ``` As you can see, it also works for any local models that you might have running on your computer! diff --git a/docs/src/examples/readme_examples.md b/docs/src/examples/readme_examples.md index 18afc5167..5ec371b73 100644 --- a/docs/src/examples/readme_examples.md +++ b/docs/src/examples/readme_examples.md @@ -303,7 +303,8 @@ It all just works, because we have registered the models in the `PromptingTools. Under the hood, we use a dedicated schema `MistralOpenAISchema` that leverages most of the OpenAI-specific code base, so you can always provide that explicitly as the first argument: ```julia -msg = aigenerate(MistralOpenAISchema(), "Say Hi!"; model="mistral-tiny", api_key=ENV["MISTRALAI_API_KEY"]) +const PT = PromptingTools +msg = aigenerate(PT.MistralOpenAISchema(), "Say Hi!"; model="mistral-tiny", api_key=ENV["MISTRALAI_API_KEY"]) ``` As you can see, we can load your API key either from the ENV or via the Preferences.jl mechanism (see `?PREFERENCES` for more information). @@ -314,7 +315,7 @@ As long as they are compatible with the OpenAI API (eg, sending `messages` with # Set your API key and the necessary base URL for the API api_key = "..." prompt = "Say hi!" -msg = aigenerate(CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) +msg = aigenerate(PT.CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) ``` As you can see, it also works for any local models that you might have running on your computer! From c1292766e36965113bd1c65e712fe0814a053164 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 16 Dec 2023 16:36:29 +0000 Subject: [PATCH 059/251] fix AICode parser --- CHANGELOG.md | 2 ++ Project.toml | 2 +- src/code_generation.jl | 13 ++++++++++--- test/code_generation.jl | 32 +++++++++++++++++++++++++++++++- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c86656267..9ff0c79f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Fixed +- Stricter code parsing in `AICode` to avoid false positives (code blocks must end with "```\n" to catch comments inside text) +- Introduced an option `skip_invalid=true` for `AICode`, which allows you to include only code blocks that parse successfully (useful when the code definition is good, but the subsequent examples are not) ## [0.4.0] diff --git a/Project.toml b/Project.toml index 355366810..17136594e 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.4.0" +version = "0.5.0-DEV" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/code_generation.jl b/src/code_generation.jl index 84e13d778..a62e1402f 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -46,6 +46,8 @@ See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` # Keyword Arguments - `auto_eval::Bool`: If set to `true`, the code block is automatically parsed and evaluated upon instantiation. Defaults to `true`. - `safe_eval::Bool`: If set to `true`, the code block checks for package operations (e.g., installing new packages) and missing imports, and then evaluates the code inside a bespoke scratch module. This is to ensure that the evaluation does not alter any user-defined variables or the global state. Defaults to `false`. +- `skip_unsafe::Bool`: If set to `true`, we skip any lines in the code block that are deemed unsafe (eg, `Pkg` operations). Defaults to `false`. +- `skip_invalid::Bool`: If set to `true`, we skip code blocks that do not even parse. Defaults to `false`. - `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. Useful to add some additional code definition or necessary imports. Defaults to an empty string. - `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. @@ -214,12 +216,17 @@ end function AICode(msg::AIMessage; verbose::Bool = false, skip_unsafe::Bool = false, + skip_invalid::Bool = false, kwargs...) code = extract_code_blocks(msg.content) if isempty(code) ## Fallback option for generic code fence, we must check if the content is parseable - code = extract_code_blocks_fallback(msg.content) |> - x -> filter(is_julia_code, x) + code = extract_code_blocks_fallback(msg.content) + skip_invalid = true # set to true if we use fallback option + end + if skip_invalid + ## Filter out extracted code blocks that do not even parse + filter!(is_julia_code, code) end code = join(code, "\n") skip_unsafe && (code = remove_unsafe_lines(code; verbose)) @@ -412,7 +419,7 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString} # Convert content and delimiters to codeunits content_units = codeunits(markdown_content) start_delim_units = codeunits("```julia") - end_delim_units = codeunits("```") + end_delim_units = codeunits("```\n") # Find all starting and ending positions of code blocks start_positions = find_subsequence_positions(start_delim_units, content_units) diff --git a/test/code_generation.jl b/test/code_generation.jl index d3adbcd5a..28aedf7d6 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -248,6 +248,17 @@ end """ @test extract_code_blocks(markdown_example) == SubString{String}["# Outer Julia code block\n\n# An example of a nested Julia code block in markdown\n\"\"\"\n```julia\nx = 5\nprintln(x)\n```\n\"\"\"\n\ny = 10\nprintln(y)"] + + # Tough case of regex inside a function + markdown_example = """ +```julia +function find_match(md::AbstractString) + return match(r"```\\n(?:(?!\\n```)\\s*.*\\n?)*\\s*```", md) +end +``` +""" + @test extract_code_blocks(markdown_example) == + SubString{String}["function find_match(md::AbstractString)\n return match(r\"```\\n(?:(?!\\n```)\\s*.*\\n?)*\\s*```\", md)\nend"] end @testset "extract_code_blocks_fallback" begin @@ -443,6 +454,7 @@ b=2 @test cb.stdout == "hello\nworld\n" @test cb.output.b == 2 end + # Fallback extraction method let msg = AIMessage(""" ``` @@ -461,8 +473,8 @@ b=2 @test cb.stdout == "hello\nworld\n" @test cb.output.b == 2 end - # skip_unsafe=true + # skip_unsafe=true let msg = AIMessage(""" ```julia a=1 @@ -476,6 +488,24 @@ b=2 @test cb.code == "a=1\nb=2\n" end + # skip_invalid=true + let msg = AIMessage(""" + ```julia + println("Hello world!") + ``` + + ```julia + println("Hello world!) # missing quote + ``` + """) + cb = AICode(msg; skip_invalid = true) + @test cb.code == "println(\"Hello world!\")" + + # if it's not switched on + cb = AICode(msg; skip_invalid = false) + @test !isvalid(cb) + end + # Methods - copy let msg = AIMessage(""" ```julia From 04b6636e2e9d60da5ac2594b8f03f03b4d97c70f Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 16 Dec 2023 16:52:34 +0000 Subject: [PATCH 060/251] make-capture-stdout-optional --- CHANGELOG.md | 2 +- src/code_generation.jl | 46 ++++++++++++++++++++++++++++++++++------- test/code_generation.jl | 12 +++++++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ff0c79f1..b9bf46e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Stricter code parsing in `AICode` to avoid false positives (code blocks must end with "```\n" to catch comments inside text) -- Introduced an option `skip_invalid=true` for `AICode`, which allows you to include only code blocks that parse successfully (useful when the code definition is good, but the subsequent examples are not) +- Introduced an option `skip_invalid=true` for `AICode`, which allows you to include only code blocks that parse successfully (useful when the code definition is good, but the subsequent examples are not), and an option `capture_stdout=false` to avoid capturing stdout if you want to evaluate `AICode` in parallel (`Pipe()` that we use is NOT thread-safe) ## [0.4.0] diff --git a/src/code_generation.jl b/src/code_generation.jl index a62e1402f..7bf801068 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -16,7 +16,9 @@ abstract type AbstractCodeBlock end """ - AICode(code::AbstractString; auto_eval::Bool=true, safe_eval::Bool=false, prefix::AbstractString="", suffix::AbstractString="") + AICode(code::AbstractString; auto_eval::Bool=true, safe_eval::Bool=false, + skip_unsafe::Bool=false, skip_unsafe::Bool=false, capture_stdout::Bool=true, + prefix::AbstractString="", suffix::AbstractString="") A mutable structure representing a code block (received from the AI model) with automatic parsing, execution, and output/error capturing capabilities. @@ -48,6 +50,7 @@ See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` - `safe_eval::Bool`: If set to `true`, the code block checks for package operations (e.g., installing new packages) and missing imports, and then evaluates the code inside a bespoke scratch module. This is to ensure that the evaluation does not alter any user-defined variables or the global state. Defaults to `false`. - `skip_unsafe::Bool`: If set to `true`, we skip any lines in the code block that are deemed unsafe (eg, `Pkg` operations). Defaults to `false`. - `skip_invalid::Bool`: If set to `true`, we skip code blocks that do not even parse. Defaults to `false`. +- `capture_stdout::Bool`: If set to `true`, we capture any stdout outputs (eg, test failures) in `cb.stdout`. Defaults to `true`. - `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. Useful to add some additional code definition or necessary imports. Defaults to an empty string. - `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. @@ -560,7 +563,11 @@ function extract_function_name(code_block::AbstractString) end """ - eval!(cb::AICode; safe_eval::Bool=true, prefix::AbstractString="", suffix::AbstractString="") + eval!(cb::AbstractCodeBlock; + safe_eval::Bool = true, + capture_stdout::Bool = true, + prefix::AbstractString = "", + suffix::AbstractString = "") Evaluates a code block `cb` in-place. It runs automatically when AICode is instantiated with a String. @@ -572,13 +579,14 @@ Steps: - Parse the text in `cb.code` - Evaluate the parsed expression - Capture outputs of the evaluated in `cb.output` -- Capture any stdout outputs (eg, test failures) in `cb.stdout` +- [OPTIONAL] Capture any stdout outputs (eg, test failures) in `cb.stdout` - If any error exception is raised, it is saved in `cb.error` - Finally, if all steps were successful, success is set to `cb.success = true` # Keyword Arguments - `safe_eval::Bool`: If `true`, we first check for any Pkg operations (eg, installing new packages) and missing imports, then the code will be evaluated inside a bespoke scratch module (not to change any user variables) +- `capture_stdout::Bool`: If `true`, we capture any stdout outputs (eg, test failures) in `cb.stdout` - `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. Useful to add some additional code definition or necessary imports. Defaults to an empty string. - `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. @@ -586,6 +594,7 @@ Steps: """ function eval!(cb::AbstractCodeBlock; safe_eval::Bool = true, + capture_stdout::Bool = true, prefix::AbstractString = "", suffix::AbstractString = "") (; code) = cb @@ -626,8 +635,33 @@ function eval!(cb::AbstractCodeBlock; ## Eval safe_module = gensym("SafeCustomModule") # Prepare to catch any stdout - pipe = Pipe() - redirect_stdout(pipe) do + if capture_stdout + pipe = Pipe() + redirect_stdout(pipe) do + try + # eval in Main module to have access to std libs, but inside a custom module for safety + if safe_eval + cb.output = @eval(Main, module $safe_module + using Test # just in case unit tests are provided + $(cb.expression) + end) + else + # Evaluate the code directly into Main + cb.output = @eval(Main, begin + using Test # just in case unit tests are provided + $(cb.expression) + end) + end + cb.success = true + catch e + cb.error = e + cb.success = false + end + end + close(Base.pipe_writer(pipe)) + cb.stdout = read(pipe, String) + else + # Ignore stdout, just eval try # eval in Main module to have access to std libs, but inside a custom module for safety if safe_eval @@ -648,7 +682,5 @@ function eval!(cb::AbstractCodeBlock; cb.success = false end end - close(Base.pipe_writer(pipe)) - cb.stdout = read(pipe, String) return cb end \ No newline at end of file diff --git a/test/code_generation.jl b/test/code_generation.jl index 28aedf7d6..b2fe65613 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -409,6 +409,18 @@ end eval!(cb; prefix = "a=1", suffix = "b=2") @test cb.output.a == 1 @test cb.output.b == 2 + + # Whether to capture stdout + cb = AICode(; code = "println(\"Hello\")") + eval!(cb; capture_stdout = false) + @test cb.stdout == nothing + @test cb.code == "println(\"Hello\")" + @test isvalid(cb) + + eval!(cb; capture_stdout = true) + @test cb.stdout == "Hello\n" + @test cb.code == "println(\"Hello\")" + @test isvalid(cb) end @testset "AICode constructors" begin From 7e4d0da3d66d00a9e46778fa214b5a2a10fd9202 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 16 Dec 2023 16:54:14 +0000 Subject: [PATCH 061/251] add kwarg --- src/code_generation.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index 7bf801068..4db36c4e5 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -108,10 +108,11 @@ end function (CB::Type{T})(md::AbstractString; auto_eval::Bool = true, safe_eval::Bool = true, + capture_stdout::Bool = true, prefix::AbstractString = "", suffix::AbstractString = "") where {T <: AbstractCodeBlock} cb = CB(; code = md) - auto_eval && eval!(cb; safe_eval, prefix, suffix) + auto_eval && eval!(cb; safe_eval, capture_stdout, prefix, suffix) return cb end Base.isvalid(cb::AbstractCodeBlock) = cb.success == true From d17ff8de63d1e3e2ba7a445a6aadf6784281c907 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 16 Dec 2023 17:15:32 +0000 Subject: [PATCH 062/251] update tests --- src/code_generation.jl | 10 +++++++++- test/code_generation.jl | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index 4db36c4e5..bb0e8c975 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -17,9 +17,13 @@ abstract type AbstractCodeBlock end """ AICode(code::AbstractString; auto_eval::Bool=true, safe_eval::Bool=false, - skip_unsafe::Bool=false, skip_unsafe::Bool=false, capture_stdout::Bool=true, + skip_unsafe::Bool=false, capture_stdout::Bool=true, verbose::Bool=false, prefix::AbstractString="", suffix::AbstractString="") + AICode(msg::AIMessage; auto_eval::Bool=true, safe_eval::Bool=false, + skip_unsafe::Bool=false, skip_invalid::Bool=false, capture_stdout::Bool=true, + verbose::Bool=false, prefix::AbstractString="", suffix::AbstractString="") + A mutable structure representing a code block (received from the AI model) with automatic parsing, execution, and output/error capturing capabilities. Upon instantiation with a string, the `AICode` object automatically runs a code parser and executor (via `PromptingTools.eval!()`), capturing any standard output (`stdout`) or errors. @@ -50,6 +54,7 @@ See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` - `safe_eval::Bool`: If set to `true`, the code block checks for package operations (e.g., installing new packages) and missing imports, and then evaluates the code inside a bespoke scratch module. This is to ensure that the evaluation does not alter any user-defined variables or the global state. Defaults to `false`. - `skip_unsafe::Bool`: If set to `true`, we skip any lines in the code block that are deemed unsafe (eg, `Pkg` operations). Defaults to `false`. - `skip_invalid::Bool`: If set to `true`, we skip code blocks that do not even parse. Defaults to `false`. +- `verbose::Bool`: If set to `true`, we print out any lines that are skipped due to being unsafe. Defaults to `false`. - `capture_stdout::Bool`: If set to `true`, we capture any stdout outputs (eg, test failures) in `cb.stdout`. Defaults to `true`. - `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. Useful to add some additional code definition or necessary imports. Defaults to an empty string. @@ -108,9 +113,12 @@ end function (CB::Type{T})(md::AbstractString; auto_eval::Bool = true, safe_eval::Bool = true, + skip_unsafe::Bool = false, capture_stdout::Bool = true, + verbose::Bool = false, prefix::AbstractString = "", suffix::AbstractString = "") where {T <: AbstractCodeBlock} + skip_unsafe && (md = remove_unsafe_lines(md; verbose)) cb = CB(; code = md) auto_eval && eval!(cb; safe_eval, capture_stdout, prefix, suffix) return cb diff --git a/test/code_generation.jl b/test/code_generation.jl index b2fe65613..6fcc53851 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -498,6 +498,11 @@ b=2 """) cb = AICode(msg; skip_unsafe = true) @test cb.code == "a=1\nb=2\n" + + # dispatch on text + code = extract_code_blocks(msg.content) |> x -> join(x, "\n") + cb = AICode(code; skip_unsafe = true) + @test cb.code == "a=1\nb=2\n" end # skip_invalid=true From 1d944f679874aef73ecb37498ee0f440e16f9c12 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 16 Dec 2023 22:11:23 +0000 Subject: [PATCH 063/251] update fallback parser to expect newlines --- src/code_generation.jl | 53 +++++++++++++++++++++++++++++++---------- test/code_generation.jl | 11 +++++++-- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index bb0e8c975..7a7a8d963 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -430,13 +430,22 @@ extract_code_blocks(markdown_multiple) function extract_code_blocks(markdown_content::T) where {T <: AbstractString} # Convert content and delimiters to codeunits content_units = codeunits(markdown_content) - start_delim_units = codeunits("```julia") - end_delim_units = codeunits("```\n") + # Ideal code fences + start_delim_units1 = codeunits("\n```julia\n") + end_delim_units1 = codeunits("\n```\n") + # Fallback code fences + start_delim_units2 = codeunits("```julia\n") + end_delim_units2 = codeunits("\n```") # Find all starting and ending positions of code blocks - start_positions = find_subsequence_positions(start_delim_units, content_units) - end_positions = setdiff(find_subsequence_positions(end_delim_units, content_units), - start_positions) + pos = find_subsequence_positions(start_delim_units1, content_units) + pos2 = find_subsequence_positions(start_delim_units2, content_units) + # the +1 offset is because the first pattern starts 1 character earlier + start_positions = vcat(pos2, pos .+ 1) |> unique + + pos = find_subsequence_positions(end_delim_units1, content_units) + pos2 = find_subsequence_positions(end_delim_units2, content_units) + end_positions = vcat(pos, pos2) |> unique unused_end_positions = trues(length(end_positions)) # Generate code block position pairs @@ -461,7 +470,9 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString} eltype_ = typeof(@view(markdown_content[begin:end])) code_blocks = Vector{eltype_}() for (start_pos, end_pos) in filtered_positions - code_block = markdown_content[(start_pos + length(start_delim_units)):(end_pos - 1)] + start_ = (start_pos + length(start_delim_units2)) + end_ = prevind(markdown_content, end_pos) + code_block = markdown_content[start_:end_] # Also remove the julia prompt push!(code_blocks, remove_julia_prompt(strip(code_block))) end @@ -470,7 +481,7 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString} end """ - extract_code_blocks_fallback(markdown_content::String, delim::AbstractString="```") + extract_code_blocks_fallback(markdown_content::String, delim::AbstractString="\\n```\\n") Extract Julia code blocks from a markdown string using a fallback method (splitting by arbitrary `delim`-iters). Much more simplistic than `extract_code_blocks` and does not support nested code blocks. @@ -497,22 +508,38 @@ code_parsed = extract_code_blocks_fallback(code) |> x -> filter(is_julia_code, x ``` """ function extract_code_blocks_fallback(markdown_content::T, - delim::AbstractString = "```") where {T <: AbstractString} + delim::AbstractString = "\n```\n") where {T <: AbstractString} # Convert content and delimiters to codeunits content_units = codeunits(markdown_content) + content_length = length(content_units) delim_units = codeunits(delim) delim_positions = find_subsequence_positions(delim_units, content_units) # Extract code blocks eltype_ = typeof(@view(markdown_content[begin:end])) code_blocks = Vector{eltype_}() - isempty(delim_positions) && return code_blocks + isempty(delim_positions) && !startswith(markdown_content, lstrip(delim)) && + return code_blocks # Run the extraction - start_pos = delim_positions[1] - for end_pos in delim_positions - if end_pos > start_pos - code_block = markdown_content[(start_pos + length(delim_units)):(end_pos - 1)] + # catch if we're missing the opening mark because of document start + no_newline_start = lstrip(delim) + start_pos = if no_newline_start != delim && + startswith(markdown_content, no_newline_start) + (length(codeunits(no_newline_start)) - length(delim_units)) + else + delim_positions[1] + end + no_new_line_end = rstrip(delim) + if no_new_line_end != delim && endswith(markdown_content, no_new_line_end) + last_end = 1 + content_length - length(codeunits(no_new_line_end)) + push!(delim_positions, last_end) + end + # start the iteration + for end_pos in unique(delim_positions) + if end_pos > start_pos && end_pos <= content_length + end_ = prevind(markdown_content, end_pos) + code_block = markdown_content[(start_pos + length(delim_units)):end_] # Also remove the julia prompt push!(code_blocks, remove_julia_prompt(strip(code_block))) # Reset the start diff --git a/test/code_generation.jl b/test/code_generation.jl index 6fcc53851..f2df62a5e 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -187,6 +187,12 @@ end @test extract_code_blocks(markdown_content) == SubString{String}["println(\"Hello, World!\")"] + # at edges (no newlines) + markdown_content = """```julia +println("hello") +```""" + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"hello\")"] # Multiple Julia Code Blocks markdown_content = """ ```julia @@ -270,13 +276,14 @@ end @test isempty(extract_code_blocks_fallback("Some text without code blocks")) # Adjacent Code Blocks Test - @test extract_code_blocks_fallback("```code1``` ```code2```") == ["code1", "", "code2"] + @test extract_code_blocks_fallback("```\ncode1\n```\n \n```\ncode2\n```") == + ["code1", "", "code2"] # Special Characters Test @test extract_code_blocks_fallback("```\n<>&\"'\n```") == ["<>&\"'"] # Large Input Test - large_input = "```" * repeat("large code block\n", 10) * "```" + large_input = "```\n" * repeat("large code block\n", 10) * "```" @test extract_code_blocks_fallback(large_input) == [strip(repeat("large code block\n", 10))] From c7915d99c3c5001abe6180563652187e827010c3 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 17 Dec 2023 11:02:12 +0000 Subject: [PATCH 064/251] add one more delimiter --- src/code_generation.jl | 7 ++++--- test/code_generation.jl | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/code_generation.jl b/src/code_generation.jl index 7a7a8d963..e8d8b1dc7 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -432,16 +432,17 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString} content_units = codeunits(markdown_content) # Ideal code fences start_delim_units1 = codeunits("\n```julia\n") - end_delim_units1 = codeunits("\n```\n") - # Fallback code fences start_delim_units2 = codeunits("```julia\n") + start_delim_units3 = codeunits("```julia ") # happens to small models + end_delim_units1 = codeunits("\n```\n") end_delim_units2 = codeunits("\n```") # Find all starting and ending positions of code blocks pos = find_subsequence_positions(start_delim_units1, content_units) pos2 = find_subsequence_positions(start_delim_units2, content_units) + pos3 = find_subsequence_positions(start_delim_units3, content_units) # the +1 offset is because the first pattern starts 1 character earlier - start_positions = vcat(pos2, pos .+ 1) |> unique + start_positions = vcat(pos2, pos .+ 1, pos3) |> unique |> sort pos = find_subsequence_positions(end_delim_units1, content_units) pos2 = find_subsequence_positions(end_delim_units2, content_units) diff --git a/test/code_generation.jl b/test/code_generation.jl index f2df62a5e..7c7c8839f 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -265,6 +265,18 @@ end """ @test extract_code_blocks(markdown_example) == SubString{String}["function find_match(md::AbstractString)\n return match(r\"```\\n(?:(?!\\n```)\\s*.*\\n?)*\\s*```\", md)\nend"] + + # Some small models forget newlines + no_newline = """ + ```julia function clean_column(col::AbstractString) + col = strip(lowercase(col)) + col = replace(col, r"[-\\s]+", "_") + col + end + ``` + """ + @test extract_code_blocks(no_newline) == + SubString{String}["function clean_column(col::AbstractString)\n col = strip(lowercase(col))\n col = replace(col, r\"[-\\s]+\", \"_\")\n col\nend"] end @testset "extract_code_blocks_fallback" begin @@ -290,6 +302,18 @@ end # Empty String Test @test isempty(extract_code_blocks_fallback("")) + # delimiter inside of code + delim_in_middle = """ + ``` + function myadd(a, b) + # here is a silly comment that ends with ``` + return a + b + end + ``` + """ + @test extract_code_blocks_fallback(delim_in_middle) == + SubString{String}["function myadd(a, b)\n # here is a silly comment that ends with ```\n return a + b\nend"] + # Different Delimiter Test @test extract_code_blocks_fallback("~~~\ncode block\n~~~", "~~~") == ["code block"] end From fef3f99fa2c6f612c9af366ed20a012ab1f4efef Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 20 Dec 2023 08:19:46 +0000 Subject: [PATCH 065/251] fix model kwarg --- CHANGELOG.md | 1 + src/llm_ollama_managed.jl | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9bf46e7a..342e114d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Stricter code parsing in `AICode` to avoid false positives (code blocks must end with "```\n" to catch comments inside text) - Introduced an option `skip_invalid=true` for `AICode`, which allows you to include only code blocks that parse successfully (useful when the code definition is good, but the subsequent examples are not), and an option `capture_stdout=false` to avoid capturing stdout if you want to evaluate `AICode` in parallel (`Pipe()` that we use is NOT thread-safe) +- `OllamaManagedSchema` was passing an incorrect model name to the Ollama server, often serving the default llama2 model instead of the requested model. This is now fixed. ## [0.4.0] diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 431976734..a68d6f27d 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -195,7 +195,7 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ if !dry_run time = @elapsed resp = ollama_api(prompt_schema, conv_rendered.prompt; - conv_rendered.system, endpoint = "generate", model_id, http_kwargs, + conv_rendered.system, endpoint = "generate", model = model_id, http_kwargs, api_kwargs...) msg = AIMessage(; content = resp.response[:response] |> strip, status = Int(resp.status), @@ -305,7 +305,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) time = @elapsed resp = ollama_api(prompt_schema, doc; - endpoint = "embeddings", model_id, http_kwargs, api_kwargs...) + endpoint = "embeddings", model = model_id, http_kwargs, api_kwargs...) msg = DataMessage(; content = postprocess(resp.response[:embedding]), status = Int(resp.status), @@ -332,7 +332,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, postprocess; verbose = false, api_key, - model, + model = model_id, kwargs...) for doc in docs] ## Aggregate results From b73e701abf9d439d3e07d1b89ac829b4afba9333 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 20 Dec 2023 08:30:17 +0000 Subject: [PATCH 066/251] switch on ollama tests --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index 34b73012a..c4cc26288 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -16,6 +16,7 @@ end include("llm_interface.jl") include("llm_shared.jl") include("llm_openai.jl") + include("llm_ollama_managed.jl") include("templates.jl") include("serialization.jl") include("code_generation.jl") From 5822069c641cc32baa64c01c42f2edb1056889ea Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 20 Dec 2023 08:36:41 +0000 Subject: [PATCH 067/251] increase tolerance on timings --- test/llm_ollama_managed.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/llm_ollama_managed.jl b/test/llm_ollama_managed.jl index 98e090961..0ddb7f93f 100644 --- a/test/llm_ollama_managed.jl +++ b/test/llm_ollama_managed.jl @@ -91,7 +91,7 @@ end @test msg.content == prompt @test msg.status == 200 @test msg.tokens == (2, 1) - @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test isapprox(msg.elapsed, 0, atol = 3e-1) @test schema.inputs == (; system, prompt) @test schema.model_id == "llama2" end @@ -105,7 +105,7 @@ end @test msg.content == "Hello John" @test msg.status == 200 @test msg.tokens == (2, 1) - @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test isapprox(msg.elapsed, 0, atol = 3e-1) @test schema.inputs == (; system = "Act as a helpful AI assistant", prompt = "Hello John") @test schema.model_id == "llama2aaaa" @@ -133,7 +133,7 @@ end @test msg.content == ones(16) @test msg.status == 200 @test msg.tokens == (0, 0) - @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test isapprox(msg.elapsed, 0, atol = 3e-1) @test schema.inputs == (; system = nothing, prompt = doc) @test schema.model_id == "llama2" end @@ -146,7 +146,7 @@ end @test msg.content == 2 * ones(16, 2) @test msg.status == 200 @test msg.tokens == (0, 0) - @test isapprox(msg.elapsed, 0, atol = 1e-2) + @test isapprox(msg.elapsed, 0, atol = 3e-1) @test schema.inputs == (; system = nothing, prompt = docs[2]) # only the last doc is caught (serial execution) @test schema.model_id == "llama2" end From c2ff9634e0444d488b9d4dcf71e81bec44da33e6 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 21 Dec 2023 21:03:20 +0000 Subject: [PATCH 068/251] add draft --- CHANGELOG.md | 1 + Project.toml | 13 +- ext/RAGToolsExperimentalExt.jl | 34 +++++ src/Experimental/Experimental.jl | 15 ++ src/Experimental/RAGTools/RAGTools.jl | 33 +++++ src/Experimental/RAGTools/evaluation.jl | 106 ++++++++++++++ src/Experimental/RAGTools/generation.jl | 128 +++++++++++++++++ src/Experimental/RAGTools/preparation.jl | 144 +++++++++++++++++++ src/Experimental/RAGTools/retrieval.jl | 45 ++++++ src/Experimental/RAGTools/types.jl | 132 +++++++++++++++++ src/Experimental/RAGTools/utils.jl | 17 +++ src/PromptingTools.jl | 3 + src/utils.jl | 125 +++++++++++++++- templates/RAG/CreateQAFromContext.json | 1 + templates/RAG/RAGAnswerFromContext.json | 1 + templates/RAG/RAGCreateQAFromContext.json | 1 + templates/RAG/RAGExtractMetadataLong.json | 1 + templates/RAG/RAGExtractMetadataShort.json | 1 + templates/RAG/RAGJudgeAnswerFromContext.json | 1 + test/Experimental/RAGTools.jl/preparation.jl | 68 +++++++++ test/Experimental/RAGTools.jl/retrieval.jl | 10 ++ test/Experimental/RAGTools.jl/runtests.jl | 10 ++ test/Experimental/RAGTools.jl/types.jl | 121 ++++++++++++++++ test/Experimental/RAGTools.jl/utils.jl | 0 test/runtests.jl | 1 + test/utils.jl | 61 ++++++-- 26 files changed, 1056 insertions(+), 17 deletions(-) create mode 100644 ext/RAGToolsExperimentalExt.jl create mode 100644 src/Experimental/Experimental.jl create mode 100644 src/Experimental/RAGTools/RAGTools.jl create mode 100644 src/Experimental/RAGTools/evaluation.jl create mode 100644 src/Experimental/RAGTools/generation.jl create mode 100644 src/Experimental/RAGTools/preparation.jl create mode 100644 src/Experimental/RAGTools/retrieval.jl create mode 100644 src/Experimental/RAGTools/types.jl create mode 100644 src/Experimental/RAGTools/utils.jl create mode 100644 templates/RAG/CreateQAFromContext.json create mode 100644 templates/RAG/RAGAnswerFromContext.json create mode 100644 templates/RAG/RAGCreateQAFromContext.json create mode 100644 templates/RAG/RAGExtractMetadataLong.json create mode 100644 templates/RAG/RAGExtractMetadataShort.json create mode 100644 templates/RAG/RAGJudgeAnswerFromContext.json create mode 100644 test/Experimental/RAGTools.jl/preparation.jl create mode 100644 test/Experimental/RAGTools.jl/retrieval.jl create mode 100644 test/Experimental/RAGTools.jl/runtests.jl create mode 100644 test/Experimental/RAGTools.jl/types.jl create mode 100644 test/Experimental/RAGTools.jl/utils.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 342e114d8..050419827 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Experimental sub-module RAGTools providing basic Retrieval-Augmented Generation functionality. See `?RAGTools` for more information. It's nested inside of `PromptingTools.Experimental.RAGTools` to signify that it might change in the future. ### Fixed - Stricter code parsing in `AICode` to avoid false positives (code blocks must end with "```\n" to catch comments inside text) diff --git a/Project.toml b/Project.toml index 17136594e..66f9ada1d 100644 --- a/Project.toml +++ b/Project.toml @@ -12,21 +12,32 @@ OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" +[weakdeps] +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[extensions] +RAGToolsExperimentalExt = ["SparseArrays","LinearAlgebra"] + [compat] Aqua = "0.7" Base64 = "<0.0.1, 1" HTTP = "1" JSON3 = "1" Logging = "<0.0.1, 1" +LinearAlgebra = "<0.0.1, 1" OpenAI = "0.8.7" PrecompileTools = "1" Preferences = "1" +SparseArrays = "<0.0.1, 1" Test = "<0.0.1, 1" julia = "1.9,1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [targets] -test = ["Aqua", "Test"] +test = ["Aqua", "Test", "SparseArrays","LinearAlgebra"] diff --git a/ext/RAGToolsExperimentalExt.jl b/ext/RAGToolsExperimentalExt.jl new file mode 100644 index 000000000..7047853c0 --- /dev/null +++ b/ext/RAGToolsExperimentalExt.jl @@ -0,0 +1,34 @@ +module RAGToolsExperimentalExt + +using PromptingTools, SparseArrays +using LinearAlgebra: normalize +const PT = PromptingTools + +using PromptingTools.Experimental.RAGTools + +# forward to LinearAlgebra.normalize +PromptingTools.Experimental.RAGTools._normalize(arr::AbstractArray) = normalize(arr) + +# "Builds a sparse matrix of tags and a vocabulary from the given vector of chunk metadata. Requires SparseArrays.jl to be loaded." +function PromptingTools.Experimental.RAGTools.build_tags(chunk_metadata::Vector{ + Vector{String}, + }) + tags_vocab_ = vcat(chunk_metadata...) |> unique |> sort + tags_vocab_index = Dict{String, Int}(t => i for (i, t) in enumerate(tags_vocab_)) + Is, Js = Int[], Int[] + for i in eachindex(chunk_metadata) + for tag in chunk_metadata[i] + push!(Is, i) + push!(Js, tags_vocab_index[tag]) + end + end + tags_ = sparse(Is, + Js, + trues(length(Is)), + length(chunk_metadata), + length(tags_vocab_), + &) + return tags_, tags_vocab_ +end + +end \ No newline at end of file diff --git a/src/Experimental/Experimental.jl b/src/Experimental/Experimental.jl new file mode 100644 index 000000000..6a1f2cb20 --- /dev/null +++ b/src/Experimental/Experimental.jl @@ -0,0 +1,15 @@ +""" + Experimental + +This module is for experimental code that is not yet ready for production. +It is not included in the main module, so it must be explicitly imported. + +Contains: +- `RAGTools`: Retrieval-Augmented Generation (RAG) functionality. +""" +module Experimental + +export RAGTools +include("RAGTools/RAGTools.jl") + +end # module Experimental \ No newline at end of file diff --git a/src/Experimental/RAGTools/RAGTools.jl b/src/Experimental/RAGTools/RAGTools.jl new file mode 100644 index 000000000..47d4113ce --- /dev/null +++ b/src/Experimental/RAGTools/RAGTools.jl @@ -0,0 +1,33 @@ +""" + RAGTools + +Provides Retrieval-Augmented Generation (RAG) functionality. + +Requires: LinearAlgebra, SparseArrays, PromptingTools for proper functionality. + +This module is experimental and may change at any time. It is intended to be moved to a separate package in the future. +""" +module RAGTools + +using PromptingTools +using JSON3 +const PT = PromptingTools + +include("utils.jl") + +export ChunkIndex, CandidateChunks # MultiIndex +include("types.jl") + +export build_index, build_tags +include("preparation.jl") + +export find_closest, find_tags, rerank +include("retrieval.jl") + +export airag +include("generation.jl") + +export build_qa_evals +include("evaluation.jl") + +end \ No newline at end of file diff --git a/src/Experimental/RAGTools/evaluation.jl b/src/Experimental/RAGTools/evaluation.jl new file mode 100644 index 000000000..5ded7896d --- /dev/null +++ b/src/Experimental/RAGTools/evaluation.jl @@ -0,0 +1,106 @@ +### For testing and eval +# This is a return_type for extraction when generating Q&A set with aiextract +@kwdef struct QAItem + question::String + answer::String +end +# This is for saving in JSON format for evaluation later +@kwdef struct QAEvalItem + source::String = "" + context::String = "" + question::String = "" + answer::String = "" +end + +"Provide the `final_rating` between 1-5. Provide the rationale for it." +@kwdef struct JudgeRating + rationale::Union{Nothing, String} = nothing + final_rating::Int +end +"Explain the `final_rating` in `rationale`" +@kwdef struct JudgeAllScores + relevance::Int + completeness::Int + clarity::Int + consistency::Int + helpfulness::Int + rationale::Union{Nothing, String} = nothing + final_rating::Int +end + +function Base.isvalid(x::QAEvalItem) + !isempty(x.question) && !isempty(x.answer) && !isempty(x.context) +end + +# Nicer show method with some colors! +function Base.show(io::IO, t::Union{QAItem, QAEvalItem}) + printstyled(io, "$(nameof(typeof(t))):\n", color = :green, bold = true) + for f in fieldnames(typeof(t)) + printstyled(io, " ", f, color = :blue, bold = true) + println(io, ": ", getfield(t, f)) + end +end +# Define how JSON3 should serialize/deserialize the struct into JSON files +JSON3.StructTypes.StructType(::Type{QAEvalItem}) = JSON3.StructTypes.Struct() + +""" + build_qa_evals(doc_chunks::Vector{<:AbstractString}, sources::Vector{<:AbstractString}; + model=PT.MODEL_CHAT, instructions="None.", qa_template::Symbol=:RAGCreateQAFromContext, verbose::Bool=true, kwargs...) -> Vector{QAEvalItem} + +Create a collection of question and answer evaluations (`QAEvalItem`) from document chunks and sources. +This function generates Q&A pairs based on the provided document chunks, using a specified AI model and template. + +# Arguments +- `doc_chunks::Vector{<:AbstractString}`: A vector of document chunks, each representing a segment of text. +- `sources::Vector{<:AbstractString}`: A vector of source identifiers corresponding to each chunk in `doc_chunks` (eg, filenames or paths). +- `model`: The AI model used for generating Q&A pairs. Default is `PT.MODEL_CHAT`. +- `instructions::String`: Additional instructions or context to provide to the model generating QA sets. Defaults to "None.". +- `qa_template::Symbol`: A template symbol that dictates the AITemplate that will be used. It must have placeholder `context`. Default is `:CreateQAFromContext`. +- `verbose::Bool`: If `true`, additional information like costs will be logged. Defaults to `true`. + +# Returns +`Vector{QAEvalItem}`: A vector of `QAEvalItem` structs, each containing a source, context, question, and answer. Invalid or empty items are filtered out. + +# Notes + +- The function internally uses `aiextract` to generate Q&A pairs based on the provided `qa_template`. So you can use any kwargs that you want. +- Each `QAEvalItem` includes the context (document chunk), the generated question and answer, and the source. +- The function tracks and reports the cost of AI calls if `verbose` is enabled. +- Items where the question, answer, or context is empty are considered invalid and are filtered out. + +# Examples + +Creating Q&A evaluations from a set of document chunks: +```julia +doc_chunks = ["Text from document 1", "Text from document 2"] +sources = ["source1", "source2"] +qa_evals = build_qa_evals(doc_chunks, sources) +``` +""" +function build_qa_evals(doc_chunks::Vector{<:AbstractString}, + sources::Vector{<:AbstractString}; + model = PT.MODEL_CHAT, instructions = "None.", + qa_template::Symbol = :RAGCreateQAFromContext, verbose::Bool = true, kwargs...) + ## + @assert length(doc_chunks)==length(sources) "Length of `doc_chunks` and `sources` must be the same." + placeholders = only(aitemplates(qa_template)).variables # only one template should be found + @assert (:context in placeholders) "Provided Q&A Template $(qa_template) is not suitable. It must have placeholder: `context`." + ## + cost_tracker = Threads.Atomic{Float64}(0.0) + output = asyncmap(zip(doc_chunks, sources)) do (context, source) + try + msg = aiextract(qa_template; + return_type = QAItem, + context, + instructions, + verbose, + model) + Threads.atomic_add!(cost_tracker, PT.call_cost(msg, model)) # track costs + QAEvalItem(; context, msg.content.question, msg.content.answer, source) + catch e + QAEvalItem() + end + end + verbose && @info "Q&A Sets built! (cost: \$$(round(cost_tracker[], digits=3)))" + return filter(isvalid, output) +end diff --git a/src/Experimental/RAGTools/generation.jl b/src/Experimental/RAGTools/generation.jl new file mode 100644 index 000000000..804e0fdd0 --- /dev/null +++ b/src/Experimental/RAGTools/generation.jl @@ -0,0 +1,128 @@ +# stub to be replaced with extension +function _normalize end + +""" + airag(index::AbstractChunkIndex, rag_template::Symbol=:RAGAnswerFromContext; + question::AbstractString, top_k::Int=3, tag_filter::Union{Symbol,Vector{String},Regex}=:auto, + rerank_strategy::RerankingStrategy=Passthrough(), model_embedding::String=PT.MODEL_EMBEDDING, + model_chat::String=PT.MODEL_CHAT, model_metadata::String=PT.MODEL_CHAT, + chunks_window_margin::Tuple{Int,Int}=(1, 1), return_context::Bool=false, verbose::Bool=true, kwargs...) -> Any + +Generates a response for a given question using a Retrieval-Augmented Generation (RAG) approach. + +The function selects relevant chunks from an `ChunkIndex`, optionally filters them based on metadata tags, reranks them, and then uses these chunks to construct a context for generating a response. + +# Arguments +- `index::AbstractChunkIndex`: The chunk index to search for relevant text. +- `rag_template::Symbol`: Template for the RAG model, defaults to `:RAGAnswerFromContext`. +- `question::AbstractString`: The question to be answered. +- `top_k::Int`: Number of top candidates to retrieve based on embedding similarity. +- `tag_filter::Union{Symbol, Vector{String}, Regex}`: Mechanism for filtering chunks based on tags (either automatically detected, specific tags, or a regex pattern). +- `rerank_strategy::RerankingStrategy`: Strategy for reranking the retrieved chunks. +- `model_embedding::String`: Model used for embedding the question, default is `PT.MODEL_EMBEDDING`. +- `model_chat::String`: Model used for generating the final response, default is `PT.MODEL_CHAT`. +- `model_metadata::String`: Model used for extracting metadata, default is `PT.MODEL_CHAT`. +- `chunks_window_margin::Tuple{Int,Int}`: The window size around each chunk to consider for context building. +- `return_context::Bool`: If `true`, returns the context used for RAG along with the response. +- `verbose::Bool`: If `true`, enables verbose logging. + +# Returns +- If `return_context` is `false`, returns the generated message (`msg`). +- If `return_context` is `true`, returns a tuple of the generated message (`msg`) and the RAG context (`rag_context`). + +# Notes +- The function first finds the closest chunks to the question embedding, then optionally filters these based on tags. After that, it reranks the candidates and builds a context for the RAG model. +- The `tag_filter` can be used to refine the search. If set to `:auto`, it attempts to automatically determine relevant tags (if `index` has them available). +- The `chunks_window_margin` allows including surrounding chunks for richer context, considering they are from the same source. +- The function currently supports only single `ChunkIndex`. + +# Examples + +Using `airag` to get a response for a question: +```julia +index = build_index(...) # create an index +question = "How to make a barplot in Makie.jl?" +msg = airag(index, :RAGAnswerFromContext; question) + +# or simply +msg = airag(index; question) +``` +""" +function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromContext; + question::AbstractString, + top_k::Int = 3, + tag_filter::Union{Symbol, Vector{String}, Regex} = :auto, + rerank_strategy::RerankingStrategy = Passthrough(), + model_embedding::String = PT.MODEL_EMBEDDING, model_chat::String = PT.MODEL_CHAT, + model_metadata::String = PT.MODEL_CHAT, + chunks_window_margin::Tuple{Int, Int} = (1, 1), + return_context::Bool = false, verbose::Bool = true, + kwargs...) + ## Note: Supports only single ChunkIndex for now + ## Checks + @assert tag_filter isa Symbol&&tag_filter == :auto "Only `:auto`, `Vector{String}`, or `Regex` are supported for `tag_filter`" + @assert chunks_window_margin[1] >= 0&&chunks_window_margin[2] >= 0 "Both `chunks_window_margin` values must be non-negative" + placeholders = only(aitemplates(rag_template)).variables # only one template should be found + @assert (:question in placeholders)&&(:context in placeholders) "Provided RAG Template $(rag_template) is not suitable. It must have placeholders: `question` and `context`." + + question_emb = aiembed(question, + _normalize; + model = model_embedding, + verbose).content .|> Float32 + emb_candidates = find_closest(index, question_emb; top_k) + + tag_candidates = if tag_filter == :auto && !isnothing(tags(index)) && + !isempty(model_metadata) + # extract metadata via LLM call + # Check that the provided model is known and that it is an OpenAI model (for the aiextract function to work) + @assert haskey(PT.MODEL_REGISTRY, + model_metadata)&&PT.MODEL_REGISTRY[model_metadata].schema == PT.OpenAISchema() "Only OpenAI models support the metadata extraction now. $model_metadata is not a registered OpenAI model." + metadata_ = try + msg = aiextract(metadata_template; return_type = MaybeMetadataItems, + text = chunk, + instructions = "In addition to extracted items, suggest 2-3 filter keywords that could be relevant to answer this question.", + verbose, model = model_metadata) + metadata_extract(msg.content.items) + catch + String[] + end + find_tags(index, metadata_) + elseif !(tag_filter isa Symbol) + find_tags(index, tag_filter) + else + ## not filtering -- use all rows and ignore this + nothing + end + + filtered_candidates = isnothing(tag_candidates) ? emb_candidates : + (emb_candidates & tag_candidates) + reranked_candidates = rerank(rerank_strategy, index, question, filtered_candidates) + + ## Build the context + context = String[] + for (i, position) in enumerate(reranked_candidates.positions) + ## Add surrounding chunks if they are from the same source (from `chunks_window_margin`) + chunks_ = chunks(index)[max(1, position - chunks_window_margin[1]):min(end, + position + chunks_window_margin[2])] + is_same_source = sources(index)[max(1, position - chunks_window_margin[1]):min(end, + position + chunks_window_margin[2])] .== sources(index)[position] + push!(context, "$(i). $(join(chunks_[is_same_source], "\n"))") + end + ## LLM call + msg = aigenerate(rag_template; question, + context = join(context, "\n\n"), model = model_chat, verbose, + kwargs...) + + if return_context # for evaluation + rag_context = RAGContext(; + question, + context, + emb_candidates, + tag_candidates, + filtered_candidates, + reranked_candidates) + return msg, rag_context + else + return msg + end +end \ No newline at end of file diff --git a/src/Experimental/RAGTools/preparation.jl b/src/Experimental/RAGTools/preparation.jl new file mode 100644 index 000000000..3c06dc403 --- /dev/null +++ b/src/Experimental/RAGTools/preparation.jl @@ -0,0 +1,144 @@ +### Preparation +# Types used to extract `tags` from document chunks +@kwdef struct MetadataItem + value::String + category::String +end +@kwdef struct MaybeMetadataItems + items::Union{Nothing, Vector{MetadataItem}} +end + +""" + metadata_extract(item::MetadataItem) + metadata_extract(items::Vector{MetadataItem}) + +Extracts the metadata item into a string of the form `category:::value` (lowercased and spaces replaced with underscores). + +# Example +```julia +msg = aiextract(:RAGExtractMetadataShort; return_type=MaybeMetadataItems, text="I like package DataFrames", instructions="None.") +metadata = metadata_extract(msg.content.items) +``` +""" +function metadata_extract(item::MetadataItem) + "$(strip(item.category)):::$(strip(item.value))" |> lowercase |> + x -> replace(x, " " => "_") +end +metadata_extract(items::Nothing) = String[] +metadata_extract(items::Vector{MetadataItem}) = metadata_extract.(items) + +"Builds a matrix of tags and a vocabulary list. REQUIRES SparseArrays and LinearAlgebra packages to be loaded!!" +function build_tags end +# Implementation in ext/RAGToolsExperimentalExt.jl + +"Build an index for RAG (Retriever-Augmented Generation) applications. REQUIRES SparseArrays and LinearAlgebra packages to be loaded!!" +function build_index end + +""" + build_index(files::Vector{<:AbstractString}; + separators=["\n\n", ". ", "\n"], max_length::Int=256, + extract_metadata::Bool=false, verbose::Bool=true, metadata_template::Symbol=:RAGExtractMetadataShort, + model_embedding::String=PT.MODEL_EMBEDDING, model_metadata::String=PT.MODEL_CHAT) + +Build an index for RAG (Retriever-Augmented Generation) applications from the provided file paths. +The function processes each file, splits its content into chunks, embeds these chunks, +optionally extracts metadata, and then compiles this information into a retrievable index. + +# Arguments +- `files`: A vector of valid file paths to be indexed. +- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `["\\n\\n", ". ", "\\n"]`. +- `max_length`: The maximum length of each chunk (if possible with provided separators). Default is 256. +- `extract_metadata`: A boolean flag indicating whether to extract metadata from each chunk (to build filter `tags` in the index). Default is `false`. + Metadata extraction incurs additional cost and requires `model_metadata` and `metadata_template` to be provided. +- `verbose`: A boolean flag for verbose output. Default is `true`. +- `metadata_template`: A symbol indicating the template to be used for metadata extraction. Default is `:RAGExtractMetadataShort`. +- `model_embedding`: The model to use for embedding. +- `model_metadata`: The model to use for metadata extraction. + +# Returns +- `ChunkIndex`: An object containing the compiled index of chunks, embeddings, tags, vocabulary, and sources. + +See also: `MultiIndex`, `CandidateChunks`, `find_closest`, `find_tags`, `rerank`, `airag` + +# Examples +```julia +# Assuming `test_files` is a vector of file paths +index = build_index(test_files; max_length=10, extract_metadata=true) + +# Another example with metadata extraction and verbose output +index = build_index(["file1.txt", "file2.txt"]; + separators=[". "], + extract_metadata=true, + verbose=true) +``` +""" +function build_index(files::Vector{<:AbstractString}; + separators = ["\n\n", ". ", "\n"], max_length::Int = 256, + extract_metadata::Bool = false, verbose::Bool = true, + metadata_template::Symbol = :RAGExtractMetadataShort, + model_embedding::String = PT.MODEL_EMBEDDING, + model_metadata::String = PT.MODEL_CHAT) + ## + @assert all(isfile, files) "Some `files` don't exist (Check: $(join(filter(!isfile,files),", "))" + + output_chunks = Vector{Vector{SubString{String}}}() + output_embeddings = Vector{Matrix{Float32}}() + output_metadata = Vector{Vector{Vector{String}}}() + output_sources = Vector{Vector{eltype(files)}}() + cost_tracker = Threads.Atomic{Float64}(0.0) + + for fn in files + verbose && @info "Processing file: $fn" + doc_raw = read(fn, String) + isempty(doc_raw) && continue + # split into chunks, if you want to start simple - just do `split(text,"\n\n")` + doc_chunks = PT.split_by_length(doc_raw, separators; max_length) .|> strip |> + x -> filter(!isempty, x) + # skip if no chunks found + isempty(doc_chunks) && continue + push!(output_chunks, doc_chunks) + push!(output_sources, fill(fn, length(doc_chunks))) + + # Notice that we embed all doc_chunks at once, not one by one + # OpenAI supports embedding multiple documents to reduce the number of API calls/network latency time + emb = aiembed(doc_chunks, _normalize; model = model_embedding, verbose) + Threads.atomic_add!(cost_tracker, PT.call_cost(emb, model_embedding)) # track costs + push!(output_embeddings, Float32.(emb.content)) + + if extract_metadata && !isempty(model_metadata) + # Check that the provided model is known and that it is an OpenAI model (for the aiextract function to work) + @assert haskey(PT.MODEL_REGISTRY, + model_metadata)&&PT.MODEL_REGISTRY[model_metadata].schema == PT.OpenAISchema() "Only OpenAI models support the metadata extraction now. $model_metadata is not a registered OpenAI model." + metadata_ = asyncmap(doc_chunks) do chunk + try + msg = aiextract(metadata_template; + return_type = MaybeMetadataItems, + text = chunk, + instructions = "None.", + verbose, + model = model_metadata) + Threads.atomic_add!(cost_tracker, PT.call_cost(msg, model_metadata)) # track costs + items = metadata_extract(msg.content.items) + catch + String[] + end + end + push!(output_metadata, metadata_) + end + end + ## Create metadata tags and associated vocabulary + tags, tags_vocab = if !isempty(output_metadata) + # Requires SparseArrays.jl! + _build_tags(vcat(output_metadata...)) # need to vcat to be on the "chunk-level" + else + tags, tags_vocab = nothing, nothing + end + verbose && @info "Index built! (cost: \$$(round(cost_tracker[], digits=3)))" + + index = ChunkIndex(; + embeddings = hcat(output_embeddings...), + tags, tags_vocab, + chunks = vcat(output_chunks...), + sources = vcat(output_sources...)) + return index +end diff --git a/src/Experimental/RAGTools/retrieval.jl b/src/Experimental/RAGTools/retrieval.jl new file mode 100644 index 000000000..b824b5396 --- /dev/null +++ b/src/Experimental/RAGTools/retrieval.jl @@ -0,0 +1,45 @@ +"Finds the indices of chunks (represented by embeddings in `emb`) that are closest (cosine similarity) to query embedding (`query_emb`). Returns only `top_k` closest indices." +function find_closest(emb::AbstractMatrix{<:Real}, + query_emb::AbstractVector{<:Real}; + top_k::Int = 100) + # emb is an embedding matrix where the first dimension is the embedding dimension + distances = query_emb' * emb |> vec + positions = distances |> sortperm |> reverse |> x -> first(x, top_k) + return positions, distances[positions] +end +function find_closest(index::AbstractChunkIndex, + query_emb::AbstractVector{<:Real}; + top_k::Int = 100) + isnothing(embeddings(index)) && CandidateChunks(; index_id = index.id) + positions, distances = find_closest(embeddings(index), query_emb; top_k) + return CandidateChunks(index.id, positions, Float32.(distances)) +end + +function find_tags(index::AbstractChunkIndex, + tag::Union{AbstractString, Regex}) + isnothing(tags(index)) && CandidateChunks(; index_id = index.id) + tag_idx = if tag isa AbstractString + findall(tags_vocab(index) .== tag) + else # assume it's a regex + findall(occursin.(tag, tags_vocab(index))) + end + # getindex.(x, 1) is to get the first dimension in each CartesianIndex + match_row_idx = @view(tags(index)[:, tag_idx]) |> findall |> + x -> getindex.(x, 1) |> unique + return CandidateChunks(index.id, match_row_idx, ones(Float32, length(match_row_idx))) +end +function find_tags(index::AbstractChunkIndex, + tags::Vector{<:AbstractString}) + pos = Int[find_tags(index, tag).positions for tag in tags] |> unique + return CandidateChunks(index.id, pos, ones(Float32, length(pos))) +end + +# Assuming the rerank and strategy definitions are in the Main module or relevant module +abstract type RerankingStrategy end + +struct Passthrough <: RerankingStrategy end + +function rerank(strategy::Passthrough, index, question, candidate_chunks; kwargs...) + # Since this is a Passthrough strategy, it returns the candidate_chunks unchanged + return candidate_chunks +end \ No newline at end of file diff --git a/src/Experimental/RAGTools/types.jl b/src/Experimental/RAGTools/types.jl new file mode 100644 index 000000000..c30f4408e --- /dev/null +++ b/src/Experimental/RAGTools/types.jl @@ -0,0 +1,132 @@ +### Types +# Defines three key types for RAG: ChunkIndex, MultiIndex, and CandidateChunks +# In addition, RAGContext is defined for debugging purposes + +abstract type AbstractDocumentIndex end +abstract type AbstractChunkIndex <: AbstractDocumentIndex end +# More advanced index would be: HybridChunkIndex + +# Stores document chunks and their embeddings +@kwdef struct ChunkIndex{ + T1 <: AbstractString, + T2 <: Union{Nothing, Matrix{<:Real}}, + T3 <: Union{Nothing, AbstractMatrix{<:Bool}}, +} <: AbstractChunkIndex + id::Symbol = gensym("ChunkIndex") + # underlying document chunks / snippets + chunks::Vector{T1} + # for semantic search + embeddings::T2 = nothing + # for exact search, filtering, etc. + # expected to be some sparse structure, eg, sparse matrix or nothing + # column oriented, ie, each column is one item in `tags_vocab` and rows are the chunks + tags::T3 = nothing + tags_vocab::Union{Nothing, Vector{<:AbstractString}} = nothing + sources::Vector{<:AbstractString} +end +embeddings(index::ChunkIndex) = index.embeddings +chunks(index::ChunkIndex) = index.chunks +tags(index::ChunkIndex) = index.tags +tags_vocab(index::ChunkIndex) = index.tags_vocab +sources(index::ChunkIndex) = index.sources + +function Base.var"=="(i1::ChunkIndex, i2::ChunkIndex) + ((i1.sources == i2.sources) && (i1.tags_vocab == i2.tags_vocab) && + (i1.embeddings == i2.embeddings) && (i1.chunks == i2.chunks) && (i1.tags == i2.tags)) +end + +function Base.vcat(i1::ChunkIndex, i2::ChunkIndex) + tags, tags_vocab = if (isnothing(tags(i1)) || isnothing(tags(i2))) + nothing, nothing + elseif tags_vocab(i1) == tags_vocab(i2) + vcat(tags(i1), tags(i2)), tags_vocab(i1) + else + merge_labeled_matrices(tags(i1), tags_vocab(i1), tags(i2), tags_vocab(i2)) + end + embeddings = (isnothing(embeddings(i1)) || isnothing(embeddings(i2))) ? nothing : + hcat(embeddings(i1), embeddings(i2)) + ChunkIndex(; + chunks = vcat(chunks(i1), chunks(i2)), + embeddings, + tags, + tags_vocab, + sources = vcat(i1.sources, i2.sources)) +end + +"Composite index that stores multiple ChunkIndex objects and their embeddings" +@kwdef struct MultiIndex <: AbstractDocumentIndex + id::Symbol = gensym("MultiIndex") + indexes::Vector{<:ChunkIndex} +end +indexes(index::MultiIndex) = index.indexes +# check that each index has a counterpart in the other MultiIndex +function Base.var"=="(i1::MultiIndex, i2::MultiIndex) + length(indexes(i1)) != length(indexes(i2)) && return false + for i in i1.indexes + if !(i in i2.indexes) + return false + end + end + for i in i2.indexes + if !(i in i1.indexes) + return false + end + end + return true +end + +abstract type AbstractCandidateChunks end +@kwdef struct CandidateChunks{T <: Real} <: AbstractCandidateChunks + index_id::Symbol + positions::Vector{Int} = Int[] + distances::Vector{T} = Float32[] +end +# combine/intersect two candidate chunks. average the score if available +function Base.var"&"(cc1::CandidateChunks, cc2::CandidateChunks) + cc1.index_id != cc2.index_id && return CandidateChunks(; index_id = cc1.index_id) + + positions = intersect(cc1.positions, cc2.positions) + distances = if !isempty(cc1.distances) && !isempty(cc2.distances) + (cc1.distances[positions] .+ cc2.distances[positions]) ./ 2 + else + Float32[] + end + CandidateChunks(cc1.index_id, positions, distances) +end +function Base.getindex(ci::ChunkIndex, candidate::CandidateChunks, field::Symbol = :chunks) + @assert field==:chunks "Only `chunks` field is supported for now" + if ci.id == candidate.index_id + chunks(ci)[candidate.positions] + else + eltype(chunks(ci))[] + end +end +function Base.getindex(mi::MultiIndex, candidate::CandidateChunks, field::Symbol = :chunks) + @assert field==:chunks "Only `chunks` field is supported for now" + valid_index = findfirst(x -> x.id == candidate.index_id, indexes(mi)) + if isnothing(valid_index) + String[] + else + getindex(indexes(mi)[valid_index], candidate) + end +end + +""" + RAGContext + +A struct for debugging RAG answers. It contains the question, context, and the candidate chunks at each step of the RAG pipeline. +""" +@kwdef struct RAGContext + question::AbstractString + context::Vector{<:AbstractString} + emb_candidates::CandidateChunks + tag_candidates::Union{Nothing, CandidateChunks} + filtered_candidates::CandidateChunks + reranked_candidates::CandidateChunks +end + +# Structured show method for easier reading (each kwarg on a new line) +function Base.show(io::IO, + t::Union{AbstractDocumentIndex, AbstractCandidateChunks, RAGContext}) + dump(IOContext(io, :limit => true), t, maxdepth = 1) +end diff --git a/src/Experimental/RAGTools/utils.jl b/src/Experimental/RAGTools/utils.jl new file mode 100644 index 000000000..d28ceae62 --- /dev/null +++ b/src/Experimental/RAGTools/utils.jl @@ -0,0 +1,17 @@ +# Utitity to be able to combine indices from different sources/documents easily +function merge_labeled_matrices(mat1::AbstractMatrix{T1}, + vocab1::Vector{String}, + mat2::AbstractMatrix{T2}, + vocab2::Vector{String}) where {T1 <: Number, T2 <: Number} + T = promote_type(T1, T2) + new_words = setdiff(vocab2, vocab1) + combined_vocab = [vocab1; new_words] + vocab2_indices = Dict(word => i for (i, word) in enumerate(vocab2)) + + aligned_mat1 = hcat(mat1, zeros(T, size(mat1, 1), length(new_words))) + aligned_mat2 = [haskey(vocab2_indices, word) ? @view(mat2[:, vocab2_indices[word]]) : + zeros(T, size(mat2, 1)) for word in combined_vocab] + aligned_mat2 = aligned_mat2 |> Base.Splat(hcat) + + return vcat(aligned_mat1, aligned_mat2), combined_vocab +end \ No newline at end of file diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 7f5c6bdd1..a137bd866 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -65,6 +65,9 @@ include("llm_ollama_managed.jl") export @ai_str, @aai_str include("macros.jl") +## Experimental modules +include("Experimental/Experimental.jl") + function __init__() # Load templates load_templates!() diff --git a/src/utils.jl b/src/utils.jl index 168d772cb..a0f6b996b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -66,7 +66,13 @@ split_by_length(text; separator=",", max_length=10000) # for 4K context window length(chunks[1]) # Output: 4 ``` """ -function split_by_length(text::String; separator::String = " ", max_length::Int = 35000) +function split_by_length(text::String; + separator::String = " ", + max_length::Int = 35000) + ## shortcut + length(text) <= max_length && return [text] + + ## split by separator minichunks = split(text, separator) sep_length = length(separator) chunks = String[] @@ -99,6 +105,66 @@ function split_by_length(text::String; separator::String = " ", max_length::Int return chunks end + +# Overload for dispatch on multiple separators +function split_by_length(text::String, + separator::String, + max_length::Int = 35000) + split_by_length(text; separator, max_length) +end + +""" + split_by_length(text::String, separators::Vector{String}; max_length::Int=35000) -> Vector{String} + +Split a given string `text` into chunks using a series of separators, with each chunk having a maximum length of `max_length`. +This function is useful for splitting large documents or texts into smaller segments that are more manageable for processing, particularly for models or systems with limited context windows. + +# Arguments +- `text::String`: The text to be split. +- `separators::Vector{String}`: An ordered list of separators used to split the text. The function iteratively applies these separators to split the text. +- `max_length::Int=35000`: The maximum length of each chunk. Defaults to 35,000 characters. This length is considered after each iteration of splitting, ensuring chunks fit within specified constraints. + +# Returns +`Vector{String}`: A vector of strings, where each string is a chunk of the original text that is smaller than or equal to `max_length`. + +# Notes + +- The function processes the text iteratively with each separator in the provided order. This ensures more nuanced splitting, especially in structured texts. +- Each chunk is as close to `max_length` as possible without exceeding it (unless we cannot split it any further) +- If the `text` is empty, the function returns an empty array. +- Separators are re-added to the text chunks after splitting, preserving the original structure of the text as closely as possible. Apply `strip` if you do not need them. + +# Examples + +Splitting text using multiple separators: +```julia +text = "Paragraph 1\n\nParagraph 2. Sentence 1. Sentence 2.\nParagraph 3" +separators = ["\n\n", ". ", "\n"] +chunks = split_by_length(text, separators, max_length=20) +``` + +Using a single separator: +```julia +text = "Hello,World," ^ 2900 # length 34900 characters +chunks = split_by_length(text, [","], max_length=10000) +``` +""" +function split_by_length(text, separators::Vector{String}; max_length) + @assert !isempty(separators) "`separators` can't be empty" + separator = popfirst!(separators) + chunks = split_by_length(text; separator, max_length) + + isempty(separators) && return chunks + ## Iteratively split by separators + for separator in separators + chunks = mapreduce(text_ -> split_by_length(text_; max_length, separator), + vcat, + chunks) + end + + return chunks +end + ### INTERNAL FUNCTIONS - DO NOT USE DIRECTLY # helper to extract handlebar variables (eg, `{{var}}`) from a prompt string function _extract_handlebar_variables(s::AbstractString) @@ -109,18 +175,63 @@ function _extract_handlebar_variables(vect::Vector{Dict{String, <:AbstractString unique([_extract_handlebar_variables(v) for d in vect for (k, v) in d if k == "text"]) end -# helper to produce summary message of how many tokens were used and for how much -function _report_stats(msg, - model::String, +""" + call_cost(msg, model::String; + cost_of_token_prompt::Number = default_prompt_cost, + cost_of_token_generation::Number = default_generation_cost) -> Number + +Calculate the cost of a call based on the number of tokens in the message and the cost per token. + +# Arguments +- `msg`: The message object, which should contain a `tokens` field + with two elements: [number_of_prompt_tokens, number_of_generation_tokens]. +- `model::String`: The name of the model to use for determining token costs. If the model + is not found in `MODEL_REGISTRY`, default costs are used. +- `cost_of_token_prompt::Number`: The cost per prompt token. Defaults to the cost in `MODEL_REGISTRY` + for the given model, or 0.0 if the model is not found. +- `cost_of_token_generation::Number`: The cost per generation token. Defaults to the cost in + `MODEL_REGISTRY` for the given model, or 0.0 if the model is not found. + +# Returns +- `Number`: The total cost of the call. + +# Examples +```julia +# Assuming MODEL_REGISTRY is set up with appropriate costs +MODEL_REGISTRY = Dict( + "model1" => (cost_of_token_prompt = 0.05, cost_of_token_generation = 0.10), + "model2" => (cost_of_token_prompt = 0.07, cost_of_token_generation = 0.02) +) + +msg1 = AIMessage([10, 20]) # 10 prompt tokens, 20 generation tokens +cost1 = call_cost(msg1, "model1") +# cost1 = 10 * 0.05 + 20 * 0.10 = 2.5 + +msg2 = DataMessage([15, 30]) # 15 prompt tokens, 30 generation tokens +cost2 = call_cost(msg2, "model2") +# cost2 = 15 * 0.07 + 30 * 0.02 = 1.35 + +# Using custom token costs +msg3 = AIMessage([5, 10]) +cost3 = call_cost(msg3, "model3", cost_of_token_prompt = 0.08, cost_of_token_generation = 0.12) +# cost3 = 5 * 0.08 + 10 * 0.12 = 1.6 +``` +""" +function call_cost(msg, model::String; cost_of_token_prompt::Number = get(MODEL_REGISTRY, model, (; cost_of_token_prompt = 0.0)).cost_of_token_prompt, cost_of_token_generation::Number = get(MODEL_REGISTRY, model, (; cost_of_token_generation = 0.0)).cost_of_token_generation) - cost = (msg.tokens[1] * cost_of_token_prompt + - msg.tokens[2] * cost_of_token_generation) + cost = msg.tokens[1] * cost_of_token_prompt + + msg.tokens[2] * cost_of_token_generation + return cost +end +# helper to produce summary message of how many tokens were used and for how much +function _report_stats(msg, + model::String) + cost = call_cost(msg, model) cost_str = iszero(cost) ? "" : " @ Cost: \$$(round(cost; digits=4))" - return "Tokens: $(sum(msg.tokens))$(cost_str) in $(round(msg.elapsed;digits=1)) seconds" end # Loads and encodes the provided image path as a base64 string diff --git a/templates/RAG/CreateQAFromContext.json b/templates/RAG/CreateQAFromContext.json new file mode 100644 index 000000000..83e900ba1 --- /dev/null +++ b/templates/RAG/CreateQAFromContext.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For RAG applications. Generate Question and Answer from the provided Context.If you don't have any special instructions, provide `instructions=\"None.\"`. Placeholders: `context`, `instructions`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You are a world-class teacher preparing contextual Question & Answer sets for evaluating AI systems.\"),\n\n**Instructions for Question Generation:**\n1. Analyze the provided Context chunk thoroughly.\n2. Formulate a question that:\n - Is specific and directly related to the information in the context chunk.\n - Is not too short or generic; it should require detailed understanding of the context to answer.\n - Can only be answered using the information from the provided context, without needing external information.\n\n**Instructions for Reference Answer Creation:**\n1. Based on the generated question, compose a reference answer that:\n - Directly and comprehensively answers the question.\n - Stays strictly within the bounds of the provided context chunk.\n - Is clear, concise, and to the point, avoiding unnecessary elaboration or repetition.\n\n**Example 1:**\n- Context Chunk: \"In 1928, Alexander Fleming discovered penicillin, which marked the beginning of modern antibiotics.\"\n- Generated Question: \"What was the significant discovery made by Alexander Fleming in 1928 and its impact?\"\n- Reference Answer: \"Alexander Fleming discovered penicillin in 1928, which led to the development of modern antibiotics.\"\n\nIf the user provides special instructions, prioritize these over the general instructions.\n","variables":[],"_type":"systemmessage"},{"content":"# Context Information\n---\n{{context}}\n---\n\n\n# Special Instructions\n\n{{instructions}}\n","variables":["context","instructions"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/RAG/RAGAnswerFromContext.json b/templates/RAG/RAGAnswerFromContext.json new file mode 100644 index 000000000..272ca4e20 --- /dev/null +++ b/templates/RAG/RAGAnswerFromContext.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For RAG applications. Answers the provided Questions based on the Context. Placeholders: `question`, `context`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"Act as a world-class AI assistant with access to the latest knowledge via Context Information. \n\n**Instructions:**\n- Answer the question based only on the provided Context.\n- If you don't know the answer, just say that you don't know, don't try to make up an answer.\n- Be brief and concise.\n\n**Context Information:**\n---\n{{context}}\n---\n","variables":["context"],"_type":"systemmessage"},{"content":"# Question\n\n{{question}}\n\n\n\n# Answer\n\n","variables":["question"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/RAG/RAGCreateQAFromContext.json b/templates/RAG/RAGCreateQAFromContext.json new file mode 100644 index 000000000..83e900ba1 --- /dev/null +++ b/templates/RAG/RAGCreateQAFromContext.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For RAG applications. Generate Question and Answer from the provided Context.If you don't have any special instructions, provide `instructions=\"None.\"`. Placeholders: `context`, `instructions`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You are a world-class teacher preparing contextual Question & Answer sets for evaluating AI systems.\"),\n\n**Instructions for Question Generation:**\n1. Analyze the provided Context chunk thoroughly.\n2. Formulate a question that:\n - Is specific and directly related to the information in the context chunk.\n - Is not too short or generic; it should require detailed understanding of the context to answer.\n - Can only be answered using the information from the provided context, without needing external information.\n\n**Instructions for Reference Answer Creation:**\n1. Based on the generated question, compose a reference answer that:\n - Directly and comprehensively answers the question.\n - Stays strictly within the bounds of the provided context chunk.\n - Is clear, concise, and to the point, avoiding unnecessary elaboration or repetition.\n\n**Example 1:**\n- Context Chunk: \"In 1928, Alexander Fleming discovered penicillin, which marked the beginning of modern antibiotics.\"\n- Generated Question: \"What was the significant discovery made by Alexander Fleming in 1928 and its impact?\"\n- Reference Answer: \"Alexander Fleming discovered penicillin in 1928, which led to the development of modern antibiotics.\"\n\nIf the user provides special instructions, prioritize these over the general instructions.\n","variables":[],"_type":"systemmessage"},{"content":"# Context Information\n---\n{{context}}\n---\n\n\n# Special Instructions\n\n{{instructions}}\n","variables":["context","instructions"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/RAG/RAGExtractMetadataLong.json b/templates/RAG/RAGExtractMetadataLong.json new file mode 100644 index 000000000..9ede8c3ca --- /dev/null +++ b/templates/RAG/RAGExtractMetadataLong.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For RAG applications. Extracts metadata from the provided text using longer instructions set and examples. If you don't have any special instructions, provide `instructions=\"None.\"`. Placeholders: `text`, `instructions`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You're a world-class data extraction engine built by OpenAI together with Google and to extract filter metadata to power the most advanced search engine in the world. \n \n **Instructions for Extraction:**\n 1. Carefully read through the provided Text\n 2. Identify and extract:\n - All relevant entities such as names, places, dates, etc.\n - Any special items like technical terms, unique identifiers, etc.\n - In the case of Julia code or Julia documentation: specifically extract package names, struct names, function names, and important variable names (eg, uppercased variables)\n 3. Keep extracted values and categories short. Maximum 2-3 words!\n 4. You can only extract 3-5 items per Text, so select the most important ones.\n 5. Assign search filter Category to each extracted Value\n \n **Example 1:**\n - Document Chunk: \"Dr. Jane Smith published her findings on neuroplasticity in 2021. The research heavily utilized the DataFrames.jl and Plots.jl packages.\"\n - Extracted keywords:\n - Name: Dr. Jane Smith\n - Date: 2021\n - Technical Term: neuroplasticity\n - JuliaPackage: DataFrames.jl, Plots.jl\n - JuliaLanguage:\n - Identifier:\n - Other: \n\n If the user provides special instructions, prioritize these over the general instructions.\n","variables":[],"_type":"systemmessage"},{"content":"# Text\n\n{{text}}\n\n\n\n# Special Instructions\n\n{{instructions}}","variables":["text","instructions"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/RAG/RAGExtractMetadataShort.json b/templates/RAG/RAGExtractMetadataShort.json new file mode 100644 index 000000000..88132e929 --- /dev/null +++ b/templates/RAG/RAGExtractMetadataShort.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For RAG applications. Extracts metadata from the provided text. If you don't have any special instructions, provide `instructions=\"None.\"`. Placeholders: `text`, `instructions`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"Extract search keywords and their categories from the Text provided below (format \"value:category\"). Each keyword must be at most 2-3 words. Provide at most 3-5 keywords. I will tip you $50 if the search is successful.","variables":[],"_type":"systemmessage"},{"content":"# Text\n\n{{text}}\n\n\n\n# Special Instructions\n\n{{instructions}}","variables":["text","instructions"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/RAG/RAGJudgeAnswerFromContext.json b/templates/RAG/RAGJudgeAnswerFromContext.json new file mode 100644 index 000000000..e988d8129 --- /dev/null +++ b/templates/RAG/RAGJudgeAnswerFromContext.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For RAG applications. Judge answer to a question on a scale from 1-5. Placeholders: `question`, `context`, `answer`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You're an impartial judge. Your task is to evaluate the quality of the Answer provided by an AI assistant in response to the User Question on a scale 1-5.\n\n1. **Scoring Criteria:**\n- **Relevance (1-5):** How well does the provided answer align with the context? \n - *1: Not relevant, 5: Highly relevant*\n- **Completeness (1-5):** Does the provided answer cover all the essential points mentioned in the context?\n - *1: Very incomplete, 5: Very complete*\n- **Clarity (1-5):** How clear and understandable is the provided answer?\n - *1: Not clear at all, 5: Extremely clear*\n- **Consistency (1-5):** How consistent is the provided answer with the overall context?\n - *1: Highly inconsistent, 5: Perfectly consistent*\n- **Helpfulness (1-5):** How helpful is the provided answer in answering the user's question?\n - *1: Not helpful at all, 5: Extremely helpful*\n\n2. **Judging Instructions:**\n- As an impartial judge, please evaluate the provided answer based on the above criteria. \n- Assign a score from 1 to 5 for each criterion, considering the original context, question and the provided answer.\n- The Final Score is an average of these individual scores, representing the overall quality and relevance of the provided answer. It must be between 1-5.\n\n```\n","variables":[],"_type":"systemmessage"},{"content":"# User Question\n---\n{{question}}\n---\n\n\n# Context Information\n---\n{{context}}\n---\n\n\n# Assistant's Answer\n---\n{{answer}}\n---\n\n\n# Judge's Evaluation\n","variables":["question","context","answer"],"_type":"usermessage"}] \ No newline at end of file diff --git a/test/Experimental/RAGTools.jl/preparation.jl b/test/Experimental/RAGTools.jl/preparation.jl new file mode 100644 index 000000000..81da327c6 --- /dev/null +++ b/test/Experimental/RAGTools.jl/preparation.jl @@ -0,0 +1,68 @@ +@testset "metadata_extract" begin + # MetadataItem Structure + item = MetadataItem("value", "category") + @test item.value == "value" + @test item.category == "category" + + # MaybeMetadataItems Structure + items = MaybeMetadataItems([ + MetadataItem("value1", "category1"), + MetadataItem("value2", "category2"), + ]) + @test length(items.items) == 2 + @test items.items[1].value == "value1" + @test items.items[1].category == "category1" + + empty_items = MaybeMetadataItems(nothing) + @test isempty(metadata_extract(empty_items.items)) + + # Metadata Extraction Function + single_item = MetadataItem("DataFrames", "Julia Package") + multiple_items = [ + MetadataItem("pandas", "Software"), + MetadataItem("Python", "Language"), + MetadataItem("DataFrames", "Julia Package"), + ] + + @test metadata_extract(single_item) == "julia_package:::dataframes" + @test metadata_extract(multiple_items) == + ["software:::pandas", "language:::python", "julia_package:::dataframes"] + + @test metadata_extract(nothing) == String[] +end + +@testset "build_tags" begin + # Single Tag + chunk_metadata = [["tag1"]] + tags_, tags_vocab_ = build_tags(chunk_metadata) + + @test length(tags_vocab_) == 1 + @test tags_vocab_ == ["tag1"] + @test nnz(tags_) == 1 + @test tags_[1, 1] == true + + # Multiple Tags with Repetition + chunk_metadata = [["tag1", "tag2"], ["tag2", "tag3"]] + tags_, tags_vocab_ = build_tags(chunk_metadata) + + @test length(tags_vocab_) == 3 + @test tags_vocab_ == ["tag1", "tag2", "tag3"] + @test nnz(tags_) == 4 + @test all([tags_[1, 1], tags_[1, 2], tags_[2, 2], tags_[2, 3]]) + + # Empty Metadata + chunk_metadata = [String[]] + tags_, tags_vocab_ = build_tags(chunk_metadata) + + @test isempty(tags_vocab_) + @test size(tags_) == (1, 0) + + # Mixed Empty and Non-Empty Metadata + chunk_metadata = [["tag1"], String[], ["tag2", "tag3"]] + tags_, tags_vocab_ = build_tags(chunk_metadata) + + @test length(tags_vocab_) == 3 + @test tags_vocab_ == ["tag1", "tag2", "tag3"] + @test nnz(tags_) == 3 + @test all([tags_[1, 1], tags_[3, 2], tags_[3, 3]]) +end \ No newline at end of file diff --git a/test/Experimental/RAGTools.jl/retrieval.jl b/test/Experimental/RAGTools.jl/retrieval.jl new file mode 100644 index 000000000..cdff05b45 --- /dev/null +++ b/test/Experimental/RAGTools.jl/retrieval.jl @@ -0,0 +1,10 @@ +@testset "rerank" begin + # Mock data for testing + index = "mock_index" + question = "mock_question" + candidate_chunks = ["chunk1", "chunk2", "chunk3"] + + # Passthrough Strategy + strategy = Passthrough() + @test rerank(strategy, index, question, candidate_chunks) === candidate_chunks +end \ No newline at end of file diff --git a/test/Experimental/RAGTools.jl/runtests.jl b/test/Experimental/RAGTools.jl/runtests.jl new file mode 100644 index 000000000..6c70c014e --- /dev/null +++ b/test/Experimental/RAGTools.jl/runtests.jl @@ -0,0 +1,10 @@ +using Test +using SparseArrays, LinearAlgebra +using PromptingTools.Experimental.RAGTools + +include("utils.jl") +include("types.jl") +include("preparation.jl") +include("retrieval.jl") +# include("generation.jl") +# include("evaluation.jl") \ No newline at end of file diff --git a/test/Experimental/RAGTools.jl/types.jl b/test/Experimental/RAGTools.jl/types.jl new file mode 100644 index 000000000..61bd47ae3 --- /dev/null +++ b/test/Experimental/RAGTools.jl/types.jl @@ -0,0 +1,121 @@ + +@testset "merge_labeled_matrices" begin + # Test with dense matrices and overlapping vocabulary + mat1 = [1 2; 3 4] + vocab1 = ["word1", "word2"] + mat2 = [5 6; 7 8] + vocab2 = ["word2", "word3"] + + merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) + + @test size(merged_mat) == (4, 3) + @test combined_vocab == ["word1", "word2", "word3"] + @test merged_mat == [1 2 0; 3 4 0; 0 5 6; 0 7 8] + + # Test with sparse matrices and disjoint vocabulary + mat1 = sparse([1 0; 0 2]) + vocab1 = ["word1", "word2"] + mat2 = sparse([3 0; 0 4]) + vocab2 = ["word3", "word4"] + + merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) + + @test size(merged_mat) == (4, 4) + @test combined_vocab == ["word1", "word2", "word3", "word4"] + @test merged_mat == sparse([1 0 0 0; 0 2 0 0; 0 0 3 0; 0 0 0 4]) + + # Test with different data types + mat1 = [1.0 2.0; 3.0 4.0] + vocab1 = ["word1", "word2"] + mat2 = [5 6; 7 8] + vocab2 = ["word2", "word3"] + + merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) + + @test eltype(merged_mat) == Float64 + @test size(merged_mat) == (4, 3) + @test combined_vocab == ["word1", "word2", "word3"] + @test merged_mat ≈ [1.0 2.0 0.0; 3.0 4.0 0.0; 0.0 5.0 6.0; 0.0 7.0 8.0] +end + +@testset "ChunkIndex and MultiIndex getindex Tests" begin + @testset "ChunkIndex getindex" begin + ci = ChunkIndex(:index1, ["chunk1", "chunk2", "chunk3"]) + candidate = CandidateChunks(:index1, [1, 3]) + + @test getindex(ci, candidate) == ["chunk1", "chunk3"] + @test getindex(ci, candidate, :chunks) == ["chunk1", "chunk3"] + @test_throws AssertionError getindex(ci, candidate, :unsupported_field) + + # Test with non-matching index_id + candidate_wrong_id = CandidateChunks(:index2, [1, 3]) + @test getindex(ci, candidate_wrong_id) == String[] + end + + @testset "MultiIndex getindex" begin + ci1 = ChunkIndex(:index1, ["chunk1", "chunk2"]) + ci2 = ChunkIndex(:index2, ["chunk3", "chunk4"]) + mi = MultiIndex([ci1, ci2]) + candidate = CandidateChunks(:index2, [2]) + + @test getindex(mi, candidate) == ["chunk4"] + @test getindex(mi, candidate, :chunks) == ["chunk4"] + @test_throws AssertionError getindex(mi, candidate, :unsupported_field) + + # Test with non-existing index_id + candidate_non_existing = CandidateChunks(:index3, [1]) + @test getindex(mi, candidate_non_existing) == String[] + end +end + +@testset "MultiIndex Equality Tests" begin + index1 = ChunkIndex(:A) + index2 = ChunkIndex(:B) + index3 = ChunkIndex(:C) + + mi1 = MultiIndex([index1, index2]) + mi2 = MultiIndex([index1, index2]) + mi3 = MultiIndex([index2, index3]) + mi4 = MultiIndex([index1, index2, index3]) + mi5 = MultiIndex([index2, index1]) + + @test mi1 == mi2 # Identical MultiIndexes + @test mi1 != mi3 # Different indexes + @test mi1 != mi4 # Different number of indexes + @test mi3 != mi4 # Different indexes and different lengths + @test mi1 == mi5 # Same indexes, different order +end + +@testset "CandidateChunks" begin + # Different Index IDs and Intersecting Positions + cc1 = CandidateChunks(index_id = :index1, + positions = [1, 2, 3], + distances = [0.1, 0.2, 0.3]) + cc2 = CandidateChunks(index_id = :index2, + positions = [2, 3, 4], + distances = [0.3, 0.2, 0.1]) + cc3 = CandidateChunks(index_id = :index1, + positions = [3, 4, 5], + distances = [0.3, 0.4, 0.5]) + + # Different index IDs + result_diff_id = cc1 & cc2 + @test result_diff_id.index_id == :index1 + @test isempty(result_diff_id.positions) + @test isempty(result_diff_id.distances) + + # Intersecting positions + result_intersect = cc1 & cc3 + @test result_intersect.index_id == :index1 + @test result_intersect.positions == [3] + @test result_intersect.distances ≈ [0.4] + + # Missing Distances + cc1 = CandidateChunks(index_id = :index1, positions = [1, 2], distances = Float32[]) + cc2 = CandidateChunks(index_id = :index1, positions = [2, 3], distances = [0.2, 0.3]) + + result = cc1 & cc2 + @test result.index_id == :index1 + @test result.positions == [2] + @test isempty(result.distances) +end diff --git a/test/Experimental/RAGTools.jl/utils.jl b/test/Experimental/RAGTools.jl/utils.jl new file mode 100644 index 000000000..e69de29bb diff --git a/test/runtests.jl b/test/runtests.jl index c4cc26288..e0776b243 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using PromptingTools using OpenAI, HTTP, JSON3 +using SparseArrays, LinearAlgebra using Test using Aqua const PT = PromptingTools diff --git a/test/utils.jl b/test/utils.jl index 1b726924a..ceabdd5e7 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,6 +1,7 @@ using PromptingTools: split_by_length, replace_words -using PromptingTools: _extract_handlebar_variables, _report_stats +using PromptingTools: _extract_handlebar_variables, call_cost, _report_stats using PromptingTools: _string_to_vector, _encode_local_image +using PromptingTools: DataMessage, AIMessage @testset "replace_words" begin words = ["Disney", "Snow White", "Mickey Mouse"] @@ -32,7 +33,7 @@ end # Test with empty text chunks = split_by_length("") - @test isempty(chunks) + @test chunks == [""] # Test custom separator text = "Hello,World,"^50 @@ -43,6 +44,34 @@ end @test length(chunks) == 34 @test maximum(length.(chunks)) <= 20 @test join(chunks, "") == text + + ### Multiple separators + # Single separator + text = "First sentence. Second sentence. Third sentence." + chunks = split_by_length(text, ["."], max_length = 15) + @test length(chunks) == 3 + @test chunks == ["First sentence.", " Second sentence.", " Third sentence."] + + # Multiple separators + text = "Paragraph 1\n\nParagraph 2. Sentence 1. Sentence 2.\nParagraph 3" + separators = ["\n\n", ". ", "\n"] + chunks = split_by_length(text, separators, max_length = 20) + @test length(chunks) == 5 + @test chunks[1] == "Paragraph 1\n\n" + @test chunks[2] == "Paragraph 2. " + @test chunks[3] == "Sentence 1. " + @test chunks[4] == "Sentence 2.\n" + @test chunks[5] == "Paragraph 3" + + # empty separators + text = "Some text without separators." + @test_throws AssertionError split_by_length(text, String[], max_length = 10) + # edge cases + text = "Short text" + separators = ["\n\n", ". ", "\n"] + chunks = split_by_length(text, separators, max_length = 50) + @test length(chunks) == 1 + @test chunks[1] == text end @testset "extract_handlebar_variables" begin @@ -68,20 +97,34 @@ end @test actual_output == expected_output end +@testset "call_cost" begin + msg = AIMessage(; content = "", tokens = (1000, 2000)) + cost = call_cost(msg, "unknown_model") + @test cost == 0.0 + @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 1.5e-6 + 2e-6 * 2000 + + msg = DataMessage(; content = nothing, tokens = (1000, 1000)) + cost = call_cost(msg, "unknown_model") + @test cost == 0.0 + @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 1.5e-6 + 2e-6 * 1000 + + @test call_cost(msg, + "gpt-3.5-turbo"; + cost_of_token_prompt = 1, + cost_of_token_generation = 1) ≈ 1000 + 1000 +end + @testset "report_stats" begin # Returns a string with the total number of tokens and elapsed time when given a message and model msg = AIMessage(; content = "", tokens = (1, 5), elapsed = 5.0) - model = "model" + model = "unknown_model" expected_output = "Tokens: 6 in 5.0 seconds" @test _report_stats(msg, model) == expected_output # Returns a string with a cost - expected_output = "Tokens: 6 @ Cost: \$0.007 in 5.0 seconds" - @test _report_stats(msg, model, 2e-3, 1e-3) == expected_output - - # Returns a string without cost when it's zero - expected_output = "Tokens: 6 in 5.0 seconds" - @test _report_stats(msg, model, 0, 0) == expected_output + msg = AIMessage(; content = "", tokens = (1000, 5000), elapsed = 5.0) + expected_output = "Tokens: 6000 @ Cost: \$0.0115 in 5.0 seconds" + @test _report_stats(msg, "gpt-3.5-turbo") == expected_output end @testset "_string_to_vector" begin From e77beba4144643aaff54ae12d51e1e6b2024e777 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 21 Dec 2023 21:03:58 +0000 Subject: [PATCH 069/251] add tests --- test/runtests.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index e0776b243..94707299d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,4 +34,9 @@ let cb = AICode(; code = """ @test !isnothing(cb.expression) # parsed @test occursin("Test Failed", cb.stdout) # capture details of the test failure @test isnothing(cb.output) # because it failed -end \ No newline at end of file +end + +## Run experimental +@testset "Experimental" begin + include("Experimental/RAGTools.jl/runtests.jl") +end From 0116ab32009314971ab1497fdd040acd985ddb00 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 21 Dec 2023 21:05:16 +0000 Subject: [PATCH 070/251] update path --- test/Experimental/{RAGTools.jl => RAGTools}/preparation.jl | 0 test/Experimental/{RAGTools.jl => RAGTools}/retrieval.jl | 0 test/Experimental/{RAGTools.jl => RAGTools}/runtests.jl | 0 test/Experimental/{RAGTools.jl => RAGTools}/types.jl | 0 test/Experimental/{RAGTools.jl => RAGTools}/utils.jl | 0 test/runtests.jl | 2 +- 6 files changed, 1 insertion(+), 1 deletion(-) rename test/Experimental/{RAGTools.jl => RAGTools}/preparation.jl (100%) rename test/Experimental/{RAGTools.jl => RAGTools}/retrieval.jl (100%) rename test/Experimental/{RAGTools.jl => RAGTools}/runtests.jl (100%) rename test/Experimental/{RAGTools.jl => RAGTools}/types.jl (100%) rename test/Experimental/{RAGTools.jl => RAGTools}/utils.jl (100%) diff --git a/test/Experimental/RAGTools.jl/preparation.jl b/test/Experimental/RAGTools/preparation.jl similarity index 100% rename from test/Experimental/RAGTools.jl/preparation.jl rename to test/Experimental/RAGTools/preparation.jl diff --git a/test/Experimental/RAGTools.jl/retrieval.jl b/test/Experimental/RAGTools/retrieval.jl similarity index 100% rename from test/Experimental/RAGTools.jl/retrieval.jl rename to test/Experimental/RAGTools/retrieval.jl diff --git a/test/Experimental/RAGTools.jl/runtests.jl b/test/Experimental/RAGTools/runtests.jl similarity index 100% rename from test/Experimental/RAGTools.jl/runtests.jl rename to test/Experimental/RAGTools/runtests.jl diff --git a/test/Experimental/RAGTools.jl/types.jl b/test/Experimental/RAGTools/types.jl similarity index 100% rename from test/Experimental/RAGTools.jl/types.jl rename to test/Experimental/RAGTools/types.jl diff --git a/test/Experimental/RAGTools.jl/utils.jl b/test/Experimental/RAGTools/utils.jl similarity index 100% rename from test/Experimental/RAGTools.jl/utils.jl rename to test/Experimental/RAGTools/utils.jl diff --git a/test/runtests.jl b/test/runtests.jl index 94707299d..d8bc24dad 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -38,5 +38,5 @@ end ## Run experimental @testset "Experimental" begin - include("Experimental/RAGTools.jl/runtests.jl") + include("Experimental/RAGTools/runtests.jl") end From 57b8b80f613a23ccf497ebbbab5eea8e83eaa6b8 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 22 Dec 2023 20:11:55 +0100 Subject: [PATCH 071/251] update tests for RAGTools --- Project.toml | 12 +- src/Experimental/RAGTools/evaluation.jl | 72 ++++- src/Experimental/RAGTools/generation.jl | 53 ++-- src/Experimental/RAGTools/preparation.jl | 25 +- src/Experimental/RAGTools/retrieval.jl | 3 +- src/Experimental/RAGTools/types.jl | 17 +- src/Experimental/RAGTools/utils.jl | 6 + src/llm_interface.jl | 8 +- .../RAG/RAGJudgeAnswerFromContextShort.json | 1 + test/Experimental/RAGTools/evaluation.jl | 25 ++ test/Experimental/RAGTools/generation.jl | 88 ++++++ test/Experimental/RAGTools/preparation.jl | 60 ++++ test/Experimental/RAGTools/retrieval.jl | 49 ++++ test/Experimental/RAGTools/runtests.jl | 15 +- test/Experimental/RAGTools/types.jl | 267 ++++++++++-------- test/Experimental/RAGTools/utils.jl | 46 +++ test/llm_openai.jl | 4 +- 17 files changed, 575 insertions(+), 176 deletions(-) create mode 100644 templates/RAG/RAGJudgeAnswerFromContextShort.json create mode 100644 test/Experimental/RAGTools/evaluation.jl create mode 100644 test/Experimental/RAGTools/generation.jl diff --git a/Project.toml b/Project.toml index 66f9ada1d..dd79425ea 100644 --- a/Project.toml +++ b/Project.toml @@ -13,19 +13,19 @@ PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" [weakdeps] -SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] -RAGToolsExperimentalExt = ["SparseArrays","LinearAlgebra"] +RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] [compat] Aqua = "0.7" Base64 = "<0.0.1, 1" HTTP = "1" JSON3 = "1" -Logging = "<0.0.1, 1" LinearAlgebra = "<0.0.1, 1" +Logging = "<0.0.1, 1" OpenAI = "0.8.7" PrecompileTools = "1" Preferences = "1" @@ -35,9 +35,9 @@ julia = "1.9,1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Test", "SparseArrays","LinearAlgebra"] +test = ["Aqua", "Test", "SparseArrays", "LinearAlgebra"] diff --git a/src/Experimental/RAGTools/evaluation.jl b/src/Experimental/RAGTools/evaluation.jl index 5ded7896d..010799142 100644 --- a/src/Experimental/RAGTools/evaluation.jl +++ b/src/Experimental/RAGTools/evaluation.jl @@ -1,8 +1,8 @@ ### For testing and eval # This is a return_type for extraction when generating Q&A set with aiextract @kwdef struct QAItem - question::String - answer::String + question::String = "" + answer::String = "" end # This is for saving in JSON format for evaluation later @kwdef struct QAEvalItem @@ -12,12 +12,24 @@ end answer::String = "" end +@kwdef struct QAEvalResult + source::AbstractString + context::AbstractString + question::AbstractString + answer::AbstractString + retrieval_score::Union{Number, Nothing} = nothing + retrieval_rank::Union{Int, Nothing} = nothing + answer_score::Union{Number, Nothing} = nothing + parameters::AbstractDict +end + "Provide the `final_rating` between 1-5. Provide the rationale for it." @kwdef struct JudgeRating rationale::Union{Nothing, String} = nothing final_rating::Int end -"Explain the `final_rating` in `rationale`" + +"`final_rating` is the average of all scoring criteria. Explain the `final_rating` in `rationale`" @kwdef struct JudgeAllScores relevance::Int completeness::Int @@ -33,7 +45,7 @@ function Base.isvalid(x::QAEvalItem) end # Nicer show method with some colors! -function Base.show(io::IO, t::Union{QAItem, QAEvalItem}) +function Base.show(io::IO, t::Union{QAItem, QAEvalItem, QAEvalResult}) printstyled(io, "$(nameof(typeof(t))):\n", color = :green, bold = true) for f in fieldnames(typeof(t)) printstyled(io, " ", f, color = :blue, bold = true) @@ -42,6 +54,7 @@ function Base.show(io::IO, t::Union{QAItem, QAEvalItem}) end # Define how JSON3 should serialize/deserialize the struct into JSON files JSON3.StructTypes.StructType(::Type{QAEvalItem}) = JSON3.StructTypes.Struct() +JSON3.StructTypes.StructType(::Type{QAEvalResult}) = JSON3.StructTypes.Struct() """ build_qa_evals(doc_chunks::Vector{<:AbstractString}, sources::Vector{<:AbstractString}; @@ -104,3 +117,54 @@ function build_qa_evals(doc_chunks::Vector{<:AbstractString}, verbose && @info "Q&A Sets built! (cost: \$$(round(cost_tracker[], digits=3)))" return filter(isvalid, output) end + +"Returns 1.0 if `context` overlaps or is contained within any of the `candidate_context`" +function score_retrieval_hit(orig_context::AbstractString, + candidate_context::Vector{<:AbstractString}) + 1.0 * (any(occursin.(Ref(orig_context), candidate_context)) || + any(occursin.(candidate_context, Ref(orig_context)))) +end + +"Returns Integer rank of the position where `context` overlaps or is contained within a `candidate_context`" +function score_retrieval_rank(orig_context::AbstractString, + candidate_context::Vector{<:AbstractString}) + findfirst((occursin.(Ref(orig_context), candidate_context)) .|| + (occursin.(candidate_context, Ref(orig_context)))) +end + +"Single QAEvalItem evalution" +function run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; + verbose::Bool = true, parameters_dict::AbstractDict, + judge_template::Symbol = :RAGJudgeAnswerFromContext, + model_judge::AbstractString) + retrieval_score = score_retrieval_hit(qa_item.context, ctx.context) + retrieval_rank = score_retrieval_rank(qa_item.context, ctx.context) + + answer_score = try + msg = aiextract(judge_template; model = model_judge, verbose, + ctx.context, + question, + msg.content, + return_type = RAG.JudgeAllScores) + final_rating = if msg.content isa AbstractDict && haskey(msg.content, :final_rating) + # if return type parsing failed + msg.content[:final_rating] + else + # if return_type worked + msg.content.final_rating + end + catch e + verbose && @warn "Error in QA eval ($(qa_item.question)): $e" + nothing + end + + return QAEvalResult(; + ctx.source, + qa_item.context, + qa_item.question, + ctx.answer, + retrieval_score, + retrieval_rank, + answer_score, + parameters = parameters_dict) +end diff --git a/src/Experimental/RAGTools/generation.jl b/src/Experimental/RAGTools/generation.jl index 804e0fdd0..733fe5e3c 100644 --- a/src/Experimental/RAGTools/generation.jl +++ b/src/Experimental/RAGTools/generation.jl @@ -1,12 +1,19 @@ -# stub to be replaced with extension +# stub to be replaced within the package extension function _normalize end """ - airag(index::AbstractChunkIndex, rag_template::Symbol=:RAGAnswerFromContext; - question::AbstractString, top_k::Int=3, tag_filter::Union{Symbol,Vector{String},Regex}=:auto, - rerank_strategy::RerankingStrategy=Passthrough(), model_embedding::String=PT.MODEL_EMBEDDING, - model_chat::String=PT.MODEL_CHAT, model_metadata::String=PT.MODEL_CHAT, - chunks_window_margin::Tuple{Int,Int}=(1, 1), return_context::Bool=false, verbose::Bool=true, kwargs...) -> Any + airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromContext; + question::AbstractString, + top_k::Int = 3, + tag_filter::Union{Symbol, Vector{String}, Regex, Nothing} = :auto, + rerank_strategy::RerankingStrategy = Passthrough(), + model_embedding::String = PT.MODEL_EMBEDDING, model_chat::String = PT.MODEL_CHAT, + model_metadata::String = PT.MODEL_CHAT, + metadata_template::Symbol = :RAGExtractMetadataShort, + chunks_window_margin::Tuple{Int, Int} = (1, 1), + return_context::Bool = false, verbose::Bool = true, + api_kwargs::NamedTuple = NamedTuple(), + kwargs...) Generates a response for a given question using a Retrieval-Augmented Generation (RAG) approach. @@ -17,14 +24,16 @@ The function selects relevant chunks from an `ChunkIndex`, optionally filters th - `rag_template::Symbol`: Template for the RAG model, defaults to `:RAGAnswerFromContext`. - `question::AbstractString`: The question to be answered. - `top_k::Int`: Number of top candidates to retrieve based on embedding similarity. -- `tag_filter::Union{Symbol, Vector{String}, Regex}`: Mechanism for filtering chunks based on tags (either automatically detected, specific tags, or a regex pattern). +- `tag_filter::Union{Symbol, Vector{String}, Regex}`: Mechanism for filtering chunks based on tags (either automatically detected, specific tags, or a regex pattern). Disabled by setting to `nothing`. - `rerank_strategy::RerankingStrategy`: Strategy for reranking the retrieved chunks. - `model_embedding::String`: Model used for embedding the question, default is `PT.MODEL_EMBEDDING`. - `model_chat::String`: Model used for generating the final response, default is `PT.MODEL_CHAT`. - `model_metadata::String`: Model used for extracting metadata, default is `PT.MODEL_CHAT`. +- `metadata_template::Symbol`: Template for the metadata extraction process from the question, defaults to: `:RAGExtractMetadataShort` - `chunks_window_margin::Tuple{Int,Int}`: The window size around each chunk to consider for context building. - `return_context::Bool`: If `true`, returns the context used for RAG along with the response. - `verbose::Bool`: If `true`, enables verbose logging. +- `api_kwargs`: API parameters that will be forwarded to the API calls # Returns - If `return_context` is `false`, returns the generated message (`msg`). @@ -51,16 +60,18 @@ msg = airag(index; question) function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromContext; question::AbstractString, top_k::Int = 3, - tag_filter::Union{Symbol, Vector{String}, Regex} = :auto, + tag_filter::Union{Symbol, Vector{String}, Regex, Nothing} = :auto, rerank_strategy::RerankingStrategy = Passthrough(), model_embedding::String = PT.MODEL_EMBEDDING, model_chat::String = PT.MODEL_CHAT, model_metadata::String = PT.MODEL_CHAT, + metadata_template::Symbol = :RAGExtractMetadataShort, chunks_window_margin::Tuple{Int, Int} = (1, 1), return_context::Bool = false, verbose::Bool = true, + api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## Note: Supports only single ChunkIndex for now ## Checks - @assert tag_filter isa Symbol&&tag_filter == :auto "Only `:auto`, `Vector{String}`, or `Regex` are supported for `tag_filter`" + @assert !(tag_filter isa Symbol && tag_filter != :auto) "Only `:auto`, `Vector{String}`, or `Regex` are supported for `tag_filter`" @assert chunks_window_margin[1] >= 0&&chunks_window_margin[2] >= 0 "Both `chunks_window_margin` values must be non-negative" placeholders = only(aitemplates(rag_template)).variables # only one template should be found @assert (:question in placeholders)&&(:context in placeholders) "Provided RAG Template $(rag_template) is not suitable. It must have placeholders: `question` and `context`." @@ -68,27 +79,30 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC question_emb = aiembed(question, _normalize; model = model_embedding, - verbose).content .|> Float32 + verbose, api_kwargs).content .|> Float32 # no need for Float64 emb_candidates = find_closest(index, question_emb; top_k) tag_candidates = if tag_filter == :auto && !isnothing(tags(index)) && !isempty(model_metadata) + _check_aiextract_capability(model_metadata) # extract metadata via LLM call - # Check that the provided model is known and that it is an OpenAI model (for the aiextract function to work) - @assert haskey(PT.MODEL_REGISTRY, - model_metadata)&&PT.MODEL_REGISTRY[model_metadata].schema == PT.OpenAISchema() "Only OpenAI models support the metadata extraction now. $model_metadata is not a registered OpenAI model." metadata_ = try msg = aiextract(metadata_template; return_type = MaybeMetadataItems, - text = chunk, + text = question, instructions = "In addition to extracted items, suggest 2-3 filter keywords that could be relevant to answer this question.", - verbose, model = model_metadata) - metadata_extract(msg.content.items) - catch + verbose, model = model_metadata, api_kwargs) + ## eg, ["software:::pandas", "language:::python", "julia_package:::dataframes"] + ## we split it and take only the keyword, not the category + metadata_extract(msg.content.items) |> + x -> split.(x, ":::") |> x -> getindex.(x, 2) + catch e String[] end find_tags(index, metadata_) - elseif !(tag_filter isa Symbol) + elseif tag_filter isa Union{Vector{String}, Regex} find_tags(index, tag_filter) + elseif isnothing(tag_filter) + nothing else ## not filtering -- use all rows and ignore this nothing @@ -106,16 +120,19 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC position + chunks_window_margin[2])] is_same_source = sources(index)[max(1, position - chunks_window_margin[1]):min(end, position + chunks_window_margin[2])] .== sources(index)[position] + # add the ranking number, eg, 1. Context #1 push!(context, "$(i). $(join(chunks_[is_same_source], "\n"))") end ## LLM call msg = aigenerate(rag_template; question, context = join(context, "\n\n"), model = model_chat, verbose, + api_kwargs, kwargs...) if return_context # for evaluation rag_context = RAGContext(; question, + answer = msg.content, context, emb_candidates, tag_candidates, diff --git a/src/Experimental/RAGTools/preparation.jl b/src/Experimental/RAGTools/preparation.jl index 3c06dc403..50e937805 100644 --- a/src/Experimental/RAGTools/preparation.jl +++ b/src/Experimental/RAGTools/preparation.jl @@ -36,9 +36,12 @@ function build_index end """ build_index(files::Vector{<:AbstractString}; - separators=["\n\n", ". ", "\n"], max_length::Int=256, - extract_metadata::Bool=false, verbose::Bool=true, metadata_template::Symbol=:RAGExtractMetadataShort, - model_embedding::String=PT.MODEL_EMBEDDING, model_metadata::String=PT.MODEL_CHAT) + separators = ["\n\n", ". ", "\n"], max_length::Int = 256, + extract_metadata::Bool = false, verbose::Bool = true, + metadata_template::Symbol = :RAGExtractMetadataShort, + model_embedding::String = PT.MODEL_EMBEDDING, + model_metadata::String = PT.MODEL_CHAT, + api_kwargs::NamedTuple = NamedTuple()) Build an index for RAG (Retriever-Augmented Generation) applications from the provided file paths. The function processes each file, splits its content into chunks, embeds these chunks, @@ -46,7 +49,7 @@ optionally extracts metadata, and then compiles this information into a retrieva # Arguments - `files`: A vector of valid file paths to be indexed. -- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `["\\n\\n", ". ", "\\n"]`. +- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `["\n\n", ". ", "\n"]`. - `max_length`: The maximum length of each chunk (if possible with provided separators). Default is 256. - `extract_metadata`: A boolean flag indicating whether to extract metadata from each chunk (to build filter `tags` in the index). Default is `false`. Metadata extraction incurs additional cost and requires `model_metadata` and `metadata_template` to be provided. @@ -54,6 +57,7 @@ optionally extracts metadata, and then compiles this information into a retrieva - `metadata_template`: A symbol indicating the template to be used for metadata extraction. Default is `:RAGExtractMetadataShort`. - `model_embedding`: The model to use for embedding. - `model_metadata`: The model to use for metadata extraction. +- `api_kwargs`: Parameters to be provided to the API endpoint. # Returns - `ChunkIndex`: An object containing the compiled index of chunks, embeddings, tags, vocabulary, and sources. @@ -77,7 +81,8 @@ function build_index(files::Vector{<:AbstractString}; extract_metadata::Bool = false, verbose::Bool = true, metadata_template::Symbol = :RAGExtractMetadataShort, model_embedding::String = PT.MODEL_EMBEDDING, - model_metadata::String = PT.MODEL_CHAT) + model_metadata::String = PT.MODEL_CHAT, + api_kwargs::NamedTuple = NamedTuple()) ## @assert all(isfile, files) "Some `files` don't exist (Check: $(join(filter(!isfile,files),", "))" @@ -101,14 +106,12 @@ function build_index(files::Vector{<:AbstractString}; # Notice that we embed all doc_chunks at once, not one by one # OpenAI supports embedding multiple documents to reduce the number of API calls/network latency time - emb = aiembed(doc_chunks, _normalize; model = model_embedding, verbose) + emb = aiembed(doc_chunks, _normalize; model = model_embedding, verbose, api_kwargs) Threads.atomic_add!(cost_tracker, PT.call_cost(emb, model_embedding)) # track costs push!(output_embeddings, Float32.(emb.content)) if extract_metadata && !isempty(model_metadata) - # Check that the provided model is known and that it is an OpenAI model (for the aiextract function to work) - @assert haskey(PT.MODEL_REGISTRY, - model_metadata)&&PT.MODEL_REGISTRY[model_metadata].schema == PT.OpenAISchema() "Only OpenAI models support the metadata extraction now. $model_metadata is not a registered OpenAI model." + _check_aiextract_capability(model_metadata) metadata_ = asyncmap(doc_chunks) do chunk try msg = aiextract(metadata_template; @@ -116,7 +119,7 @@ function build_index(files::Vector{<:AbstractString}; text = chunk, instructions = "None.", verbose, - model = model_metadata) + model = model_metadata, api_kwargs) Threads.atomic_add!(cost_tracker, PT.call_cost(msg, model_metadata)) # track costs items = metadata_extract(msg.content.items) catch @@ -129,7 +132,7 @@ function build_index(files::Vector{<:AbstractString}; ## Create metadata tags and associated vocabulary tags, tags_vocab = if !isempty(output_metadata) # Requires SparseArrays.jl! - _build_tags(vcat(output_metadata...)) # need to vcat to be on the "chunk-level" + build_tags(vcat(output_metadata...)) # need to vcat to be on the "chunk-level" else tags, tags_vocab = nothing, nothing end diff --git a/src/Experimental/RAGTools/retrieval.jl b/src/Experimental/RAGTools/retrieval.jl index b824b5396..db33a90e0 100644 --- a/src/Experimental/RAGTools/retrieval.jl +++ b/src/Experimental/RAGTools/retrieval.jl @@ -30,7 +30,8 @@ function find_tags(index::AbstractChunkIndex, end function find_tags(index::AbstractChunkIndex, tags::Vector{<:AbstractString}) - pos = Int[find_tags(index, tag).positions for tag in tags] |> unique + pos = [find_tags(index, tag).positions for tag in tags] |> + Base.Splat(vcat) |> unique |> x -> convert(Vector{Int}, x) return CandidateChunks(index.id, pos, ones(Float32, length(pos))) end diff --git a/src/Experimental/RAGTools/types.jl b/src/Experimental/RAGTools/types.jl index c30f4408e..cd8ee1607 100644 --- a/src/Experimental/RAGTools/types.jl +++ b/src/Experimental/RAGTools/types.jl @@ -36,20 +36,20 @@ function Base.var"=="(i1::ChunkIndex, i2::ChunkIndex) end function Base.vcat(i1::ChunkIndex, i2::ChunkIndex) - tags, tags_vocab = if (isnothing(tags(i1)) || isnothing(tags(i2))) + tags_, tags_vocab_ = if (isnothing(tags(i1)) || isnothing(tags(i2))) nothing, nothing elseif tags_vocab(i1) == tags_vocab(i2) vcat(tags(i1), tags(i2)), tags_vocab(i1) else merge_labeled_matrices(tags(i1), tags_vocab(i1), tags(i2), tags_vocab(i2)) end - embeddings = (isnothing(embeddings(i1)) || isnothing(embeddings(i2))) ? nothing : - hcat(embeddings(i1), embeddings(i2)) + embeddings_ = (isnothing(embeddings(i1)) || isnothing(embeddings(i2))) ? nothing : + hcat(embeddings(i1), embeddings(i2)) ChunkIndex(; chunks = vcat(chunks(i1), chunks(i2)), - embeddings, - tags, - tags_vocab, + embeddings = embeddings_, + tags = tags_, + tags_vocab = tags_vocab_, sources = vcat(i1.sources, i2.sources)) end @@ -95,6 +95,8 @@ function Base.var"&"(cc1::CandidateChunks, cc2::CandidateChunks) end function Base.getindex(ci::ChunkIndex, candidate::CandidateChunks, field::Symbol = :chunks) @assert field==:chunks "Only `chunks` field is supported for now" + len_ = length(chunks(ci)) + @assert all(1 .<= candidate.positions .<= len_) "Some positions are out of bounds" if ci.id == candidate.index_id chunks(ci)[candidate.positions] else @@ -114,10 +116,11 @@ end """ RAGContext -A struct for debugging RAG answers. It contains the question, context, and the candidate chunks at each step of the RAG pipeline. +A struct for debugging RAG answers. It contains the question, answer, context, and the candidate chunks at each step of the RAG pipeline. """ @kwdef struct RAGContext question::AbstractString + answer::AbstractString context::Vector{<:AbstractString} emb_candidates::CandidateChunks tag_candidates::Union{Nothing, CandidateChunks} diff --git a/src/Experimental/RAGTools/utils.jl b/src/Experimental/RAGTools/utils.jl index d28ceae62..f980a61e0 100644 --- a/src/Experimental/RAGTools/utils.jl +++ b/src/Experimental/RAGTools/utils.jl @@ -1,3 +1,9 @@ +# Utility to check model suitability +function _check_aiextract_capability(model::AbstractString) + # Check that the provided model is known and that it is an OpenAI model (for the aiextract function to work) + @assert haskey(PT.MODEL_REGISTRY, + model)&&PT.MODEL_REGISTRY[model].schema isa PT.AbstractOpenAISchema "Only OpenAI models support the metadata extraction now. $model is not a registered OpenAI model." +end # Utitity to be able to combine indices from different sources/documents easily function merge_labeled_matrices(mat1::AbstractMatrix{T1}, vocab1::Vector{String}, diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 3dc8d2f22..aead1cf77 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -139,19 +139,19 @@ end function aiembed(doc_or_docs, args...; model = MODEL_EMBEDDING, kwargs...) global MODEL_REGISTRY schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema - aiembed(schema, doc_or_docs, args...; kwargs...) + aiembed(schema, doc_or_docs, args...; model, kwargs...) end function aiclassify(prompt; model = MODEL_CHAT, kwargs...) global MODEL_REGISTRY schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema - aiclassify(schema, prompt; kwargs...) + aiclassify(schema, prompt; model, kwargs...) end function aiextract(prompt; model = MODEL_CHAT, kwargs...) global MODEL_REGISTRY schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema - aiextract(schema, prompt; kwargs...) + aiextract(schema, prompt; model, kwargs...) end function aiscan(prompt; model = MODEL_CHAT, kwargs...) schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema - aiscan(schema, prompt; kwargs...) + aiscan(schema, prompt; model, kwargs...) end \ No newline at end of file diff --git a/templates/RAG/RAGJudgeAnswerFromContextShort.json b/templates/RAG/RAGJudgeAnswerFromContextShort.json new file mode 100644 index 000000000..93ea6447f --- /dev/null +++ b/templates/RAG/RAGJudgeAnswerFromContextShort.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For RAG applications. Simple and short prompt to judge answer to a question on a scale from 1-5. Placeholders: `question`, `context`, `answer`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You re an impartial judge. \nRead carefully the provided question and the answer based on the context. \nProvide a rating on a scale 1-5 (1=worst quality, 5=best quality) that reflects how relevant, helpful, clear, and consistent with the provided context the answer was.\n```\n","variables":[],"_type":"systemmessage"},{"content":"# User Question\n---\n{{question}}\n---\n\n\n# Context Information\n---\n{{context}}\n---\n\n\n# Assistant's Answer\n---\n{{answer}}\n---\n\n\n# Judge's Evaluation\n","variables":["question","context","answer"],"_type":"usermessage"}] \ No newline at end of file diff --git a/test/Experimental/RAGTools/evaluation.jl b/test/Experimental/RAGTools/evaluation.jl new file mode 100644 index 000000000..1a8ef5203 --- /dev/null +++ b/test/Experimental/RAGTools/evaluation.jl @@ -0,0 +1,25 @@ +using PromptingTools.Experimental.RAGTools: QAEvalItem +using PromptingTools.Experimental.RAGTools: score_retrieval_hit, score_retrieval_rank + +@testset "QAEvalItem" begin + empty_qa = QAEvalItem() + @test !isvalid(empty_qa) + full_qa = QAEvalItem(; question = "a", answer = "b", context = "c") + @test isvalid(full_qa) +end + +@testset "score_retrieval_hit,score_retrieval_rank" begin + orig_context = "I am a horse." + candidate_context = ["Hello", "World", "I am a horse...."] + candidate_context2 = ["Hello", "I am a hors"] + candidate_context3 = ["Hello", "World", "I am X horse...."] + @test score_retrieval_hit(orig_context, candidate_context) == 1.0 + @test score_retrieval_hit(orig_context, candidate_context2) == 1.0 + @test score_retrieval_hit(orig_context, candidate_context[1:2]) == 0.0 + @test score_retrieval_hit(orig_context, candidate_context3) == 0.0 + + @test score_retrieval_rank(orig_context, candidate_context) == 3 + @test score_retrieval_rank(orig_context, candidate_context2) == 2 + @test score_retrieval_rank(orig_context, candidate_context[1:2]) == nothing + @test score_retrieval_rank(orig_context, candidate_context3) == nothing +end \ No newline at end of file diff --git a/test/Experimental/RAGTools/generation.jl b/test/Experimental/RAGTools/generation.jl new file mode 100644 index 000000000..bfacf4cb5 --- /dev/null +++ b/test/Experimental/RAGTools/generation.jl @@ -0,0 +1,88 @@ +using PromptingTools.Experimental.RAGTools: MaybeMetadataItems, MetadataItem +@testset "airag" begin + # test with a mock server + PORT = rand(1000:2000) + PT.register_model!(; name = "mock-emb", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-meta", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-gen", schema = PT.CustomOpenAISchema()) + + echo_server = HTTP.serve!(PORT; verbose = -1) do req + content = JSON3.read(req.body) + + if content[:model] == "mock-gen" + user_msg = last(content[:messages]) + response = Dict(:choices => [Dict(:message => user_msg)], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + elseif content[:model] == "mock-emb" + # for i in 1:length(content[:input]) + response = Dict(:data => [Dict(:embedding => ones(Float32, 128))], + :usage => Dict(:total_tokens => length(content[:input]), + :prompt_tokens => length(content[:input]), + :completion_tokens => 0)) + elseif content[:model] == "mock-meta" + user_msg = last(content[:messages]) + response = Dict(:choices => [ + Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(MaybeMetadataItems([ + MetadataItem("yes", "category"), + ]))))), + ], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + else + @info content + end + return HTTP.Response(200, JSON3.write(response)) + end + + ## Index + index = ChunkIndex(; + sources = [".", ".", "."], + chunks = ["a", "b", "c"], + embeddings = zeros(128, 3), + tags = vcat(trues(2, 2), falses(1, 2)), + tags_vocab = ["yes", "no"],) + ## Sub-calls + question_emb = aiembed(["x", "x"]; + model = "mock-emb", + api_kwargs = (; url = "http://localhost:$(PORT)")) + @test question_emb.content == ones(128) + metadata_msg = aiextract(:RAGExtractMetadataShort; return_type = MaybeMetadataItems, + text = "x", + model = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)")) + @test metadata_msg.content.items == [MetadataItem("yes", "category")] + answer_msg = aigenerate(:RAGAnswerFromContext; + question = "Time?", + context = "XYZ", + model = "mock-gen", api_kwargs = (; url = "http://localhost:$(PORT)")) + @test occursin("Time?", answer_msg.content) + ## E2E + msg = airag(index; question = "Time?", model_embedding = "mock-emb", + model_chat = "mock-gen", + model_metadata = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)"), + tag_filter = ["yes"], + return_context = false) + @test occursin("Time?", msg.content) + # different kwargs + msg, ctx = airag(index; question = "Time?", model_embedding = "mock-emb", + model_chat = "mock-gen", + model_metadata = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)"), + tag_filter = :auto, + extract_metadata = false, verbose = false, + return_context = true) + @test ctx.context == ["1. a\nb\nc", "2. a\nb"] + @test ctx.emb_candidates.positions == [3, 2, 1] + @test ctx.emb_candidates.distances == zeros(3) + @test ctx.tag_candidates.positions == [1, 2] + @test ctx.tag_candidates.distances == ones(2) + @test ctx.filtered_candidates.positions == [2, 1] #re-sort + @test ctx.filtered_candidates.distances == 0.5ones(2) + @test ctx.reranked_candidates.positions == [2, 1] # no change + @test ctx.reranked_candidates.distances == 0.5ones(2) # no change + # clean up + close(echo_server) +end diff --git a/test/Experimental/RAGTools/preparation.jl b/test/Experimental/RAGTools/preparation.jl index 81da327c6..3e8396fbc 100644 --- a/test/Experimental/RAGTools/preparation.jl +++ b/test/Experimental/RAGTools/preparation.jl @@ -1,3 +1,6 @@ +using PromptingTools.Experimental.RAGTools: metadata_extract, MetadataItem +using PromptingTools.Experimental.RAGTools: MaybeMetadataItems, build_tags, build_index + @testset "metadata_extract" begin # MetadataItem Structure item = MetadataItem("value", "category") @@ -65,4 +68,61 @@ end @test tags_vocab_ == ["tag1", "tag2", "tag3"] @test nnz(tags_) == 3 @test all([tags_[1, 1], tags_[3, 2], tags_[3, 3]]) +end + +@testset "build_index" begin + # test with a mock server + PORT = rand(1000:2000) + PT.register_model!(; name = "mock-emb", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-meta", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-get", schema = PT.CustomOpenAISchema()) + + echo_server = HTTP.serve!(PORT; verbose = -1) do req + content = JSON3.read(req.body) + + if content[:model] == "mock-gen" + user_msg = last(content[:messages]) + response = Dict(:choices => [Dict(:message => user_msg)], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + elseif content[:model] == "mock-emb" + response = Dict(:data => [Dict(:embedding => ones(Float32, 128)) + for i in 1:length(content[:input])], + :usage => Dict(:total_tokens => length(content[:input]), + :prompt_tokens => length(content[:input]), + :completion_tokens => 0)) + elseif content[:model] == "mock-meta" + user_msg = last(content[:messages]) + response = Dict(:choices => [ + Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(MaybeMetadataItems([ + MetadataItem("yes", "category"), + ]))))), + ], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + else + @info content + end + return HTTP.Response(200, JSON3.write(response)) + end + + text = "This is a long text that will be split into chunks.\n\n It will be split by the separator. And also by the separator '\n'." + tmp, _ = mktemp() + write(tmp, text) + mini_files = [tmp, tmp] + index = build_index(mini_files; max_length = 10, extract_metadata = true, + model_embedding = "mock-emb", + model_metadata = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)")) + @test index.embeddings == hcat(fill(normalize(ones(Float32, 128)), 8)...) + @test index.chunks[1:4] == index.chunks[5:8] + @test index.sources == fill(tmp, 8) + @test index.tags == ones(8, 1) + @test index.tags_vocab == ["category:::yes"] + + # clean up + close(echo_server) end \ No newline at end of file diff --git a/test/Experimental/RAGTools/retrieval.jl b/test/Experimental/RAGTools/retrieval.jl index cdff05b45..9f21b561e 100644 --- a/test/Experimental/RAGTools/retrieval.jl +++ b/test/Experimental/RAGTools/retrieval.jl @@ -1,3 +1,52 @@ +using PromptingTools.Experimental.RAGTools: find_closest, find_tags +using PromptingTools.Experimental.RAGTools: Passthrough, rerank + +@testset "find_closest" begin + test_embeddings = [1.0 2.0; 3.0 4.0; 5.0 6.0] |> + x -> mapreduce(normalize, hcat, eachcol(x)) + query_embedding = [0.1, 0.35, 0.5] |> normalize + positions, distances = find_closest(test_embeddings, query_embedding, top_k = 2) + # The query vector should be closer to the first embedding + @test positions == [1, 2] + @test isapprox(distances, [0.9975694083904584 + 0.9939123761133188], atol = 1e-3) + + # Test when top_k is more than available embeddings + positions, _ = find_closest(test_embeddings, query_embedding, top_k = 5) + @test length(positions) == size(test_embeddings, 2) + + # Test behavior with edge values (top_k == 0) + @test find_closest(test_embeddings, query_embedding, top_k = 0) == ([], []) +end + +@testset "find_tags" begin + test_embeddings = [1.0 2.0; 3.0 4.0; 5.0 6.0] |> + x -> mapreduce(normalize, hcat, eachcol(x)) + query_embedding = [0.1, 0.35, 0.5] |> normalize + test_tags_vocab = ["julia", "python", "jr"] + test_tags_matrix = sparse([1, 2], [1, 3], [true, true], 2, 3) + index = ChunkIndex(; + sources = [".", "."], + chunks = ["julia", "jr"], + embeddings = test_embeddings, + tags = test_tags_matrix, + tags_vocab = test_tags_vocab) + + # Test for finding the correct positions of a specific tag + @test find_tags(index, "julia").positions == [1] + @test find_tags(index, "julia").distances == [1.0] + + # Test for no tag found // not in vocab + @test find_tags(index, "python").positions |> isempty + @test find_tags(index, "java").positions |> isempty + + # Test with regex matching + @test find_tags(index, r"^j").positions == [1, 2] + + # Test with multiple tags in vocab + @test find_tags(index, ["python", "jr", "x"]).positions == [2] +end + @testset "rerank" begin # Mock data for testing index = "mock_index" diff --git a/test/Experimental/RAGTools/runtests.jl b/test/Experimental/RAGTools/runtests.jl index 6c70c014e..605ce5df5 100644 --- a/test/Experimental/RAGTools/runtests.jl +++ b/test/Experimental/RAGTools/runtests.jl @@ -1,10 +1,13 @@ using Test using SparseArrays, LinearAlgebra using PromptingTools.Experimental.RAGTools +using JSON3, HTTP -include("utils.jl") -include("types.jl") -include("preparation.jl") -include("retrieval.jl") -# include("generation.jl") -# include("evaluation.jl") \ No newline at end of file +@testset "RAGTools" begin + include("utils.jl") + include("types.jl") + include("preparation.jl") + include("retrieval.jl") + include("generation.jl") + include("evaluation.jl") +end diff --git a/test/Experimental/RAGTools/types.jl b/test/Experimental/RAGTools/types.jl index 61bd47ae3..bfb915919 100644 --- a/test/Experimental/RAGTools/types.jl +++ b/test/Experimental/RAGTools/types.jl @@ -1,121 +1,154 @@ - -@testset "merge_labeled_matrices" begin - # Test with dense matrices and overlapping vocabulary - mat1 = [1 2; 3 4] - vocab1 = ["word1", "word2"] - mat2 = [5 6; 7 8] - vocab2 = ["word2", "word3"] - - merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) - - @test size(merged_mat) == (4, 3) - @test combined_vocab == ["word1", "word2", "word3"] - @test merged_mat == [1 2 0; 3 4 0; 0 5 6; 0 7 8] - - # Test with sparse matrices and disjoint vocabulary - mat1 = sparse([1 0; 0 2]) - vocab1 = ["word1", "word2"] - mat2 = sparse([3 0; 0 4]) - vocab2 = ["word3", "word4"] - - merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) - - @test size(merged_mat) == (4, 4) - @test combined_vocab == ["word1", "word2", "word3", "word4"] - @test merged_mat == sparse([1 0 0 0; 0 2 0 0; 0 0 3 0; 0 0 0 4]) - - # Test with different data types - mat1 = [1.0 2.0; 3.0 4.0] - vocab1 = ["word1", "word2"] - mat2 = [5 6; 7 8] - vocab2 = ["word2", "word3"] - - merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) - - @test eltype(merged_mat) == Float64 - @test size(merged_mat) == (4, 3) - @test combined_vocab == ["word1", "word2", "word3"] - @test merged_mat ≈ [1.0 2.0 0.0; 3.0 4.0 0.0; 0.0 5.0 6.0; 0.0 7.0 8.0] +using PromptingTools.Experimental.RAGTools: ChunkIndex, MultiIndex, CandidateChunks +using PromptingTools.Experimental.RAGTools: embeddings, chunks, tags, tags_vocab, sources + +@testset "ChunkIndex" begin + # Test constructors and basic accessors + chunks_test = ["chunk1", "chunk2"] + emb_test = ones(2, 2) + tags_test = sparse([1, 2], [1, 2], [true, true], 2, 2) + tags_vocab_test = ["vocab1", "vocab2"] + sources_test = ["source1", "source2"] + ci = ChunkIndex(chunks = chunks_test, + embeddings = emb_test, + tags = tags_test, + tags_vocab = tags_vocab_test, + sources = sources_test) + + @test chunks(ci) == chunks_test + @test (embeddings(ci)) == emb_test + @test tags(ci) == tags_test + @test tags_vocab(ci) == tags_vocab_test + @test sources(ci) == sources_test + + # Test identity/equality + ci1 = ChunkIndex(chunks = ["chunk1", "chunk2"], sources = ["source1", "source2"]) + ci2 = ChunkIndex(chunks = ["chunk1", "chunk2"], sources = ["source1", "source2"]) + @test ci1 == ci2 + + # Test equality with different chunks and sources + ci2 = ChunkIndex(chunks = ["chunk3", "chunk4"], sources = ["source3", "source4"]) + @test ci1 != ci2 + + # Test hcat with ChunkIndex + # Setup two different ChunkIndex with different tags and then hcat them + chunks1 = ["chunk1", "chunk2"] + tags1 = sparse([1, 2], [1, 2], [true, true], 2, 3) + tags_vocab1 = ["vocab1", "vocab2", "vocab3"] + sources1 = ["source1", "source1"] + ci1 = ChunkIndex(chunks = chunks1, + tags = tags1, + tags_vocab = tags_vocab1, + sources = sources1) + + chunks2 = ["chunk3", "chunk4"] + tags2 = sparse([1, 2], [1, 3], [true, true], 2, 3) + tags_vocab2 = ["vocab1", "vocab3", "vocab4"] + sources2 = ["source2", "source2"] + ci2 = ChunkIndex(chunks = chunks2, + tags = tags2, + tags_vocab = tags_vocab2, + sources = sources2) + + combined_ci = vcat(ci1, ci2) + @test size(tags(combined_ci), 1) == 4 + @test size(tags(combined_ci), 2) == 4 + @test length(unique(vcat(tags_vocab(ci1), tags_vocab(ci2)))) == + length(tags_vocab(combined_ci)) + @test sources(combined_ci) == vcat(sources(ci1), (sources(ci2))) + + # Test base var"==" with ChunkIndex + ci1 = ChunkIndex(chunks = ["chunk1"], + tags = trues(3, 1), + tags_vocab = ["vocab1"], + sources = ["source1"]) + ci2 = ChunkIndex(chunks = ["chunk1"], + tags = trues(3, 1), + tags_vocab = ["vocab1"], + sources = ["source1"]) + @test ci1 == ci2 end -@testset "ChunkIndex and MultiIndex getindex Tests" begin - @testset "ChunkIndex getindex" begin - ci = ChunkIndex(:index1, ["chunk1", "chunk2", "chunk3"]) - candidate = CandidateChunks(:index1, [1, 3]) - - @test getindex(ci, candidate) == ["chunk1", "chunk3"] - @test getindex(ci, candidate, :chunks) == ["chunk1", "chunk3"] - @test_throws AssertionError getindex(ci, candidate, :unsupported_field) - - # Test with non-matching index_id - candidate_wrong_id = CandidateChunks(:index2, [1, 3]) - @test getindex(ci, candidate_wrong_id) == String[] - end - - @testset "MultiIndex getindex" begin - ci1 = ChunkIndex(:index1, ["chunk1", "chunk2"]) - ci2 = ChunkIndex(:index2, ["chunk3", "chunk4"]) - mi = MultiIndex([ci1, ci2]) - candidate = CandidateChunks(:index2, [2]) - - @test getindex(mi, candidate) == ["chunk4"] - @test getindex(mi, candidate, :chunks) == ["chunk4"] - @test_throws AssertionError getindex(mi, candidate, :unsupported_field) - - # Test with non-existing index_id - candidate_non_existing = CandidateChunks(:index3, [1]) - @test getindex(mi, candidate_non_existing) == String[] - end +@testset "MultiIndex" begin + # Test constructors/accessors + # MultiIndex behaves as a container for ChunkIndexes + cin1 = ChunkIndex(chunks = ["chunk1"], sources = ["source1"]) + cin2 = ChunkIndex(chunks = ["chunk2"], sources = ["source2"]) + multi_index = MultiIndex(indexes = [cin1, cin2]) + @test length(multi_index.indexes) == 2 + @test cin1 in multi_index.indexes + @test cin2 in multi_index.indexes + + # Test base var"==" with MultiIndex + # Case where MultiIndexes are equal + cin1 = ChunkIndex(chunks = ["chunk1"], sources = ["source1"]) + cin2 = ChunkIndex(chunks = ["chunk2"], sources = ["source2"]) + mi1 = MultiIndex(indexes = [cin1, cin2]) + mi2 = MultiIndex(indexes = [cin1, cin2]) + @test mi1 == mi2 + + # Test equality with different ChunkIndexes inside + cin1 = ChunkIndex(chunks = ["chunk1"], sources = ["source1"]) + cin2 = ChunkIndex(chunks = ["chunk2"], sources = ["source2"]) + mi1 = MultiIndex(indexes = [cin1]) + mi2 = MultiIndex(indexes = [cin2]) + @test mi1 != mi2 end -@testset "MultiIndex Equality Tests" begin - index1 = ChunkIndex(:A) - index2 = ChunkIndex(:B) - index3 = ChunkIndex(:C) - - mi1 = MultiIndex([index1, index2]) - mi2 = MultiIndex([index1, index2]) - mi3 = MultiIndex([index2, index3]) - mi4 = MultiIndex([index1, index2, index3]) - mi5 = MultiIndex([index2, index1]) - - @test mi1 == mi2 # Identical MultiIndexes - @test mi1 != mi3 # Different indexes - @test mi1 != mi4 # Different number of indexes - @test mi3 != mi4 # Different indexes and different lengths - @test mi1 == mi5 # Same indexes, different order -end - -@testset "CandidateChunks" begin - # Different Index IDs and Intersecting Positions - cc1 = CandidateChunks(index_id = :index1, - positions = [1, 2, 3], - distances = [0.1, 0.2, 0.3]) - cc2 = CandidateChunks(index_id = :index2, - positions = [2, 3, 4], - distances = [0.3, 0.2, 0.1]) - cc3 = CandidateChunks(index_id = :index1, - positions = [3, 4, 5], - distances = [0.3, 0.4, 0.5]) - - # Different index IDs - result_diff_id = cc1 & cc2 - @test result_diff_id.index_id == :index1 - @test isempty(result_diff_id.positions) - @test isempty(result_diff_id.distances) - - # Intersecting positions - result_intersect = cc1 & cc3 - @test result_intersect.index_id == :index1 - @test result_intersect.positions == [3] - @test result_intersect.distances ≈ [0.4] - - # Missing Distances - cc1 = CandidateChunks(index_id = :index1, positions = [1, 2], distances = Float32[]) - cc2 = CandidateChunks(index_id = :index1, positions = [2, 3], distances = [0.2, 0.3]) - - result = cc1 & cc2 - @test result.index_id == :index1 - @test result.positions == [2] - @test isempty(result.distances) -end +@testset "getindex with CandidateChunks" begin + # Initialize a ChunkIndex with test data + chunks_data = ["First chunk", "Second chunk", "Third chunk"] + embeddings_data = rand(3, 3) # Random matrix with 3 embeddings + tags_data = sparse(Bool[1 1; 0 1; 1 0]) # Some arbitrary sparse matrix representation + tags_vocab_data = ["tag1", "tag2"] + chunk_sym = Symbol("TestChunkIndex") + test_chunk_index = ChunkIndex(chunks = chunks_data, + embeddings = embeddings_data, + tags = tags_data, + tags_vocab = tags_vocab_data, + sources = repeat(["test_source"], 3), + id = chunk_sym) + + # Test to get chunks based on valid CandidateChunks + candidate_chunks = CandidateChunks(index_id = chunk_sym, + positions = [1, 3], + distances = [0.1, 0.2]) + @test collect(test_chunk_index[candidate_chunks]) == ["First chunk", "Third chunk"] + + # Test with empty positions, which should result in an empty array + candidate_chunks_empty = CandidateChunks(index_id = chunk_sym, + positions = Int[], + distances = Float32[]) + @test isempty(test_chunk_index[candidate_chunks_empty]) + + # Test with positions out of bounds, should handle gracefully without errors + candidate_chunks_oob = CandidateChunks(index_id = chunk_sym, + positions = [10, -1], + distances = [0.5, 0.6]) + @test_throws AssertionError test_chunk_index[candidate_chunks_oob] + + # Test with an incorrect index_id, which should also result in an empty array + wrong_sym = Symbol("InvalidIndex") + candidate_chunks_wrong_id = CandidateChunks(index_id = wrong_sym, + positions = [1, 2], + distances = [0.3, 0.4]) + @test isempty(test_chunk_index[candidate_chunks_wrong_id]) + + # Test when chunks are requested from a MultiIndex, only chunks from the corresponding ChunkIndex should be returned + another_chuck_index = ChunkIndex(chunks = chunks_data, + embeddings = nothing, + tags = nothing, + tags_vocab = nothing, + sources = repeat(["another_source"], 3), + id = Symbol("AnotherChunkIndex")) + test_multi_index = MultiIndex(indexes = [ + test_chunk_index, + another_chuck_index, + ]) + @test collect(test_multi_index[candidate_chunks]) == ["First chunk", "Third chunk"] + + # Test when wrong index_id is used with MultiIndex, resulting in an empty array + @test isempty(test_multi_index[candidate_chunks_wrong_id]) + + # Test error case when trying to use a non-chunks field, should assert error as only :chunks field is supported + @test_throws AssertionError test_chunk_index[candidate_chunks, :nonexistent_field] +end \ No newline at end of file diff --git a/test/Experimental/RAGTools/utils.jl b/test/Experimental/RAGTools/utils.jl index e69de29bb..cc93c31f9 100644 --- a/test/Experimental/RAGTools/utils.jl +++ b/test/Experimental/RAGTools/utils.jl @@ -0,0 +1,46 @@ +using PromptingTools.Experimental.RAGTools: _check_aiextract_capability, + merge_labeled_matrices + +@testset "_check_aiextract_capability" begin + @test _check_aiextract_capability("gpt-3.5-turbo") == nothing + @test_throws AssertionError _check_aiextract_capability("llama2") +end + +@testset "merge_labeled_matrices" begin + # Test with dense matrices and overlapping vocabulary + mat1 = [1 2; 3 4] + vocab1 = ["word1", "word2"] + mat2 = [5 6; 7 8] + vocab2 = ["word2", "word3"] + + merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) + + @test size(merged_mat) == (4, 3) + @test combined_vocab == ["word1", "word2", "word3"] + @test merged_mat == [1 2 0; 3 4 0; 0 5 6; 0 7 8] + + # Test with sparse matrices and disjoint vocabulary + mat1 = sparse([1 0; 0 2]) + vocab1 = ["word1", "word2"] + mat2 = sparse([3 0; 0 4]) + vocab2 = ["word3", "word4"] + + merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) + + @test size(merged_mat) == (4, 4) + @test combined_vocab == ["word1", "word2", "word3", "word4"] + @test merged_mat == sparse([1 0 0 0; 0 2 0 0; 0 0 3 0; 0 0 0 4]) + + # Test with different data types + mat1 = [1.0 2.0; 3.0 4.0] + vocab1 = ["word1", "word2"] + mat2 = [5 6; 7 8] + vocab2 = ["word2", "word3"] + + merged_mat, combined_vocab = merge_labeled_matrices(mat1, vocab1, mat2, vocab2) + + @test eltype(merged_mat) == Float64 + @test size(merged_mat) == (4, 3) + @test combined_vocab == ["word1", "word2", "word3"] + @test merged_mat ≈ [1.0 2.0 0.0; 3.0 4.0 0.0; 0.0 5.0 6.0; 0.0 7.0 8.0] +end \ No newline at end of file diff --git a/test/llm_openai.jl b/test/llm_openai.jl index 95364b847..cc45494d0 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -180,7 +180,7 @@ end @testset "OpenAI.create_chat" begin # Test CustomOpenAISchema() with a mock server PORT = rand(1000:2000) - echo_server = HTTP.serve!(PORT) do req + echo_server = HTTP.serve!(PORT, verbose = -1) do req content = JSON3.read(req.body) user_msg = last(content[:messages]) response = Dict(:choices => [Dict(:message => user_msg)], @@ -206,7 +206,7 @@ end @testset "OpenAI.create_embeddings" begin # Test CustomOpenAISchema() with a mock server PORT = rand(1000:2000) - echo_server = HTTP.serve!(PORT) do req + echo_server = HTTP.serve!(PORT, verbose = -1) do req content = JSON3.read(req.body) response = Dict(:data => [Dict(:embedding => ones(128))], :usage => Dict(:total_tokens => length(content[:input]), From bda0ce2651c6155de92310b32c602d6e19c6bde3 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 22 Dec 2023 20:57:50 +0100 Subject: [PATCH 072/251] update docs --- src/Experimental/RAGTools/RAGTools.jl | 2 +- src/Experimental/RAGTools/evaluation.jl | 51 +++++++++++++++++++++---- src/Experimental/RAGTools/generation.jl | 1 + src/Experimental/RAGTools/types.jl | 1 + 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/Experimental/RAGTools/RAGTools.jl b/src/Experimental/RAGTools/RAGTools.jl index 47d4113ce..76fda1a55 100644 --- a/src/Experimental/RAGTools/RAGTools.jl +++ b/src/Experimental/RAGTools/RAGTools.jl @@ -27,7 +27,7 @@ include("retrieval.jl") export airag include("generation.jl") -export build_qa_evals +export build_qa_evals, run_qa_evals include("evaluation.jl") end \ No newline at end of file diff --git a/src/Experimental/RAGTools/evaluation.jl b/src/Experimental/RAGTools/evaluation.jl index 010799142..67dae42c1 100644 --- a/src/Experimental/RAGTools/evaluation.jl +++ b/src/Experimental/RAGTools/evaluation.jl @@ -132,20 +132,57 @@ function score_retrieval_rank(orig_context::AbstractString, (occursin.(candidate_context, Ref(orig_context)))) end -"Single QAEvalItem evalution" +""" + run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; verbose::Bool = true, + parameters_dict::AbstractDict, judge_template::Symbol = :RAGJudgeAnswerFromContext, + model_judge::AbstractString) -> QAEvalResult + +Evaluates a single `QAEvalItem` using a RAG context (`RAGContext`) and returns a `QAEvalResult` structure. This function assesses the relevance and accuracy of the answers generated in a QA evaluation context. + +# Arguments +- `qa_item::QAEvalItem`: The QA evaluation item containing the question and its answer. +- `ctx::RAGContext`: The context used for generating the QA pair, including the original context and the answers. + Comes from `airag(...; return_context=true)` +- `verbose::Bool`: If `true`, enables verbose logging. Defaults to `true`. +- `parameters_dict::AbstractDict`: Track any parameters used for later evaluations. +- `judge_template::Symbol`: The template symbol for the AI model used to judge the answer. Defaults to `:RAGJudgeAnswerFromContext`. +- `model_judge::AbstractString`: The AI model used for judging the answer's quality. + Defaults to standard chat model, but it is advisable to use more powerful model GPT-4. + +# Returns +`QAEvalResult`: An evaluation result that includes various scores and metadata related to the QA evaluation. + +# Notes +- The function computes a retrieval score and rank based on how well the context matches the QA context. +- It then uses the `judge_template` and `model_judge` to score the answer's accuracy and relevance. +- In case of errors during evaluation, the function logs a warning (if `verbose` is `true`) and the `answer_score` will be set to `nothing`. + +# Examples + +Evaluating a QA pair using a specific context and model: +```julia +qa_item = QAEvalItem(question="What is the capital of France?", answer="Paris", context="France is a country in Europe.") +ctx = RAGContext(source="Wikipedia", context="France is a country in Europe.", answer="Paris") +parameters_dict = Dict("param1" => "value1", "param2" => "value2") + +eval_result = run_qa_evals(qa_item, ctx, parameters_dict=parameters_dict, model_judge="MyAIJudgeModel") +``` +""" function run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; verbose::Bool = true, parameters_dict::AbstractDict, - judge_template::Symbol = :RAGJudgeAnswerFromContext, - model_judge::AbstractString) + judge_template::Symbol = :RAGJudgeAnswerFromContextShort, + model_judge::AbstractString = PT.MODEL_CHAT) retrieval_score = score_retrieval_hit(qa_item.context, ctx.context) retrieval_rank = score_retrieval_rank(qa_item.context, ctx.context) + # Note we could evaluate if RAGContext and QAEvalItem are at least using the same sources etc. + answer_score = try msg = aiextract(judge_template; model = model_judge, verbose, ctx.context, - question, - msg.content, - return_type = RAG.JudgeAllScores) + ctx.question, + ctx.answer, + return_type = JudgeAllScores) final_rating = if msg.content isa AbstractDict && haskey(msg.content, :final_rating) # if return type parsing failed msg.content[:final_rating] @@ -159,7 +196,7 @@ function run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; end return QAEvalResult(; - ctx.source, + qa_item.source, qa_item.context, qa_item.question, ctx.answer, diff --git a/src/Experimental/RAGTools/generation.jl b/src/Experimental/RAGTools/generation.jl index 733fe5e3c..93d10fd99 100644 --- a/src/Experimental/RAGTools/generation.jl +++ b/src/Experimental/RAGTools/generation.jl @@ -134,6 +134,7 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC question, answer = msg.content, context, + sources = sources(index)[reranked_candidates.positions], emb_candidates, tag_candidates, filtered_candidates, diff --git a/src/Experimental/RAGTools/types.jl b/src/Experimental/RAGTools/types.jl index cd8ee1607..2aeb7a4d0 100644 --- a/src/Experimental/RAGTools/types.jl +++ b/src/Experimental/RAGTools/types.jl @@ -122,6 +122,7 @@ A struct for debugging RAG answers. It contains the question, answer, context, a question::AbstractString answer::AbstractString context::Vector{<:AbstractString} + sources::Vector{<:AbstractString} emb_candidates::CandidateChunks tag_candidates::Union{Nothing, CandidateChunks} filtered_candidates::CandidateChunks From 33770f31d2bdc7e032e9f642e90df4f585f6ff36 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 22 Dec 2023 21:09:06 +0100 Subject: [PATCH 073/251] add example --- examples/building_a_RAG.jl | 137 +++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 examples/building_a_RAG.jl diff --git a/examples/building_a_RAG.jl b/examples/building_a_RAG.jl new file mode 100644 index 000000000..877ef5995 --- /dev/null +++ b/examples/building_a_RAG.jl @@ -0,0 +1,137 @@ +# # Small example for how to build a RAG system with new RAGTools +# Note: RAGTools is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package + +using LinearAlgebra, SparseArrays +using PromptingTools +using PromptingTools.Experimental.RAGTools +using JSON3, Serialization, DataFramesMeta +using Statistics: mean +const PT = PromptingTools +const RT = PromptingTools.Experimental.RAGTools + +# ## Ask questions E2E +# Let's put together a few copy&pasted text files from DataFrames.jl docs +dir_raw = joinpath("markdown", "DataFrames") # folder with text documents +files = ["comparison_with_python.txt", "database_style_joins.txt", "what_is_dataframes.txt"] +index = build_index(joinpath.(dir_raw, files); extract_metadata = false) + +# Ask a question +answer = airag(index; question = "I like dplyr, what is the equivalent in Julia?") +# AIMessage("The equivalent package in Julia to the dplyr package in R is DataFrames.jl.") +# The equivalent package in Julia to the dplyr package in R is DataFrames.jl. + +# First RAG in two lines? Done! +# +# What does it do? +# - `build_index` will chunk the documents into smaller pieces, embed them into numbers (to be able to judge similarity of chunks) and, optionally, create a lookup index of metadata/tags for each chunk) +# - `index` is the result of this step and it holds your chunks, embeddings, and other metadata! Just show it :) +# - `airag` will +# - embed your question +# - find the closest chunks in the index +# - [OPTIONAL] extract any potential tags/filters from the question and apply them to filter down the potential candidates +# - [OPTIONAL] rerank the candidate chunks +# - generate an answer from the closest chunks + +# You should save the index for later! +serialize("examples/index.jls", index) +index = deserialize("examples/index.jls") + +# # Evaluations +# However, we want to evaluate the quality of the system. For that, we need a set of questions and answers. +# Ideally, we would hand-craft a set of high quality Q&A pairs. However, this is time consuming and expensive. +# Let's generate them from the chunks in our index! + +# ## Generate Q&A pairs + +# We need to provide: chunks and sources (filepaths for future reference) +evals = build_qa_evals(RT.chunks(index), + RT.sources(index); + instructions = "None.", + verbose = true); +# Info: Q&A Sets built! (cost: $0.143) -- not bad! + +# Note: In practice, you would review each item in this golden evaluation set (and delete any generic/poor questions). +# It will determine the future success of your app, so you need to make sure it's good! + +## Save the evals for later +JSON3.write("examples/evals.json", evals) +evals = JSON3.read("examples/evals.json", Vector{RT.QAEvalItem}); + +# ## Explore one Q&A pair +## Let's explore one evals item -- it's not the best but gives you the idea! +evals[1] +# QAEvalItem: +# source: markdown/DataFrames/comparison_with_python.txt +# context: Comparisons +# This section compares DataFrames.jl with other data manipulation frameworks in Python, R, and Stata. + +# A sample data set can be created using the following code: + +# using DataFrames +# using Statistics +# question: What frameworks are compared with DataFrames.jl? +# answer: Python, R, and Stata + +# ## Evaluate this Q&A pair + +## Let's answer and evaluate this QA item with the judge +# Note: that we used the same question, but generated a different context and answer via `airag` +msg, ctx = airag(index; evals[1].question, return_context = true); + +# ctx is a RAGContext object that keeps all intermediate states of the RAG pipeline for easy evaluation +judged = aiextract(:RAGJudgeAnswerFromContext; + ctx.context, + ctx.question, + ctx.answer, + return_type = RT.JudgeAllScores) +judged.content +# Dict{Symbol, Any} with 7 entries: +# :final_rating => 4.8 +# :clarity => 5 +# :completeness => 5 +# :relevance => 5 +# :consistency => 4 +# :helpfulness => 5 +# :rationale => "The answer is highly relevant to the user's question, as it provides a comprehensive list of frameworks that are compared with DataFrames.jl. The answer is complete, covering all + +x = run_qa_evals(evals[10], ctx; + parameters_dict = Dict(:top_k => 3), verbose = true, model_judge = "gpt4t") +# Fortunately, we don't have to do this one by one -- let's evaluate all out Q&A pairs at once. + +# ## Evaluate the whole set + +# Let's run each question&answer through our eval loop in async (we do it only for the first 10) +# See the `?airag` for which parameters you can tweak, eg, top_k +results = asyncmap(evals[1:10]) do qa_item + ## Generate an answer -- often you want the model_judge to be the highest quality possible, eg, "GPT-4 Turbo" (alias "gpt4t) + msg, ctx = airag(index; qa_item.question, return_context = true, + top_k = 3, verbose = false, model_judge = "gpt4t") + ## Evaluate the response + # Note: you can log key parameters for easier analysis later + run_qa_evals(qa_item, ctx; parameters_dict = Dict(:top_k => 3), verbose = false) +end +# Note that failed evals can show as "nothing", so make sure to handle them +results = filter(!isnothing, results) + +## Let's take a simple average to calculate our score +@info "RAG Evals: $(length(results)) results, Avg. score: $(round(mean(x->x.answer_score, results);digits=1)), Retrieval score: $(100*round(mean(x->x.retrieval_score,results);digits=1))%" +# [ Info: RAG Evals: 10 results, Avg. score: 4.5, Retrieval score: 70.0% + +# or you can analyze it in a DataFrame +df = DataFrame(results) +# 10×8 DataFrame +# Row │ source context ... + +# We're done for today! + +# # What would we do next? +# - Review your evaluation golden data set and keep only the good items +# - Play with the chunk sizes (max_length in build_index) and see how it affects the quality +# - Explore using metadata/key filters (`extract_metadata=true` in build_index) +# - Add filtering for semantic similarity (embedding distance) to make sure we don't pick up irrelevant chunks in the context +# - Use multiple indices or a hybrid index (add a simple BM25 lookup from TextAnalysis.jl) +# - Data processing is the most important step - properly parsed and split text could make wonders +# - Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API)`) +# - Improve the question embedding (eg, rephrase it, generate hypothetical answers and use them to find better context) +# +# ... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) From c936c4d5215a1fc5c3ed3e66cede600e37757e6c Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 22 Dec 2023 21:11:45 +0100 Subject: [PATCH 074/251] example --- CHANGELOG.md | 2 +- examples/{building_a_RAG.jl => building_RAG.jl} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename examples/{building_a_RAG.jl => building_RAG.jl} (97%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 050419827..1d17f6ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Experimental sub-module RAGTools providing basic Retrieval-Augmented Generation functionality. See `?RAGTools` for more information. It's nested inside of `PromptingTools.Experimental.RAGTools` to signify that it might change in the future. +- Experimental sub-module RAGTools providing basic Retrieval-Augmented Generation functionality. See `?RAGTools` for more information. It's nested inside of `PromptingTools.Experimental.RAGTools` to signify that it might change in the future. Key functions are `build_index` and `airag`, but it also provides a suite to make evaluation easier (see `?build_qa_evals` and `?run_qa_evals` or just see the example `src/building_RAG.jl`) ### Fixed - Stricter code parsing in `AICode` to avoid false positives (code blocks must end with "```\n" to catch comments inside text) diff --git a/examples/building_a_RAG.jl b/examples/building_RAG.jl similarity index 97% rename from examples/building_a_RAG.jl rename to examples/building_RAG.jl index 877ef5995..c680b64de 100644 --- a/examples/building_a_RAG.jl +++ b/examples/building_RAG.jl @@ -1,9 +1,9 @@ -# # Small example for how to build a RAG system with new RAGTools +# # Small example for how to build a RAG system with the new RAGTools # Note: RAGTools is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package using LinearAlgebra, SparseArrays using PromptingTools -using PromptingTools.Experimental.RAGTools +using PromptingTools.Experimental.RAGTools # Experimental! May change using JSON3, Serialization, DataFramesMeta using Statistics: mean const PT = PromptingTools From 0a8a19c1fb363d49c341f3a4881bea8ef5fd7b10 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 22 Dec 2023 21:41:16 +0100 Subject: [PATCH 075/251] update docs --- docs/make.jl | 1 + docs/src/examples/building_RAG.md | 214 ++++++++++++++ examples/building_RAG.jl | 53 ++-- examples/data/database_style_joins.txt | 392 +++++++++++++++++++++++++ examples/data/what_is_dataframes.txt | 141 +++++++++ 5 files changed, 778 insertions(+), 23 deletions(-) create mode 100644 docs/src/examples/building_RAG.md create mode 100644 examples/data/database_style_joins.txt create mode 100644 examples/data/what_is_dataframes.txt diff --git a/docs/make.jl b/docs/make.jl index d3e676100..533af9d08 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -24,6 +24,7 @@ makedocs(; "Various examples" => "examples/readme_examples.md", "Using AITemplates" => "examples/working_with_aitemplates.md", "Local models with Ollama.ai" => "examples/working_with_ollama.md", + "Building RAG Application" => "examples/building_rag_application.md", ], "F.A.Q." => "frequently_asked_questions.md", "Reference" => "reference.md", diff --git a/docs/src/examples/building_RAG.md b/docs/src/examples/building_RAG.md new file mode 100644 index 000000000..4c0e57d43 --- /dev/null +++ b/docs/src/examples/building_RAG.md @@ -0,0 +1,214 @@ +```@meta +EditURL = "../../../examples/building_RAG.jl" +``` + +# Building a Simple Retrieval-Augmented Generation (RAG) System with RAGTools + +Note: RAGTools module is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package + +````julia +using LinearAlgebra, SparseArrays +using PromptingTools +using PromptingTools.Experimental.RAGTools # Experimental! May change +using JSON3, Serialization, DataFramesMeta +using Statistics: mean +const PT = PromptingTools +const RT = PromptingTools.Experimental.RAGTools +```` + +## Ask questions E2E +Let's put together a few copy&pasted text files from DataFrames.jl docs + +````julia +files = [ + joinpath("examples", "data", "database_style_joins.txt"), + joinpath("examples", "data", "what_is_dataframes.txt"), +] +index = build_index(files; extract_metadata = false); +```` + +Let's ask a question + +````julia +answer = airag(index; question = "I like dplyr, what is the equivalent in Julia?") +```` + +```` +AIMessage("The equivalent package in Julia to dplyr in R is DataFramesMeta.jl. It provides convenience functions for data manipulation with syntax similar to dplyr.") +```` + +First RAG in two lines? Done! + +What does it do? +- `build_index` will chunk the documents into smaller pieces, embed them into numbers (to be able to judge similarity of chunks) and, optionally, create a lookup index of metadata/tags for each chunk) + - `index` is the result of this step and it holds your chunks, embeddings, and other metadata! Just show it :) +- `airag` will + - embed your question + - find the closest chunks in the index + - [OPTIONAL] extract any potential tags/filters from the question and apply them to filter down the potential candidates + - [OPTIONAL] rerank the candidate chunks +- generate an answer from the closest chunks + +You should save the index for later! + +````julia +serialize("examples/index.jls", index) +index = deserialize("examples/index.jls"); +```` + +# Evaluations +However, we want to evaluate the quality of the system. For that, we need a set of questions and answers. +Ideally, we would handcraft a set of high-quality Q&A pairs. However, this is time-consuming and expensive. +Let's generate them from the chunks in our index! + +## Generate Q&A pairs + +We need to provide: chunks and sources (file paths for future reference) + +````julia +evals = build_qa_evals(RT.chunks(index), + RT.sources(index); + instructions = "None.", + verbose = true); +```` + +```` +[ Info: Q&A Sets built! (cost: $0.102) + +```` + +> [!TIP] +> In practice, you would review each item in this golden evaluation set (and delete any generic/poor questions). +> It will determine the future success of your app, so you need to make sure it's good! + +````julia +# Save the evals for later +JSON3.write("examples/evals.json", evals) +evals = JSON3.read("examples/evals.json", Vector{RT.QAEvalItem}); +```` + +## Explore one Q&A pair + +Let's explore one evals item -- it's not the best quality but gives you the idea! +````julia +evals[1] +```` + +```` +QAEvalItem: + source: examples/data/database_style_joins.txt + context: Database-Style Joins +Introduction to joins +We often need to combine two or more data sets together to provide a complete picture of the topic we are studying. For example, suppose that we have the following two data sets: + +julia> using DataFrames + question: What is the purpose of joining two or more data sets together? + answer: The purpose of joining two or more data sets together is to provide a complete picture of the topic being studied. + +```` + +## Evaluate this Q&A pair + +````julia +# Let's answer and evaluate this QA item with the judge +# Note: that we used the same question, but generated a different context and answer via `airag` +msg, ctx = airag(index; evals[1].question, return_context = true); +# ctx is a RAGContext object that keeps all intermediate states of the RAG pipeline for easy evaluation +judged = aiextract(:RAGJudgeAnswerFromContext; + ctx.context, + ctx.question, + ctx.answer, + return_type = RT.JudgeAllScores) +judged.content +```` + +```` +Dict{Symbol, Any} with 6 entries: + :final_rating => 4.8 + :clarity => 5 + :completeness => 4 + :relevance => 5 + :consistency => 5 + :helpfulness => 5 +```` + +We can also run the whole evaluation in a function (a few more metrics are available): +````julia +x = run_qa_evals(evals[10], ctx; + parameters_dict = Dict(:top_k => 3), verbose = true, model_judge = "gpt4t") +```` + +```` +QAEvalResult: + source: examples/data/database_style_joins.txt + context: outerjoin: the output contains rows for values of the key that exist in any of the passed data frames. +semijoin: Like an inner join, but output is restricted to columns from the first (left) argument. + question: What is the difference between outer join and semi join? + answer: The purpose of joining two or more data sets together is to combine them in order to provide a complete picture or analysis of a specific topic or dataset. By joining data sets, we can combine information from multiple sources to gain more insights and make more informed decisions. + retrieval_score: 0.0 + retrieval_rank: nothing + answer_score: 5 + parameters: Dict(:top_k => 3) + +```` + +Fortunately, we don't have to do this one by one -- let's evaluate all our Q&A pairs at once. + +## Evaluate the whole set + +Let's run each question&answer through our eval loop in async (we do it only for the first 10) +See the `?airag` for which parameters you can tweak, eg, top_k + +````julia +results = asyncmap(evals[1:10]) do qa_item + # Generate an answer -- often you want the model_judge to be the highest quality possible, eg, "GPT-4 Turbo" (alias "gpt4t) + msg, ctx = airag(index; qa_item.question, return_context = true, + top_k = 3, verbose = false, model_judge = "gpt4t") + # Evaluate the response + # Note: you can log key parameters for easier analysis later + run_qa_evals(qa_item, ctx; parameters_dict = Dict(:top_k => 3), verbose = false) +end +## Note that the "failed" evals can show as "nothing", so make sure to handle them. +results = filter(!isnothing, results); +```` + + +````julia + +# Let's take a simple average to calculate our score +@info "RAG Evals: $(length(results)) results, Avg. score: $(round(mean(x->x.answer_score, results);digits=1)), Retrieval score: $(100*round(mean(x->x.retrieval_score,results);digits=1))%" +```` + +```` +[ Info: RAG Evals: 10 results, Avg. score: 4.6, Retrieval score: 100.0% + +```` + +or you can analyze it in a DataFrame + +````julia +df = DataFrame(results) +```` + +```@raw html +
10×8 DataFrame
Rowsourcecontextquestionanswerretrieval_scoreretrieval_rankanswer_scoreparameters
StringStringStringSubStrin…Float64Int64Float64Dict…
1examples/data/database_style_joins.txtDatabase-Style Joins\nIntroduction to joins\nWe often need to combine two or more data sets together to provide a complete picture of the topic we are studying. For example, suppose that we have the following two data sets:\n\njulia> using DataFramesWhat is the purpose of joining two or more data sets together?The purpose of joining two or more data sets together is to combine the data sets based on a common key and provide a complete picture of the topic being studied.1.015.0Dict(:top_k=>3)
2examples/data/database_style_joins.txtjulia> people = DataFrame(ID=[20, 40], Name=["John Doe", "Jane Doe"])\n2×2 DataFrame\n Row │ ID Name\n │ Int64 String\n─────┼─────────────────\n 1 │ 20 John Doe\n 2 │ 40 Jane DoeWhat is the DataFrame called 'people' composed of?The DataFrame called 'people' consists of two columns: 'ID' and 'Name'. The 'ID' column contains integers, and the 'Name' column contains strings.1.014.0Dict(:top_k=>3)
3examples/data/database_style_joins.txtjulia> jobs = DataFrame(ID=[20, 40], Job=["Lawyer", "Doctor"])\n2×2 DataFrame\n Row │ ID Job\n │ Int64 String\n─────┼───────────────\n 1 │ 20 Lawyer\n 2 │ 40 DoctorWhat are the jobs and IDs listed in the dataframe?The jobs and IDs listed in the dataframe are as follows:\n\nID: 20\nJob: Lawyer\n\nID: 40\nJob: Doctor1.014.67Dict(:top_k=>3)
4examples/data/database_style_joins.txtWe might want to work with a larger data set that contains both the names and jobs for each ID. We can do this using the innerjoin function:How can we combine the names and jobs for each ID in a larger data set?We can use the `innerjoin` function to combine the names and jobs for each ID in a larger data set.1.014.33333Dict(:top_k=>3)
5examples/data/database_style_joins.txtjulia> innerjoin(people, jobs, on = :ID)\n2×3 DataFrame\n Row │ ID Name Job\n │ Int64 String String\n─────┼─────────────────────────\n 1 │ 20 John Doe Lawyer\n 2 │ 40 Jane Doe DoctorWhat is the name of the person with the ID 40 and their job?The name of the person with the ID 40 is Jane Doe and their job is Doctor.1.024.67Dict(:top_k=>3)
6examples/data/database_style_joins.txtIn relational database theory, this operation is generally referred to as a join. The columns used to determine which rows should be combined during a join are called keys.\n\nThe following functions are provided to perform seven kinds of joins:What are the different kinds of joins?The different kinds of joins are:\n\n1. Inner Join: Returns only the rows that have matching values in both data frames.\n2. Left Join: Returns all rows from the left data frame and the matching rows from the right data frame.\n3. Right Join: Returns all rows from the right data frame and the matching rows from the left data frame.\n4. Full Outer Join: Returns all rows from both data frames and fills in missing values with null.\n5. Cross Join: Returns the cartesian product of the rows from both data frames.\n6. Semi Join: Returns only the rows from the left data frame that have matching values in the right data frame.\n7. Anti Join: Returns only the rows from the left data frame that do not have matching values in the right data frame.1.014.66667Dict(:top_k=>3)
7examples/data/database_style_joins.txtinnerjoin: the output contains rows for values of the key that exist in all passed data frames.What does the output of the inner join operation contain?The output of the inner join operation contains only the rows for values of the key that exist in all passed data frames.1.015.0Dict(:top_k=>3)
8examples/data/database_style_joins.txtleftjoin: the output contains rows for values of the key that exist in the first (left) argument, whether or not that value exists in the second (right) argument.What is the purpose of the left join operation?The purpose of the left join operation is to combine data from two tables based on a common key, where all rows from the left (first) table are included in the output, regardless of whether there is a match in the right (second) table.1.014.66667Dict(:top_k=>3)
9examples/data/database_style_joins.txtrightjoin: the output contains rows for values of the key that exist in the second (right) argument, whether or not that value exists in the first (left) argument.What is the purpose of the right join operation?The purpose of the right join operation is to include all the rows from the second (right) argument, regardless of whether a match is found in the first (left) argument.1.014.67Dict(:top_k=>3)
10examples/data/database_style_joins.txtouterjoin: the output contains rows for values of the key that exist in any of the passed data frames.\nsemijoin: Like an inner join, but output is restricted to columns from the first (left) argument.What is the difference between outer join and semi join?The difference between outer join and semi join is that outer join includes rows for values of the key that exist in any of the passed data frames, whereas semi join is like an inner join but only outputs columns from the first argument.1.014.66667Dict(:top_k=>3)
+``` + +We're done for today! + +# What would we do next? +- Review your evaluation golden data set and keep only the good items +- Play with the chunk sizes (max_length in build_index) and see how it affects the quality +- Explore using metadata/key filters (`extract_metadata=true` in build_index) +- Add filtering for semantic similarity (embedding distance) to make sure we don't pick up irrelevant chunks in the context +- Use multiple indices or a hybrid index (add a simple BM25 lookup from TextAnalysis.jl) +- Data processing is the most important step - properly parsed and split text could make wonders +- Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API)`) +- Improve the question embedding (eg, rephrase it, generate hypothetical answers and use them to find better context) + +... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) + +--- + +*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* + diff --git a/examples/building_RAG.jl b/examples/building_RAG.jl index c680b64de..0f0c2640f 100644 --- a/examples/building_RAG.jl +++ b/examples/building_RAG.jl @@ -1,6 +1,8 @@ -# # Small example for how to build a RAG system with the new RAGTools +# # Building a Simple Retrieval-Augmented Generation (RAG) System with RAGTools + # Note: RAGTools is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package +## Imports using LinearAlgebra, SparseArrays using PromptingTools using PromptingTools.Experimental.RAGTools # Experimental! May change @@ -11,9 +13,11 @@ const RT = PromptingTools.Experimental.RAGTools # ## Ask questions E2E # Let's put together a few copy&pasted text files from DataFrames.jl docs -dir_raw = joinpath("markdown", "DataFrames") # folder with text documents -files = ["comparison_with_python.txt", "database_style_joins.txt", "what_is_dataframes.txt"] -index = build_index(joinpath.(dir_raw, files); extract_metadata = false) +files = [ + joinpath("examples", "data", "database_style_joins.txt"), + joinpath("examples", "data", "what_is_dataframes.txt"), +] +index = build_index(files; extract_metadata = false) # Ask a question answer = airag(index; question = "I like dplyr, what is the equivalent in Julia?") @@ -48,17 +52,18 @@ evals = build_qa_evals(RT.chunks(index), RT.sources(index); instructions = "None.", verbose = true); -# Info: Q&A Sets built! (cost: $0.143) -- not bad! +## Info: Q&A Sets built! (cost: $0.143) -- not bad! -# Note: In practice, you would review each item in this golden evaluation set (and delete any generic/poor questions). -# It will determine the future success of your app, so you need to make sure it's good! +# > [!TIP] +# > In practice, you would review each item in this golden evaluation set (and delete any generic/poor questions). +# > It will determine the future success of your app, so you need to make sure it's good! ## Save the evals for later JSON3.write("examples/evals.json", evals) evals = JSON3.read("examples/evals.json", Vector{RT.QAEvalItem}); # ## Explore one Q&A pair -## Let's explore one evals item -- it's not the best but gives you the idea! +# Let's explore one evals item -- it's not the best but gives you the idea! evals[1] # QAEvalItem: # source: markdown/DataFrames/comparison_with_python.txt @@ -75,28 +80,30 @@ evals[1] # ## Evaluate this Q&A pair ## Let's answer and evaluate this QA item with the judge -# Note: that we used the same question, but generated a different context and answer via `airag` +## Note: that we used the same question, but generated a different context and answer via `airag` msg, ctx = airag(index; evals[1].question, return_context = true); -# ctx is a RAGContext object that keeps all intermediate states of the RAG pipeline for easy evaluation +## ctx is a RAGContext object that keeps all intermediate states of the RAG pipeline for easy evaluation judged = aiextract(:RAGJudgeAnswerFromContext; ctx.context, ctx.question, ctx.answer, return_type = RT.JudgeAllScores) judged.content -# Dict{Symbol, Any} with 7 entries: -# :final_rating => 4.8 -# :clarity => 5 -# :completeness => 5 -# :relevance => 5 -# :consistency => 4 -# :helpfulness => 5 -# :rationale => "The answer is highly relevant to the user's question, as it provides a comprehensive list of frameworks that are compared with DataFrames.jl. The answer is complete, covering all - +## Dict{Symbol, Any} with 7 entries: +## :final_rating => 4.8 +## :clarity => 5 +## :completeness => 5 +## :relevance => 5 +## :consistency => 4 +## :helpfulness => 5 +## :rationale => "The answer is highly relevant to the user's question, as it provides a comprehensive list of frameworks that are compared with DataFrames.jl. The answer is complete, covering all + +# We can also run the whole evaluation in a function (a few more metrics are available): x = run_qa_evals(evals[10], ctx; parameters_dict = Dict(:top_k => 3), verbose = true, model_judge = "gpt4t") -# Fortunately, we don't have to do this one by one -- let's evaluate all out Q&A pairs at once. + +# Fortunately, we don't have to do this one by one -- let's evaluate all our Q&A pairs at once. # ## Evaluate the whole set @@ -107,11 +114,11 @@ results = asyncmap(evals[1:10]) do qa_item msg, ctx = airag(index; qa_item.question, return_context = true, top_k = 3, verbose = false, model_judge = "gpt4t") ## Evaluate the response - # Note: you can log key parameters for easier analysis later + ## Note: you can log key parameters for easier analysis later run_qa_evals(qa_item, ctx; parameters_dict = Dict(:top_k => 3), verbose = false) end -# Note that failed evals can show as "nothing", so make sure to handle them -results = filter(!isnothing, results) +## Note that the "failed" evals can show as "nothing", so make sure to handle them. +results = filter(!isnothing, results); ## Let's take a simple average to calculate our score @info "RAG Evals: $(length(results)) results, Avg. score: $(round(mean(x->x.answer_score, results);digits=1)), Retrieval score: $(100*round(mean(x->x.retrieval_score,results);digits=1))%" diff --git a/examples/data/database_style_joins.txt b/examples/data/database_style_joins.txt new file mode 100644 index 000000000..9e04ecab1 --- /dev/null +++ b/examples/data/database_style_joins.txt @@ -0,0 +1,392 @@ +Database-Style Joins +Introduction to joins +We often need to combine two or more data sets together to provide a complete picture of the topic we are studying. For example, suppose that we have the following two data sets: + +julia> using DataFrames + +julia> people = DataFrame(ID=[20, 40], Name=["John Doe", "Jane Doe"]) +2×2 DataFrame + Row │ ID Name + │ Int64 String +─────┼───────────────── + 1 │ 20 John Doe + 2 │ 40 Jane Doe + +julia> jobs = DataFrame(ID=[20, 40], Job=["Lawyer", "Doctor"]) +2×2 DataFrame + Row │ ID Job + │ Int64 String +─────┼─────────────── + 1 │ 20 Lawyer + 2 │ 40 Doctor + +We might want to work with a larger data set that contains both the names and jobs for each ID. We can do this using the innerjoin function: + +julia> innerjoin(people, jobs, on = :ID) +2×3 DataFrame + Row │ ID Name Job + │ Int64 String String +─────┼───────────────────────── + 1 │ 20 John Doe Lawyer + 2 │ 40 Jane Doe Doctor + +In relational database theory, this operation is generally referred to as a join. The columns used to determine which rows should be combined during a join are called keys. + +The following functions are provided to perform seven kinds of joins: + +innerjoin: the output contains rows for values of the key that exist in all passed data frames. +leftjoin: the output contains rows for values of the key that exist in the first (left) argument, whether or not that value exists in the second (right) argument. +rightjoin: the output contains rows for values of the key that exist in the second (right) argument, whether or not that value exists in the first (left) argument. +outerjoin: the output contains rows for values of the key that exist in any of the passed data frames. +semijoin: Like an inner join, but output is restricted to columns from the first (left) argument. +antijoin: The output contains rows for values of the key that exist in the first (left) but not the second (right) argument. As with semijoin, output is restricted to columns from the first (left) argument. +crossjoin: The output is the cartesian product of rows from all passed data frames. +See the Wikipedia page on SQL joins for more information. + +Here are examples of different kinds of join: + +julia> jobs = DataFrame(ID=[20, 60], Job=["Lawyer", "Astronaut"]) +2×2 DataFrame + Row │ ID Job + │ Int64 String +─────┼────────────────── + 1 │ 20 Lawyer + 2 │ 60 Astronaut + +julia> innerjoin(people, jobs, on = :ID) +1×3 DataFrame + Row │ ID Name Job + │ Int64 String String +─────┼───────────────────────── + 1 │ 20 John Doe Lawyer + +julia> leftjoin(people, jobs, on = :ID) +2×3 DataFrame + Row │ ID Name Job + │ Int64 String String? +─────┼────────────────────────── + 1 │ 20 John Doe Lawyer + 2 │ 40 Jane Doe missing + +julia> rightjoin(people, jobs, on = :ID) +2×3 DataFrame + Row │ ID Name Job + │ Int64 String? String +─────┼──────────────────────────── + 1 │ 20 John Doe Lawyer + 2 │ 60 missing Astronaut + +julia> outerjoin(people, jobs, on = :ID) +3×3 DataFrame + Row │ ID Name Job + │ Int64 String? String? +─────┼──────────────────────────── + 1 │ 20 John Doe Lawyer + 2 │ 40 Jane Doe missing + 3 │ 60 missing Astronaut + +julia> semijoin(people, jobs, on = :ID) +1×2 DataFrame + Row │ ID Name + │ Int64 String +─────┼───────────────── + 1 │ 20 John Doe + +julia> antijoin(people, jobs, on = :ID) +1×2 DataFrame + Row │ ID Name + │ Int64 String +─────┼───────────────── + 1 │ 40 Jane Doe + +Cross joins are the only kind of join that does not use a on key: + +julia> crossjoin(people, jobs, makeunique = true) +4×4 DataFrame + Row │ ID Name ID_1 Job + │ Int64 String Int64 String +─────┼─────────────────────────────────── + 1 │ 20 John Doe 20 Lawyer + 2 │ 20 John Doe 60 Astronaut + 3 │ 40 Jane Doe 20 Lawyer + 4 │ 40 Jane Doe 60 Astronaut + +Key value comparisons and floating point values +Key values from the two or more data frames are compared using the isequal function. This is consistent with the Set and Dict types in Julia Base. + +It is not recommended to use floating point numbers as keys: floating point comparisons can be surprising and unpredictable. If you do use floating point keys, note that by default an error is raised when keys include -0.0 (negative zero) or NaN values. Here is an example: + +julia> innerjoin(DataFrame(id=[-0.0]), DataFrame(id=[0.0]), on=:id) +ERROR: ArgumentError: Currently for numeric values `NaN` and `-0.0` in their real or imaginary components are not allowed. Such value was found in column :id in left data frame. Use CategoricalArrays.jl to wrap these values in a CategoricalVector to perform the requested join. + +This can be overridden by wrapping the key values in a categorical vector. + +Joining on key columns with different names +In order to join data frames on keys which have different names in the left and right tables, you may pass left => right pairs as on argument: + +julia> a = DataFrame(ID=[20, 40], Name=["John Doe", "Jane Doe"]) +2×2 DataFrame + Row │ ID Name + │ Int64 String +─────┼───────────────── + 1 │ 20 John Doe + 2 │ 40 Jane Doe + +julia> b = DataFrame(IDNew=[20, 40], Job=["Lawyer", "Doctor"]) +2×2 DataFrame + Row │ IDNew Job + │ Int64 String +─────┼─────────────── + 1 │ 20 Lawyer + 2 │ 40 Doctor + +julia> innerjoin(a, b, on = :ID => :IDNew) +2×3 DataFrame + Row │ ID Name Job + │ Int64 String String +─────┼───────────────────────── + 1 │ 20 John Doe Lawyer + 2 │ 40 Jane Doe Doctor + +Here is another example with multiple columns: + +julia> a = DataFrame(City=["Amsterdam", "London", "London", "New York", "New York"], + Job=["Lawyer", "Lawyer", "Lawyer", "Doctor", "Doctor"], + Category=[1, 2, 3, 4, 5]) +5×3 DataFrame + Row │ City Job Category + │ String String Int64 +─────┼───────────────────────────── + 1 │ Amsterdam Lawyer 1 + 2 │ London Lawyer 2 + 3 │ London Lawyer 3 + 4 │ New York Doctor 4 + 5 │ New York Doctor 5 + +julia> b = DataFrame(Location=["Amsterdam", "London", "London", "New York", "New York"], + Work=["Lawyer", "Lawyer", "Lawyer", "Doctor", "Doctor"], + Name=["a", "b", "c", "d", "e"]) +5×3 DataFrame + Row │ Location Work Name + │ String String String +─────┼─────────────────────────── + 1 │ Amsterdam Lawyer a + 2 │ London Lawyer b + 3 │ London Lawyer c + 4 │ New York Doctor d + 5 │ New York Doctor e + +julia> innerjoin(a, b, on = [:City => :Location, :Job => :Work]) +9×4 DataFrame + Row │ City Job Category Name + │ String String Int64 String +─────┼───────────────────────────────────── + 1 │ Amsterdam Lawyer 1 a + 2 │ London Lawyer 2 b + 3 │ London Lawyer 3 b + 4 │ London Lawyer 2 c + 5 │ London Lawyer 3 c + 6 │ New York Doctor 4 d + 7 │ New York Doctor 5 d + 8 │ New York Doctor 4 e + 9 │ New York Doctor 5 e + +Handling of duplicate keys and tracking source data frame +Additionally, notice that in the last join rows 2 and 3 had the same values on on variables in both joined DataFrames. In such a situation innerjoin, outerjoin, leftjoin and rightjoin will produce all combinations of matching rows. In our example rows from 2 to 5 were created as a result. The same behavior can be observed for rows 4 and 5 in both joined DataFrames. + +In order to check that columns passed as the on argument define unique keys (according to isequal) in each input data frame you can set the validate keyword argument to a two-element tuple or a pair of Bool values, with each element indicating whether to run check for the corresponding data frame. Here is an example for the join operation described above: + +julia> innerjoin(a, b, on = [(:City => :Location), (:Job => :Work)], validate=(true, true)) +ERROR: ArgumentError: Merge key(s) are not unique in both df1 and df2. df1 contains 2 duplicate keys: (City = "London", Job = "Lawyer") and (City = "New York", Job = "Doctor"). df2 contains 2 duplicate keys: (Location = "London", Work = "Lawyer") and (Location = "New York", Work = "Doctor"). + +Finally, using the source keyword argument you can add a column to the resulting data frame indicating whether the given row appeared only in the left, the right or both data frames. Here is an example: + +julia> a = DataFrame(ID=[20, 40], Name=["John", "Jane"]) +2×2 DataFrame + Row │ ID Name + │ Int64 String +─────┼─────────────── + 1 │ 20 John + 2 │ 40 Jane + +julia> b = DataFrame(ID=[20, 60], Job=["Lawyer", "Doctor"]) +2×2 DataFrame + Row │ ID Job + │ Int64 String +─────┼─────────────── + 1 │ 20 Lawyer + 2 │ 60 Doctor + +julia> outerjoin(a, b, on=:ID, validate=(true, true), source=:source) +3×4 DataFrame + Row │ ID Name Job source + │ Int64 String? String? String +─────┼───────────────────────────────────── + 1 │ 20 John Lawyer both + 2 │ 40 Jane missing left_only + 3 │ 60 missing Doctor right_only + +Note that this time we also used the validate keyword argument and it did not produce errors as the keys defined in both source data frames were unique. + +Renaming joined columns +Often you want to keep track of the source data frame. This feature is supported with the renamecols keyword argument: + +julia> innerjoin(a, b, on=:ID, renamecols = "_left" => "_right") +1×3 DataFrame + Row │ ID Name_left Job_right + │ Int64 String String +─────┼───────────────────────────── + 1 │ 20 John Lawyer + +In the above example we added the "_left" suffix to the non-key columns from the left table and the "_right" suffix to the non-key columns from the right table. + +Alternatively it is allowed to pass a function transforming column names: + +julia> innerjoin(a, b, on=:ID, renamecols = lowercase => uppercase) +1×3 DataFrame + Row │ ID name JOB + │ Int64 String String +─────┼─────────────────────── + 1 │ 20 John Lawyer + +Matching missing values in joins +By default when you try to to perform a join on a key that has missing values you get an error: + +julia> df1 = DataFrame(id=[1, missing, 3], a=1:3) +3×2 DataFrame + Row │ id a + │ Int64? Int64 +─────┼──────────────── + 1 │ 1 1 + 2 │ missing 2 + 3 │ 3 3 + +julia> df2 = DataFrame(id=[1, 2, missing], b=1:3) +3×2 DataFrame + Row │ id b + │ Int64? Int64 +─────┼──────────────── + 1 │ 1 1 + 2 │ 2 2 + 3 │ missing 3 + +julia> innerjoin(df1, df2, on=:id) +ERROR: ArgumentError: Missing values in key columns are not allowed when matchmissing == :error. `missing` found in column :id in left data frame. + +If you would prefer missing values to be treated as equal pass the matchmissing=:equal keyword argument: + +julia> innerjoin(df1, df2, on=:id, matchmissing=:equal) +2×3 DataFrame + Row │ id a b + │ Int64? Int64 Int64 +─────┼─────────────────────── + 1 │ 1 1 1 + 2 │ missing 2 3 + +Alternatively you might want to drop all rows with missing values. In this case pass matchmissing=:notequal: + +julia> innerjoin(df1, df2, on=:id, matchmissing=:notequal) +1×3 DataFrame + Row │ id a b + │ Int64? Int64 Int64 +─────┼────────────────────── + 1 │ 1 1 1 + +Specifying row order in the join result +By default the order of rows produced by the join operation is undefined: + +julia> df_left = DataFrame(id=[1, 2, 4, 5], left=1:4) +4×2 DataFrame + Row │ id left + │ Int64 Int64 +─────┼────────────── + 1 │ 1 1 + 2 │ 2 2 + 3 │ 4 3 + 4 │ 5 4 + +julia> df_right = DataFrame(id=[2, 1, 3, 6, 7], right=1:5) +5×2 DataFrame + Row │ id right + │ Int64 Int64 +─────┼────────────── + 1 │ 2 1 + 2 │ 1 2 + 3 │ 3 3 + 4 │ 6 4 + 5 │ 7 5 + +julia> outerjoin(df_left, df_right, on=:id) +7×3 DataFrame + Row │ id left right + │ Int64 Int64? Int64? +─────┼───────────────────────── + 1 │ 2 2 1 + 2 │ 1 1 2 + 3 │ 4 3 missing + 4 │ 5 4 missing + 5 │ 3 missing 3 + 6 │ 6 missing 4 + 7 │ 7 missing 5 + +If you would like the result to keep the row order of the left table pass the order=:left keyword argument: + +julia> outerjoin(df_left, df_right, on=:id, order=:left) +7×3 DataFrame + Row │ id left right + │ Int64 Int64? Int64? +─────┼───────────────────────── + 1 │ 1 1 2 + 2 │ 2 2 1 + 3 │ 4 3 missing + 4 │ 5 4 missing + 5 │ 3 missing 3 + 6 │ 6 missing 4 + 7 │ 7 missing 5 + +Note that in this case keys missing from the left table are put after the keys present in it. + +Similarly order=:right keeps the order of the right table (and puts keys not present in it at the end): + +julia> outerjoin(df_left, df_right, on=:id, order=:right) +7×3 DataFrame + Row │ id left right + │ Int64 Int64? Int64? +─────┼───────────────────────── + 1 │ 2 2 1 + 2 │ 1 1 2 + 3 │ 3 missing 3 + 4 │ 6 missing 4 + 5 │ 7 missing 5 + 6 │ 4 3 missing + 7 │ 5 4 missing + +In-place left join +A common operation is adding data from a reference table to some main table. It is possible to perform such an in-place update using the leftjoin! function. In this case the left table is updated in place with matching rows from the right table. + +julia> main = DataFrame(id=1:4, main=1:4) +4×2 DataFrame + Row │ id main + │ Int64 Int64 +─────┼────────────── + 1 │ 1 1 + 2 │ 2 2 + 3 │ 3 3 + 4 │ 4 4 + +julia> leftjoin!(main, DataFrame(id=[2, 4], info=["a", "b"]), on=:id); + +julia> main +4×3 DataFrame + Row │ id main info + │ Int64 Int64 String? +─────┼─────────────────────── + 1 │ 1 1 missing + 2 │ 2 2 a + 3 │ 3 3 missing + 4 │ 4 4 b + +Note that in this case the order and number of rows in the left table is not changed. Therefore, in particular, it is not allowed to have duplicate keys in the right table: + +julia> leftjoin!(main, DataFrame(id=[2, 2], info_bad=["a", "b"]), on=:id) +ERROR: ArgumentError: duplicate rows found in right table \ No newline at end of file diff --git a/examples/data/what_is_dataframes.txt b/examples/data/what_is_dataframes.txt new file mode 100644 index 000000000..c641aa202 --- /dev/null +++ b/examples/data/what_is_dataframes.txt @@ -0,0 +1,141 @@ +Welcome to the DataFrames.jl documentation! + +This resource aims to teach you everything you need to know to get up and running with tabular data manipulation using the DataFrames.jl package. + +For more illustrations of DataFrames.jl usage, in particular in conjunction with other packages you can check-out the following resources (they are kept up to date with the released version of DataFrames.jl): + +What is DataFrames.jl? +DataFrames.jl provides a set of tools for working with tabular data in Julia. Its design and functionality are similar to those of pandas (in Python) and data.frame, data.table and dplyr (in R), making it a great general purpose data science tool. + +DataFrames.jl plays a central role in the Julia Data ecosystem, and has tight integrations with a range of different libraries. DataFrames.jl isn't the only tool for working with tabular data in Julia – as noted below, there are some other great libraries for certain use-cases – but it provides great data wrangling functionality through a familiar interface. + +To understand the toolchain in more detail, have a look at the tutorials in this manual. New users can start with the First Steps with DataFrames.jl section. + +You may find the DataFramesMeta.jl package or one of the other convenience packages discussed in the Data manipulation frameworks section of this manual helpful when writing more advanced data transformations, especially if you do not have a significant programming experience. These packages provide convenience syntax similar to dplyr in R. + +If you use metadata when working with DataFrames.jl you might find the TableMetadataTools.jl package useful. This package defines several convenience functions for performing typical metadata operations. + +DataFrames.jl and the Julia Data Ecosystem +The Julia data ecosystem can be a difficult space for new users to navigate, in part because the Julia ecosystem tends to distribute functionality across different libraries more than some other languages. Because many people coming to DataFrames.jl are just starting to explore the Julia data ecosystem, below is a list of well-supported libraries that provide different data science tools, along with a few notes about what makes each library special, and how well integrated they are with DataFrames.jl. + +Statistics +StatsKit.jl: A convenience meta-package which loads a set of essential packages for statistics, including those mentioned below in this section and DataFrames.jl itself. +Statistics: The Julia standard library comes with a wide range of statistics functionality, but to gain access to these functions you must call using Statistics. +LinearAlgebra: Like Statistics, many linear algebra features (factorizations, inversions, etc.) live in a library you have to load to use. +SparseArrays are also in the standard library but must be loaded to be used. +FreqTables.jl: Create frequency tables / cross-tabulations. Tightly integrated with DataFrames.jl. +HypothesisTests.jl: A range of hypothesis testing tools. +GLM.jl: Tools for estimating linear and generalized linear models. Tightly integrated with DataFrames.jl. +StatsModels.jl: For converting heterogeneous DataFrame into homogeneous matrices for use with linear algebra libraries or machine learning applications that don't directly support DataFrames. Will do things like convert categorical variables into indicators/one-hot-encodings, create interaction terms, etc. +MultivariateStats.jl: linear regression, ridge regression, PCA, component analyses tools. Not well integrated with DataFrames.jl, but easily used in combination with StatsModels. +Machine Learning +MLJ.jl: if you're more of an applied user, there is a single package the pulls from all these different libraries and provides a single, scikit-learn inspired API: MLJ.jl. MLJ.jl provides a common interface for a wide range of machine learning algorithms. +ScikitLearn.jl: A Julia wrapper around the full Python scikit-learn machine learning library. Not well integrated with DataFrames.jl, but can be combined using StatsModels.jl. +AutoMLPipeline: A package that makes it trivial to create complex ML pipeline structures using simple expressions. It leverages on the built-in macro programming features of Julia to symbolically process, manipulate pipeline expressions, and makes it easy to discover optimal structures for machine learning regression and classification. +Deep learning: KNet.jl and Flux.jl. +Plotting +Plots.jl: Powerful, modern plotting library with a syntax akin to that of matplotlib (in Python) or plot (in R). StatsPlots.jl provides Plots.jl with recipes for many standard statistical plots. +Gadfly.jl: High-level plotting library with a "grammar of graphics" syntax akin to that of ggplot (in R). +AlgebraOfGraphics.jl: A "grammar of graphics" library build upon Makie.jl. +VegaLite.jl: High-level plotting library that uses a different "grammar of graphics" syntax and has an emphasis on interactive graphics. +Data Wrangling: +Impute.jl: various methods for handling missing data in vectors, matrices and tables. +DataFramesMeta.jl: A range of convenience functions for DataFrames.jl that augment select and transform to provide a user experience similar to that provided by dplyr in R. +DataFrameMacros.jl: Provides macro versions of the common DataFrames.jl functions similar to DataFramesMeta.jl, with convenient syntax for the manipulation of multiple columns at once. +Query.jl: Query.jl provides a single framework for data wrangling that works with a range of libraries, including DataFrames.jl, other tabular data libraries (more on those below), and even non-tabular data. Provides many convenience functions analogous to those in dplyr in R or LINQ. +You can find more information on these packages in the Data manipulation frameworks section of this manual. +And More! +Graphs.jl: A pure-Julia, high performance network analysis library. Edgelists in DataFrames can be easily converted into graphs using the GraphDataFrameBridge.jl package. +IO: +DataFrames.jl work well with a range of formats, including: +CSV files (using CSV.jl), +Apache Arrow (using Arrow.jl) +reading Stata, SAS and SPSS files (using ReadStatTables.jl; alternatively Queryverse users can choose StatFiles.jl), +Parquet files (using Parquet2.jl), +reading R data files (.rda, .RData) (using RData.jl). +While not all of these libraries are tightly integrated with DataFrames.jl, because DataFrames are essentially collections of aligned Julia vectors, so it is easy to (a) pull out a vector for use with a non-DataFrames-integrated library, or (b) convert your table into a homogeneously-typed matrix using the Matrix constructor or StatsModels.jl. + +Other Julia Tabular Libraries +DataFrames.jl is a great general purpose tool for data manipulation and wrangling, but it's not ideal for all applications. For users with more specialized needs, consider using: + +TypedTables.jl: Type-stable heterogeneous tables. Useful for improved performance when the structure of your table is relatively stable and does not feature thousands of columns. +JuliaDB.jl: For users working with data that is too large to fit in memory, we suggest JuliaDB.jl, which offers better performance for large datasets, and can handle out-of-core data manipulations (Python users can think of JuliaDB.jl as the Julia version of dask). +Note that most tabular data libraries in the Julia ecosystem (including DataFrames.jl) support a common interface (defined in the Tables.jl package). As a result, some libraries are capable or working with a range of tabular data structures, making it easy to move between tabular libraries as your needs change. A user of Query.jl, for example, can use the same code to manipulate data in a DataFrame, a Table (defined by TypedTables.jl), or a JuliaDB table. + +Questions? +If there is something you expect DataFrames to be capable of, but cannot figure out how to do, please reach out with questions in Domains/Data on Discourse. Additionally you might want to listen to an introduction to DataFrames.jl on JuliaAcademy. + +Please report bugs by opening an issue. + +You can follow the source links throughout the documentation to jump right to the source files on GitHub to make pull requests for improving the documentation and function capabilities. + +Please review DataFrames contributing guidelines before submitting your first PR! + +Information on specific versions can be found on the Release page. + +Package Manual +First Steps with DataFrames.jl +Setting up the Environment +Constructors and Basic Utility Functions +Getting and Setting Data in a Data Frame +Basic Usage of Transformation Functions +Getting Started +Installation +The DataFrame Type +Database-Style Joins +Introduction to joins +Key value comparisons and floating point values +Joining on key columns with different names +Handling of duplicate keys and tracking source data frame +Renaming joined columns +Matching missing values in joins +Specifying row order in the join result +In-place left join +The Split-Apply-Combine Strategy +Design of the split-apply-combine support +Examples of the split-apply-combine operations +Using GroupedDataFrame as an iterable and indexable object +Simulating the SQL where clause +Column-independent operations +Column-independent operations versus functions +Specifying group order in groupby +Reshaping and Pivoting Data +Sorting +Categorical Data +Missing Data +Comparisons +Comparison with the Python package pandas +Comparison with the R package dplyr +Comparison with the R package data.table +Comparison with Stata (version 8 and above) +Data manipulation frameworks +DataFramesMeta.jl +DataFrameMacros.jl +Query.jl +API +Only exported (i.e. available for use without DataFrames. qualifier after loading the DataFrames.jl package with using DataFrames) types and functions are considered a part of the public API of the DataFrames.jl package. In general all such objects are documented in this manual (in case some documentation is missing please kindly report an issue here). + +Note +Breaking changes to public and documented API are avoided in DataFrames.jl where possible. + +The following changes are not considered breaking: + +specific floating point values computed by operations may change at any time; users should rely only on approximate accuracy; +in functions that use the default random number generator provided by Base Julia the specific random numbers computed may change across Julia versions; +if the changed functionality is classified as a bug; +if the changed behavior was not documented; two major cases are: +in its implementation some function accepted a wider range of arguments that it was documented to handle - changes in handling of undocumented arguments are not considered as breaking; +the type of the value returned by a function changes, but it still follows the contract specified in the documentation; for example if a function is documented to return a vector then changing its type from Vector to PooledVector is not considered as breaking; +error behavior: code that threw an exception can change exception type thrown or stop throwing an exception; +changes in display (how objects are printed); +changes to the state of global objects from Base Julia whose state normally is considered volatile (e.g. state of global random number generator). +All types and functions that are part of public API are guaranteed to go through a deprecation period before a breaking change is made to them or they would be removed. + +The standard practice is that breaking changes are implemented when a major release of DataFrames.jl is made (e.g. functionalities deprecated in a 1.x release would be changed in the 2.0 release). + +In rare cases a breaking change might be introduced in a minor release. In such a case the changed behavior still goes through one minor release during which it is deprecated. The situations where such a breaking change might be allowed are (still such breaking changes will be avoided if possible): + +the affected functionality was previously clearly identified in the documentation as being subject to changes (for example in DataFrames.jl 1.4 release propagation rules of :note-style metadata are documented as such); +the change is on the border of being classified as a bug (in rare cases even if a behavior of some function was documented its consequences for certain argument combinations could be decided to be unintended and not wanted); +the change is needed to adjust DataFrames.jl functionality to changes in Base Julia. +Please be warned that while Julia allows you to access internal functions or types of DataFrames.jl these can change without warning between versions of DataFrames.jl. In particular it is not safe to directly access fields of types that are a part of public API of the DataFrames.jl package using e.g. the getfield function. Whenever some operation on fields of defined types is considered allowed an appropriate exported function should be used instead. \ No newline at end of file From e917ff3ed168c7d536be4a30534cfb6107f8f45d Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 22 Dec 2023 21:43:16 +0100 Subject: [PATCH 076/251] update path --- CHANGELOG.md | 2 +- docs/generate_examples.jl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d17f6ea5..7b62c4688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Experimental sub-module RAGTools providing basic Retrieval-Augmented Generation functionality. See `?RAGTools` for more information. It's nested inside of `PromptingTools.Experimental.RAGTools` to signify that it might change in the future. Key functions are `build_index` and `airag`, but it also provides a suite to make evaluation easier (see `?build_qa_evals` and `?run_qa_evals` or just see the example `src/building_RAG.jl`) +- Experimental sub-module RAGTools providing basic Retrieval-Augmented Generation functionality. See `?RAGTools` for more information. It's all nested inside of `PromptingTools.Experimental.RAGTools` to signify that it might change in the future. Key functions are `build_index` and `airag`, but it also provides a suite to make evaluation easier (see `?build_qa_evals` and `?run_qa_evals` or just see the example `examples/building_RAG.jl`) ### Fixed - Stricter code parsing in `AICode` to avoid false positives (code blocks must end with "```\n" to catch comments inside text) diff --git a/docs/generate_examples.jl b/docs/generate_examples.jl index 5c56923e6..f71187d5b 100644 --- a/docs/generate_examples.jl +++ b/docs/generate_examples.jl @@ -8,4 +8,6 @@ output_dir = joinpath(@__DIR__, "src", "examples") filter!(endswith(".jl"), example_files) for fn in example_files Literate.markdown(fn, output_dir; execute = true) -end \ No newline at end of file +end + +# TODO: change meta fields at the top of each file! \ No newline at end of file From 0a287333123998527bbb8faaf9443a7b6c9b81e7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 22 Dec 2023 21:47:06 +0100 Subject: [PATCH 077/251] Update make.jl --- docs/make.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index 533af9d08..86edc0cd8 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -24,7 +24,7 @@ makedocs(; "Various examples" => "examples/readme_examples.md", "Using AITemplates" => "examples/working_with_aitemplates.md", "Local models with Ollama.ai" => "examples/working_with_ollama.md", - "Building RAG Application" => "examples/building_rag_application.md", + "Building RAG Application" => "examples/building_RAG.md", ], "F.A.Q." => "frequently_asked_questions.md", "Reference" => "reference.md", From 1a8a623816fa5676eccba2a18bad3c493e4239db Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 12:27:24 +0100 Subject: [PATCH 078/251] improve tests and documentation --- docs/Project.toml | 5 + docs/make.jl | 6 +- docs/src/examples/building_RAG.md | 53 ++++--- src/Experimental/RAGTools/RAGTools.jl | 2 +- src/Experimental/RAGTools/evaluation.jl | 100 ++++++++++-- src/Experimental/RAGTools/generation.jl | 58 +++++-- src/Experimental/RAGTools/retrieval.jl | 26 +++- test/Experimental/RAGTools/evaluation.jl | 184 ++++++++++++++++++++++- test/Experimental/RAGTools/generation.jl | 30 +++- test/Experimental/RAGTools/retrieval.jl | 7 +- 10 files changed, 417 insertions(+), 54 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index b0857977a..7011d3c5f 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,10 @@ [deps] +DataFramesMeta = "1313f7d8-7da2-5740-9ea0-a2ca25f37964" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589" PromptingTools = "670122d1-24a8-4d70-bfce-740807c42192" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/docs/make.jl b/docs/make.jl index 86edc0cd8..08c18a617 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,9 @@ using PromptingTools using Documenter +using SparseArrays, LinearAlgebra +using PromptingTools.Experimental.RAGTools +using JSON3, Serialization, DataFramesMeta +using Statistics: mean DocMeta.setdocmeta!(PromptingTools, :DocTestSetup, @@ -7,7 +11,7 @@ DocMeta.setdocmeta!(PromptingTools, recursive = true) makedocs(; - modules = [PromptingTools], + modules = [PromptingTools, PromptingTools.Experimental.RAGTools], authors = "J S <49557684+svilupp@users.noreply.github.com> and contributors", repo = "https://github.com/svilupp/PromptingTools.jl/blob/{commit}{path}#{line}", sitename = "PromptingTools.jl", diff --git a/docs/src/examples/building_RAG.md b/docs/src/examples/building_RAG.md index 4c0e57d43..148f8c469 100644 --- a/docs/src/examples/building_RAG.md +++ b/docs/src/examples/building_RAG.md @@ -4,32 +4,41 @@ EditURL = "../../../examples/building_RAG.jl" # Building a Simple Retrieval-Augmented Generation (RAG) System with RAGTools -Note: RAGTools module is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package +Let's build a Retrieval-Augmented Generation (RAG) chatbot, tailored to navigate and interact with the DataFrames.jl documentation. +"RAG" is probably the most common and valuable pattern in Generative AI at the moment. + +If you're not familiar with "RAG", start with this [article](https://towardsdatascience.com/add-your-own-data-to-an-llm-using-retrieval-augmented-generation-rag-b1958bf56a5a). + ````julia using LinearAlgebra, SparseArrays using PromptingTools -using PromptingTools.Experimental.RAGTools # Experimental! May change +using PromptingTools.Experimental.RAGTools +## Note: RAGTools module is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package using JSON3, Serialization, DataFramesMeta using Statistics: mean const PT = PromptingTools const RT = PromptingTools.Experimental.RAGTools ```` -## Ask questions E2E -Let's put together a few copy&pasted text files from DataFrames.jl docs +## RAG in Two Lines + +Let's put together a few text pages from DataFrames.jl docs. +Simply go to [DataFrames.jl docs](https://dataframes.juliadata.org/stable/) and copy&paste a few pages into separate text files. Save them in the `examples/data` folder (see some example pages provided). Ideally, delete all the noise (like headers, footers, etc.) and keep only the text you want to use for the chatbot. Remember, garbage in, garbage out! ````julia files = [ joinpath("examples", "data", "database_style_joins.txt"), joinpath("examples", "data", "what_is_dataframes.txt"), ] +# Build an index of chunks, embed them, and create a lookup index of metadata/tags for each chunk index = build_index(files; extract_metadata = false); ```` Let's ask a question ````julia +# Embeds the question, finds the closest chunks in the index, and generates an answer from the closest chunks answer = airag(index; question = "I like dplyr, what is the equivalent in Julia?") ```` @@ -40,16 +49,17 @@ AIMessage("The equivalent package in Julia to dplyr in R is DataFramesMeta.jl. I First RAG in two lines? Done! What does it do? -- `build_index` will chunk the documents into smaller pieces, embed them into numbers (to be able to judge similarity of chunks) and, optionally, create a lookup index of metadata/tags for each chunk) +- `build_index` will chunk the documents into smaller pieces, embed them into numbers (to be able to judge the similarity of chunks) and, optionally, create a lookup index of metadata/tags for each chunk) - `index` is the result of this step and it holds your chunks, embeddings, and other metadata! Just show it :) - `airag` will - embed your question - - find the closest chunks in the index - - [OPTIONAL] extract any potential tags/filters from the question and apply them to filter down the potential candidates - - [OPTIONAL] rerank the candidate chunks -- generate an answer from the closest chunks + - find the closest chunks in the index (use parameters `top_k` and `minimum_similarity` to tweak the "relevant" chunks) + - [OPTIONAL] extracts any potential tags/filters from the question and applies them to filter down the potential candidates (use `extract_metadata=true` in `build_index`, you can also provide some filters explicitly via `tag_filter`) + - [OPTIONAL] re-ranks the candidate chunks (define and provide your own `rerank_strategy`, eg Cohere ReRank API) + - build a context from the closest chunks (use `chunks_window_margin` to tweak if we include preceding and succeeding chunks as well, see `?build_context` for more details) +- generate an answer from the closest chunks (use `return_context=true` to see under the hood and debug your application) -You should save the index for later! +You should save the index for later to avoid re-embedding / re-extracting the document chunks! ````julia serialize("examples/index.jls", index) @@ -109,8 +119,9 @@ julia> using DataFrames ## Evaluate this Q&A pair +Let's evaluate this QA item with a "judge model" (often GPT-4 is used as a judge). + ````julia -# Let's answer and evaluate this QA item with the judge # Note: that we used the same question, but generated a different context and answer via `airag` msg, ctx = airag(index; evals[1].question, return_context = true); # ctx is a RAGContext object that keeps all intermediate states of the RAG pipeline for easy evaluation @@ -132,7 +143,7 @@ Dict{Symbol, Any} with 6 entries: :helpfulness => 5 ```` -We can also run the whole evaluation in a function (a few more metrics are available): +We can also run the generation + evaluation in a function (a few more metrics are available, eg, retrieval score): ````julia x = run_qa_evals(evals[10], ctx; parameters_dict = Dict(:top_k => 3), verbose = true, model_judge = "gpt4t") @@ -154,10 +165,9 @@ semijoin: Like an inner join, but output is restricted to columns from the first Fortunately, we don't have to do this one by one -- let's evaluate all our Q&A pairs at once. -## Evaluate the whole set +## Evaluate the Whole Set -Let's run each question&answer through our eval loop in async (we do it only for the first 10) -See the `?airag` for which parameters you can tweak, eg, top_k +Let's run each question & answer through our eval loop in async (we do it only for the first 10 to save time). See the `?airag` for which parameters you can tweak, eg, `top_k` ````julia results = asyncmap(evals[1:10]) do qa_item @@ -168,23 +178,26 @@ results = asyncmap(evals[1:10]) do qa_item # Note: you can log key parameters for easier analysis later run_qa_evals(qa_item, ctx; parameters_dict = Dict(:top_k => 3), verbose = false) end -## Note that the "failed" evals can show as "nothing", so make sure to handle them. -results = filter(!isnothing, results); +## Note that the "failed" evals can show as "nothing" (failed as in there was some API error or parsing error), so make sure to handle them. +results = filter(x->!isnothing(x.answer_score), results); ```` +Note: You could also use the vectorized version `results = run_qa_evals(evals)` to evaluate all items at once. ````julia # Let's take a simple average to calculate our score -@info "RAG Evals: $(length(results)) results, Avg. score: $(round(mean(x->x.answer_score, results);digits=1)), Retrieval score: $(100*round(mean(x->x.retrieval_score,results);digits=1))%" +@info "RAG Evals: $(length(results)) results, Avg. score: $(round(mean(x->x.answer_score, results);digits=1)), Retrieval score: $(100*round(Int,mean(x->x.retrieval_score,results)))%" ```` ```` -[ Info: RAG Evals: 10 results, Avg. score: 4.6, Retrieval score: 100.0% +[ Info: RAG Evals: 10 results, Avg. score: 4.6, Retrieval score: 100% ```` -or you can analyze it in a DataFrame +Note: The retrieval score is 100% only because we have two small documents and running on 10 items only. In practice, you would have a much larger document set and a much larger eval set, which would result in a more representative retrieval score. + +You can also analyze the results in a DataFrame: ````julia df = DataFrame(results) diff --git a/src/Experimental/RAGTools/RAGTools.jl b/src/Experimental/RAGTools/RAGTools.jl index 76fda1a55..a315a2a7f 100644 --- a/src/Experimental/RAGTools/RAGTools.jl +++ b/src/Experimental/RAGTools/RAGTools.jl @@ -24,7 +24,7 @@ include("preparation.jl") export find_closest, find_tags, rerank include("retrieval.jl") -export airag +export airag, build_context include("generation.jl") export build_qa_evals, run_qa_evals diff --git a/src/Experimental/RAGTools/evaluation.jl b/src/Experimental/RAGTools/evaluation.jl index 67dae42c1..fe25b32f1 100644 --- a/src/Experimental/RAGTools/evaluation.jl +++ b/src/Experimental/RAGTools/evaluation.jl @@ -20,7 +20,7 @@ end retrieval_score::Union{Number, Nothing} = nothing retrieval_rank::Union{Int, Nothing} = nothing answer_score::Union{Number, Nothing} = nothing - parameters::AbstractDict + parameters::Dict{Symbol, Any} = Dict{Symbol, Any}() end "Provide the `final_rating` between 1-5. Provide the rationale for it." @@ -37,12 +37,18 @@ end consistency::Int helpfulness::Int rationale::Union{Nothing, String} = nothing - final_rating::Int + final_rating::Float64 end function Base.isvalid(x::QAEvalItem) !isempty(x.question) && !isempty(x.answer) && !isempty(x.context) end +# for equality tests +function Base.var"=="(x::Union{QAItem, QAEvalItem, QAEvalResult}, + y::Union{QAItem, QAEvalItem, QAEvalResult}) + typeof(x) == typeof(y) && + all([getfield(x, f) == getfield(y, f) for f in fieldnames(typeof(x))]) +end # Nicer show method with some colors! function Base.show(io::IO, t::Union{QAItem, QAEvalItem, QAEvalResult}) @@ -58,7 +64,8 @@ JSON3.StructTypes.StructType(::Type{QAEvalResult}) = JSON3.StructTypes.Struct() """ build_qa_evals(doc_chunks::Vector{<:AbstractString}, sources::Vector{<:AbstractString}; - model=PT.MODEL_CHAT, instructions="None.", qa_template::Symbol=:RAGCreateQAFromContext, verbose::Bool=true, kwargs...) -> Vector{QAEvalItem} + model=PT.MODEL_CHAT, instructions="None.", qa_template::Symbol=:RAGCreateQAFromContext, + verbose::Bool=true, api_kwargs::NamedTuple = NamedTuple(), kwargs...) -> Vector{QAEvalItem} Create a collection of question and answer evaluations (`QAEvalItem`) from document chunks and sources. This function generates Q&A pairs based on the provided document chunks, using a specified AI model and template. @@ -69,6 +76,7 @@ This function generates Q&A pairs based on the provided document chunks, using a - `model`: The AI model used for generating Q&A pairs. Default is `PT.MODEL_CHAT`. - `instructions::String`: Additional instructions or context to provide to the model generating QA sets. Defaults to "None.". - `qa_template::Symbol`: A template symbol that dictates the AITemplate that will be used. It must have placeholder `context`. Default is `:CreateQAFromContext`. +- `api_kwargs::NamedTuple`: Parameters that will be forwarded to the API endpoint. - `verbose::Bool`: If `true`, additional information like costs will be logged. Defaults to `true`. # Returns @@ -93,7 +101,8 @@ qa_evals = build_qa_evals(doc_chunks, sources) function build_qa_evals(doc_chunks::Vector{<:AbstractString}, sources::Vector{<:AbstractString}; model = PT.MODEL_CHAT, instructions = "None.", - qa_template::Symbol = :RAGCreateQAFromContext, verbose::Bool = true, kwargs...) + qa_template::Symbol = :RAGCreateQAFromContext, verbose::Bool = true, + api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## @assert length(doc_chunks)==length(sources) "Length of `doc_chunks` and `sources` must be the same." placeholders = only(aitemplates(qa_template)).variables # only one template should be found @@ -107,10 +116,11 @@ function build_qa_evals(doc_chunks::Vector{<:AbstractString}, context, instructions, verbose, - model) + model, api_kwargs) Threads.atomic_add!(cost_tracker, PT.call_cost(msg, model)) # track costs QAEvalItem(; context, msg.content.question, msg.content.answer, source) catch e + verbose && @warn e QAEvalItem() end end @@ -134,8 +144,8 @@ end """ run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; verbose::Bool = true, - parameters_dict::AbstractDict, judge_template::Symbol = :RAGJudgeAnswerFromContext, - model_judge::AbstractString) -> QAEvalResult + parameters_dict::Dict{Symbol, Any}, judge_template::Symbol = :RAGJudgeAnswerFromContext, + model_judge::AbstractString,api_kwargs::NamedTuple = NamedTuple()) -> QAEvalResult Evaluates a single `QAEvalItem` using a RAG context (`RAGContext`) and returns a `QAEvalResult` structure. This function assesses the relevance and accuracy of the answers generated in a QA evaluation context. @@ -144,10 +154,11 @@ Evaluates a single `QAEvalItem` using a RAG context (`RAGContext`) and returns a - `ctx::RAGContext`: The context used for generating the QA pair, including the original context and the answers. Comes from `airag(...; return_context=true)` - `verbose::Bool`: If `true`, enables verbose logging. Defaults to `true`. -- `parameters_dict::AbstractDict`: Track any parameters used for later evaluations. +- `parameters_dict::Dict{Symbol, Any}`: Track any parameters used for later evaluations. Keys must be Symbols. - `judge_template::Symbol`: The template symbol for the AI model used to judge the answer. Defaults to `:RAGJudgeAnswerFromContext`. - `model_judge::AbstractString`: The AI model used for judging the answer's quality. Defaults to standard chat model, but it is advisable to use more powerful model GPT-4. +- `api_kwargs::NamedTuple`: Parameters that will be forwarded to the API endpoint. # Returns `QAEvalResult`: An evaluation result that includes various scores and metadata related to the QA evaluation. @@ -169,9 +180,10 @@ eval_result = run_qa_evals(qa_item, ctx, parameters_dict=parameters_dict, model_ ``` """ function run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; - verbose::Bool = true, parameters_dict::AbstractDict, + verbose::Bool = true, parameters_dict::Dict{Symbol, Any} = Dict{Symbol, Any}(), judge_template::Symbol = :RAGJudgeAnswerFromContextShort, - model_judge::AbstractString = PT.MODEL_CHAT) + model_judge::AbstractString = PT.MODEL_CHAT, + api_kwargs::NamedTuple = NamedTuple()) retrieval_score = score_retrieval_hit(qa_item.context, ctx.context) retrieval_rank = score_retrieval_rank(qa_item.context, ctx.context) @@ -182,7 +194,7 @@ function run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; ctx.context, ctx.question, ctx.answer, - return_type = JudgeAllScores) + return_type = JudgeAllScores, api_kwargs) final_rating = if msg.content isa AbstractDict && haskey(msg.content, :final_rating) # if return type parsing failed msg.content[:final_rating] @@ -205,3 +217,69 @@ function run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; answer_score, parameters = parameters_dict) end + +""" + run_qa_evals(index::AbstractChunkIndex, qa_items::AbstractVector{<:QAEvalItem}; + api_kwargs::NamedTuple = NamedTuple(), + airag_kwargs::NamedTuple = NamedTuple(), + qa_evals_kwargs::NamedTuple = NamedTuple(), + verbose::Bool = true, parameters_dict::Dict{Symbol, Any} = Dict{Symbol, Any}()) + +Evaluates a vector of `QAEvalItem`s and returns a vector `QAEvalResult`. +This function assesses the relevance and accuracy of the answers generated in a QA evaluation context. + +See `?run_qa_evals` for more details. + +# Arguments +- `qa_items::AbstractVector{<:QAEvalItem}`: The vector of QA evaluation items containing the questions and their answers. +- `verbose::Bool`: If `true`, enables verbose logging. Defaults to `true`. +- `api_kwargs::NamedTuple`: Parameters that will be forwarded to the API calls. See `?aiextract` for details. +- `airag_kwargs::NamedTuple`: Parameters that will be forwarded to `airag` calls. See `?airag` for details. +- `qa_evals_kwargs::NamedTuple`: Parameters that will be forwarded to `run_qa_evals` calls. See `?run_qa_evals` for details. +- `parameters_dict::Dict{Symbol, Any}`: Track any parameters used for later evaluations. Keys must be Symbols. + +# Returns +`Vector{QAEvalResult}`: Vector of evaluation results that includes various scores and metadata related to the QA evaluation. + +# Example +```julia +index = "..." # Assuming a proper index is defined +qa_items = [QAEvalItem(question="What is the capital of France?", answer="Paris", context="France is a country in Europe."), + QAEvalItem(question="What is the capital of Germany?", answer="Berlin", context="Germany is a country in Europe.")] + +# Let's run a test with `top_k=5` +results = run_qa_evals(index, qa_items; airag_kwargs=(;top_k=5), parameters_dict=Dict(:top_k => 5)) + +# Filter out the "failed" calls +results = filter(x->!isnothing(x.answer_score), results); + +# See average judge score +mean(x->x.answer_score, results) +``` + +""" +function run_qa_evals(index::AbstractChunkIndex, qa_items::AbstractVector{<:QAEvalItem}; + api_kwargs::NamedTuple = NamedTuple(), + airag_kwargs::NamedTuple = NamedTuple(), + qa_evals_kwargs::NamedTuple = NamedTuple(), + verbose::Bool = true, parameters_dict::Dict{Symbol, Any} = Dict{Symbol, Any}()) + # Run evaluations in parallel + results = asyncmap(qa_items) do qa_item + # Generate an answer -- often you want the model_judge to be the highest quality possible, eg, "GPT-4 Turbo" (alias "gpt4t) + msg, ctx = airag(index; qa_item.question, return_context = true, + verbose, api_kwargs, airag_kwargs...) + + # Evaluate the response + # Note: you can log key parameters for easier analysis later + run_qa_evals(qa_item, + ctx; + parameters_dict, + verbose, + api_kwargs, + qa_evals_kwargs...) + end + success_count = count(x -> !isnothing(x.answer_score), results) + verbose && + @info "QA Evaluations complete ($((success_count)/length(qa_items)) evals successful)!" + return results +end \ No newline at end of file diff --git a/src/Experimental/RAGTools/generation.jl b/src/Experimental/RAGTools/generation.jl index 93d10fd99..59b130b3c 100644 --- a/src/Experimental/RAGTools/generation.jl +++ b/src/Experimental/RAGTools/generation.jl @@ -1,10 +1,45 @@ # stub to be replaced within the package extension function _normalize end +""" + build_context(index::AbstractChunkIndex, reranked_candidates::CandidateChunks; chunks_window_margin::Tuple{Int, Int}) -> Vector{String} + +Build context strings for each position in `reranked_candidates` considering a window margin around each position. + +# Arguments +- `reranked_candidates::CandidateChunks`: Candidate chunks which contain positions to extract context from. +- `index::ChunkIndex`: The index containing chunks and sources. +- `chunks_window_margin::Tuple{Int, Int}`: A tuple indicating the margin (before, after) around each position to include in the context. + Defaults to `(1,1)`, which means 1 preceding and 1 suceeding chunk will be included. With `(0,0)`, only the matching chunks will be included. + +# Returns +- `Vector{String}`: A vector of context strings, each corresponding to a position in `reranked_candidates`. + +# Examples +```julia +index = ChunkIndex(...) # Assuming a proper index is defined +candidates = CandidateChunks(index.id, [2, 4], [0.1, 0.2]) +context = build_context(index, candidates; chunks_window_margin=(0, 1)) # include only one following chunk for each matching chunk +``` +""" +function build_context(index::AbstractChunkIndex, reranked_candidates::CandidateChunks; + chunks_window_margin::Tuple{Int, Int} = (1, 1)) + @assert chunks_window_margin[1] >= 0&&chunks_window_margin[2] >= 0 "Both `chunks_window_margin` values must be non-negative" + context = String[] + for (i, position) in enumerate(reranked_candidates.positions) + chunks_ = chunks(index)[max(1, position - chunks_window_margin[1]):min(end, + position + chunks_window_margin[2])] + is_same_source = sources(index)[max(1, position - chunks_window_margin[1]):min(end, + position + chunks_window_margin[2])] .== sources(index)[position] + push!(context, "$(i). $(join(chunks_[is_same_source], "\n"))") + end + return context +end + """ airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromContext; question::AbstractString, - top_k::Int = 3, + top_k::Int = 3, `minimum_similarity::AbstractFloat`= -1.0, tag_filter::Union{Symbol, Vector{String}, Regex, Nothing} = :auto, rerank_strategy::RerankingStrategy = Passthrough(), model_embedding::String = PT.MODEL_EMBEDDING, model_chat::String = PT.MODEL_CHAT, @@ -24,13 +59,14 @@ The function selects relevant chunks from an `ChunkIndex`, optionally filters th - `rag_template::Symbol`: Template for the RAG model, defaults to `:RAGAnswerFromContext`. - `question::AbstractString`: The question to be answered. - `top_k::Int`: Number of top candidates to retrieve based on embedding similarity. +- `minimum_similarity::AbstractFloat`: Minimum similarity threshold (between -1 and 1) for filtering chunks based on embedding similarity. Defaults to -1.0. - `tag_filter::Union{Symbol, Vector{String}, Regex}`: Mechanism for filtering chunks based on tags (either automatically detected, specific tags, or a regex pattern). Disabled by setting to `nothing`. - `rerank_strategy::RerankingStrategy`: Strategy for reranking the retrieved chunks. - `model_embedding::String`: Model used for embedding the question, default is `PT.MODEL_EMBEDDING`. - `model_chat::String`: Model used for generating the final response, default is `PT.MODEL_CHAT`. - `model_metadata::String`: Model used for extracting metadata, default is `PT.MODEL_CHAT`. - `metadata_template::Symbol`: Template for the metadata extraction process from the question, defaults to: `:RAGExtractMetadataShort` -- `chunks_window_margin::Tuple{Int,Int}`: The window size around each chunk to consider for context building. +- `chunks_window_margin::Tuple{Int,Int}`: The window size around each chunk to consider for context building. See `?build_context` for more information. - `return_context::Bool`: If `true`, returns the context used for RAG along with the response. - `verbose::Bool`: If `true`, enables verbose logging. - `api_kwargs`: API parameters that will be forwarded to the API calls @@ -56,10 +92,12 @@ msg = airag(index, :RAGAnswerFromContext; question) # or simply msg = airag(index; question) ``` + +See also `build_index`, `build_context`, `CandidateChunks`, `find_closest`, `find_tags`, `rerank` """ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromContext; question::AbstractString, - top_k::Int = 3, + top_k::Int = 3, minimum_similarity::AbstractFloat = -1.0, tag_filter::Union{Symbol, Vector{String}, Regex, Nothing} = :auto, rerank_strategy::RerankingStrategy = Passthrough(), model_embedding::String = PT.MODEL_EMBEDDING, model_chat::String = PT.MODEL_CHAT, @@ -80,7 +118,7 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC _normalize; model = model_embedding, verbose, api_kwargs).content .|> Float32 # no need for Float64 - emb_candidates = find_closest(index, question_emb; top_k) + emb_candidates = find_closest(index, question_emb; top_k, minimum_similarity) tag_candidates = if tag_filter == :auto && !isnothing(tags(index)) && !isempty(model_metadata) @@ -113,16 +151,8 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC reranked_candidates = rerank(rerank_strategy, index, question, filtered_candidates) ## Build the context - context = String[] - for (i, position) in enumerate(reranked_candidates.positions) - ## Add surrounding chunks if they are from the same source (from `chunks_window_margin`) - chunks_ = chunks(index)[max(1, position - chunks_window_margin[1]):min(end, - position + chunks_window_margin[2])] - is_same_source = sources(index)[max(1, position - chunks_window_margin[1]):min(end, - position + chunks_window_margin[2])] .== sources(index)[position] - # add the ranking number, eg, 1. Context #1 - push!(context, "$(i). $(join(chunks_[is_same_source], "\n"))") - end + context = build_context(index, reranked_candidates; chunks_window_margin) + ## LLM call msg = aigenerate(rag_template; question, context = join(context, "\n\n"), model = model_chat, verbose, diff --git a/src/Experimental/RAGTools/retrieval.jl b/src/Experimental/RAGTools/retrieval.jl index db33a90e0..a3d136aa4 100644 --- a/src/Experimental/RAGTools/retrieval.jl +++ b/src/Experimental/RAGTools/retrieval.jl @@ -1,17 +1,35 @@ -"Finds the indices of chunks (represented by embeddings in `emb`) that are closest (cosine similarity) to query embedding (`query_emb`). Returns only `top_k` closest indices." +""" + find_closest(emb::AbstractMatrix{<:Real}, + query_emb::AbstractVector{<:Real}; + top_k::Int = 100, minimum_similarity::AbstractFloat = -1.0) + +Finds the indices of chunks (represented by embeddings in `emb`) that are closest (cosine similarity) to query embedding (`query_emb`). + +If `minimum_similarity` is provided, only indices with similarity greater than or equal to it are returned. +Similarity can be between -1 and 1 (-1 = completely opposite, 1 = exactly the same). + +Returns only `top_k` closest indices. +""" function find_closest(emb::AbstractMatrix{<:Real}, query_emb::AbstractVector{<:Real}; - top_k::Int = 100) + top_k::Int = 100, minimum_similarity::AbstractFloat = -1.0) # emb is an embedding matrix where the first dimension is the embedding dimension distances = query_emb' * emb |> vec positions = distances |> sortperm |> reverse |> x -> first(x, top_k) + if minimum_similarity > -1.0 + mask = distances[positions] .>= minimum_similarity + positions = positions[mask] + end return positions, distances[positions] end function find_closest(index::AbstractChunkIndex, query_emb::AbstractVector{<:Real}; - top_k::Int = 100) + top_k::Int = 100, minimum_similarity::AbstractFloat = -1.0) isnothing(embeddings(index)) && CandidateChunks(; index_id = index.id) - positions, distances = find_closest(embeddings(index), query_emb; top_k) + positions, distances = find_closest(embeddings(index), + query_emb; + top_k, + minimum_similarity) return CandidateChunks(index.id, positions, Float32.(distances)) end diff --git a/test/Experimental/RAGTools/evaluation.jl b/test/Experimental/RAGTools/evaluation.jl index 1a8ef5203..f9ea295e8 100644 --- a/test/Experimental/RAGTools/evaluation.jl +++ b/test/Experimental/RAGTools/evaluation.jl @@ -1,5 +1,7 @@ -using PromptingTools.Experimental.RAGTools: QAEvalItem +using PromptingTools.Experimental.RAGTools: QAItem, QAEvalItem, QAEvalResult using PromptingTools.Experimental.RAGTools: score_retrieval_hit, score_retrieval_rank +using PromptingTools.Experimental.RAGTools: build_qa_evals, run_qa_evals, chunks, sources +using PromptingTools.Experimental.RAGTools: JudgeAllScores, MetadataItem, MaybeMetadataItems @testset "QAEvalItem" begin empty_qa = QAEvalItem() @@ -8,6 +10,53 @@ using PromptingTools.Experimental.RAGTools: score_retrieval_hit, score_retrieval @test isvalid(full_qa) end +@testset "Base.show,JSON3.write" begin + # Helper function to simulate the IO capture for custom show methods + function capture_show(io::IOBuffer, x) + show(io, x) + return String(take!(io)) + end + + # Testing Base.show for QAItem + qa_item = QAItem("What is Julia?", + "Julia is a high-level, high-performance programming language.") + + test_output = capture_show(IOBuffer(), qa_item) + @test test_output == + "QAItem:\n question: What is Julia?\n answer: Julia is a high-level, high-performance programming language.\n" + json_output = JSON3.write(qa_item) + @test JSON3.read(json_output, QAItem) == qa_item + + # Testing Base.show for QAEvalItem + qa_eval_item = QAEvalItem(source = "testsource.jl", + context = "Julia is a high-level, high-performance programming language.", + question = "What is Julia?", + answer = "A language.") + + test_output = capture_show(IOBuffer(), qa_eval_item) + @test test_output == + "QAEvalItem:\n source: testsource.jl\n context: Julia is a high-level, high-performance programming language.\n question: What is Julia?\n answer: A language.\n" + json_output = JSON3.write(qa_eval_item) + @test JSON3.read(json_output, QAEvalItem) == qa_eval_item + + # Testing Base.show for QAEvalResult + params = Dict(:key1 => "value1", :key2 => 2) + qa_eval_result = QAEvalResult(source = "testsource.jl", + context = "Julia is amazing for technical computing.", + question = "Why is Julia good?", + answer = "Because of its speed and ease of use.", + retrieval_score = 0.89, + retrieval_rank = 1, + answer_score = 100.0, + parameters = params) + + test_output = capture_show(IOBuffer(), qa_eval_result) + @test test_output == + "QAEvalResult:\n source: testsource.jl\n context: Julia is amazing for technical computing.\n question: Why is Julia good?\n answer: Because of its speed and ease of use.\n retrieval_score: 0.89\n retrieval_rank: 1\n answer_score: 100.0\n parameters: Dict{Symbol, Any}(:key2 => 2, :key1 => \"value1\")\n" + json_output = JSON3.write(qa_eval_result) + @test JSON3.read(json_output, QAEvalResult) == qa_eval_result +end + @testset "score_retrieval_hit,score_retrieval_rank" begin orig_context = "I am a horse." candidate_context = ["Hello", "World", "I am a horse...."] @@ -22,4 +71,137 @@ end @test score_retrieval_rank(orig_context, candidate_context2) == 2 @test score_retrieval_rank(orig_context, candidate_context[1:2]) == nothing @test score_retrieval_rank(orig_context, candidate_context3) == nothing +end + +@testset "build_qa_evals" begin + # test with a mock server + PORT = rand(1000:2000) + PT.register_model!(; name = "mock-emb", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-meta", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-gen", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-qa", schema = PT.CustomOpenAISchema()) + PT.register_model!(; name = "mock-judge", schema = PT.CustomOpenAISchema()) + + echo_server = HTTP.serve!(PORT; verbose = -1) do req + content = JSON3.read(req.body) + + if content[:model] == "mock-gen" + user_msg = last(content[:messages]) + response = Dict(:choices => [Dict(:message => user_msg)], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + elseif content[:model] == "mock-emb" + # for i in 1:length(content[:input]) + response = Dict(:data => [Dict(:embedding => ones(Float32, 128))], + :usage => Dict(:total_tokens => length(content[:input]), + :prompt_tokens => length(content[:input]), + :completion_tokens => 0)) + elseif content[:model] == "mock-meta" + user_msg = last(content[:messages]) + response = Dict(:choices => [ + Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(MaybeMetadataItems([ + MetadataItem("yes", "category"), + ]))))), + ], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + elseif content[:model] == "mock-qa" + user_msg = last(content[:messages]) + response = Dict(:choices => [ + Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(QAItem("Question", + "Answer"))))), + ], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + elseif content[:model] == "mock-judge" + user_msg = last(content[:messages]) + response = Dict(:choices => [ + Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(JudgeAllScores(5, + 5, + 5, + 5, + 5, + "Some reasons", + 5.0))))), + ], + :model => content[:model], + :usage => Dict(:total_tokens => length(user_msg[:content]), + :prompt_tokens => length(user_msg[:content]), + :completion_tokens => 0)) + else + @info content + end + return HTTP.Response(200, JSON3.write(response)) + end + + # Index setup + index = ChunkIndex(; + sources = [".", ".", "."], + chunks = ["a", "b", "c"], + embeddings = zeros(128, 3), + tags = vcat(trues(2, 2), falses(1, 2)), + tags_vocab = ["yes", "no"],) + + # Test for successful Q&A extraction from document chunks + qa_evals = build_qa_evals(chunks(index), + sources(index), + instructions = "Some instructions.", + model = "mock-qa", + api_kwargs = (; url = "http://localhost:$(PORT)")) + + @test length(qa_evals) == length(chunks(index)) + @test all(getproperty.(qa_evals, :source) .== ".") + @test all(getproperty.(qa_evals, :context) == ["a", "b", "c"]) + @test all(getproperty.(qa_evals, :question) .== "Question") + @test all(getproperty.(qa_evals, :answer) .== "Answer") + + # Error checks + @test_throws AssertionError build_qa_evals(chunks(index), + String[]) + @test_throws AssertionError build_qa_evals(chunks(index), + String[]; qa_template = :BlankSystemUser) + + # Test run_qa_evals on 1 item + msg, ctx = airag(index; question = qa_evals[1].question, model_embedding = "mock-emb", + model_chat = "mock-gen", + model_metadata = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)"), + tag_filter = :auto, + extract_metadata = false, verbose = false, + return_context = true) + + result = run_qa_evals(qa_evals[1], ctx; + model_judge = "mock-judge", + api_kwargs = (; url = "http://localhost:$(PORT)"), + parameters_dict = Dict(:key1 => "value1", :key2 => 2)) + @test result.retrieval_score == 1.0 + @test result.retrieval_rank == 1 + @test result.answer_score == 5 + @test result.parameters == Dict(:key1 => "value1", :key2 => 2) + + # Test all evals at once + # results = run_qa_evals(index, qa_evals; model_judge = "mock-judge", + # api_kwargs = (; url = "http://localhost:$(PORT)")) + results = run_qa_evals(index, qa_evals; + airag_kwargs = (; + model_embedding = "mock-emb", + model_chat = "mock-gen", + model_metadata = "mock-meta"), + qa_evals_kwargs = (; model_judge = "mock-judge"), + api_kwargs = (; url = "http://localhost:$(PORT)"), + parameters_dict = Dict(:key1 => "value1", :key2 => 2)) + + @test length(results) == length(qa_evals) + @test all(getproperty.(results, :retrieval_score) .== 1.0) + @test all(getproperty.(results, :retrieval_rank) .== 1) + @test all(getproperty.(results, :answer_score) .== 5) + @test all(getproperty.(results, :parameters) .== + Ref(Dict(:key1 => "value1", :key2 => 2))) + # clean up + close(echo_server) end \ No newline at end of file diff --git a/test/Experimental/RAGTools/generation.jl b/test/Experimental/RAGTools/generation.jl index bfacf4cb5..51d70edbb 100644 --- a/test/Experimental/RAGTools/generation.jl +++ b/test/Experimental/RAGTools/generation.jl @@ -1,4 +1,32 @@ -using PromptingTools.Experimental.RAGTools: MaybeMetadataItems, MetadataItem +using PromptingTools.Experimental.RAGTools: MaybeMetadataItems, MetadataItem, build_context + +@testset "build_context" begin + index = ChunkIndex(; + sources = [".", ".", "."], + chunks = ["a", "b", "c"], + embeddings = zeros(128, 3), + tags = vcat(trues(2, 2), falses(1, 2)), + tags_vocab = ["yes", "no"],) + candidates = CandidateChunks(index.id, [1, 2], [0.1, 0.2]) + + # Standard Case + context = build_context(index, candidates) + expected_output = ["1. a\nb", + "2. a\nb\nc"] + @test context == expected_output + + # No Surrounding Chunks + context = build_context(index, candidates; chunks_window_margin = (0, 0)) + expected_output = ["1. a", + "2. b"] + @test context == expected_output + + # Wrong inputs + @test_throws AssertionError build_context(index, + candidates; + chunks_window_margin = (-1, 0)) +end + @testset "airag" begin # test with a mock server PORT = rand(1000:2000) diff --git a/test/Experimental/RAGTools/retrieval.jl b/test/Experimental/RAGTools/retrieval.jl index 9f21b561e..fcb9bc819 100644 --- a/test/Experimental/RAGTools/retrieval.jl +++ b/test/Experimental/RAGTools/retrieval.jl @@ -2,7 +2,7 @@ using PromptingTools.Experimental.RAGTools: find_closest, find_tags using PromptingTools.Experimental.RAGTools: Passthrough, rerank @testset "find_closest" begin - test_embeddings = [1.0 2.0; 3.0 4.0; 5.0 6.0] |> + test_embeddings = [1.0 2.0 -1.0; 3.0 4.0 -3.0; 5.0 6.0 -6.0] |> x -> mapreduce(normalize, hcat, eachcol(x)) query_embedding = [0.1, 0.35, 0.5] |> normalize positions, distances = find_closest(test_embeddings, query_embedding, top_k = 2) @@ -15,6 +15,11 @@ using PromptingTools.Experimental.RAGTools: Passthrough, rerank positions, _ = find_closest(test_embeddings, query_embedding, top_k = 5) @test length(positions) == size(test_embeddings, 2) + # Test with minimum_similarity + positions, _ = find_closest(test_embeddings, query_embedding, top_k = 5, + minimum_similarity = 0.995) + @test length(positions) == 1 + # Test behavior with edge values (top_k == 0) @test find_closest(test_embeddings, query_embedding, top_k = 0) == ([], []) end From b5ddf77010520f3ef75195d9ebc2fb8df29ade0c Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 12:33:50 +0100 Subject: [PATCH 079/251] update example --- docs/src/examples/building_RAG.md | 1 + examples/building_RAG.jl | 71 ++++++++++++++++--------------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/docs/src/examples/building_RAG.md b/docs/src/examples/building_RAG.md index 148f8c469..a14416093 100644 --- a/docs/src/examples/building_RAG.md +++ b/docs/src/examples/building_RAG.md @@ -100,6 +100,7 @@ evals = JSON3.read("examples/evals.json", Vector{RT.QAEvalItem}); ## Explore one Q&A pair Let's explore one evals item -- it's not the best quality but gives you the idea! + ````julia evals[1] ```` diff --git a/examples/building_RAG.jl b/examples/building_RAG.jl index 0f0c2640f..d868830f1 100644 --- a/examples/building_RAG.jl +++ b/examples/building_RAG.jl @@ -1,42 +1,50 @@ # # Building a Simple Retrieval-Augmented Generation (RAG) System with RAGTools -# Note: RAGTools is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package +# Let's build a Retrieval-Augmented Generation (RAG) chatbot, tailored to navigate and interact with the DataFrames.jl documentation. +# "RAG" is probably the most common and valuable pattern in Generative AI at the moment. + +# If you're not familiar with "RAG", start with this [article](https://towardsdatascience.com/add-your-own-data-to-an-llm-using-retrieval-augmented-generation-rag-b1958bf56a5a). ## Imports using LinearAlgebra, SparseArrays using PromptingTools -using PromptingTools.Experimental.RAGTools # Experimental! May change +## Note: RAGTools is still experimental and will change in the future. Ideally, they will be cleaned up and moved to a dedicated package +using PromptingTools.Experimental.RAGTools using JSON3, Serialization, DataFramesMeta using Statistics: mean const PT = PromptingTools const RT = PromptingTools.Experimental.RAGTools -# ## Ask questions E2E -# Let's put together a few copy&pasted text files from DataFrames.jl docs +# ## RAG in Two Lines + +# Let's put together a few text pages from DataFrames.jl docs. +# Simply go to [DataFrames.jl docs](https://dataframes.juliadata.org/stable/) and copy&paste a few pages into separate text files. Save them in the `examples/data` folder (see some example pages provided). Ideally, delete all the noise (like headers, footers, etc.) and keep only the text you want to use for the chatbot. Remember, garbage in, garbage out! + files = [ joinpath("examples", "data", "database_style_joins.txt"), joinpath("examples", "data", "what_is_dataframes.txt"), ] +## Build an index of chunks, embed them, and create a lookup index of metadata/tags for each chunk index = build_index(files; extract_metadata = false) -# Ask a question +# Let's ask a question +## Embeds the question, finds the closest chunks in the index, and generates an answer from the closest chunks answer = airag(index; question = "I like dplyr, what is the equivalent in Julia?") -# AIMessage("The equivalent package in Julia to the dplyr package in R is DataFrames.jl.") -# The equivalent package in Julia to the dplyr package in R is DataFrames.jl. # First RAG in two lines? Done! # # What does it do? -# - `build_index` will chunk the documents into smaller pieces, embed them into numbers (to be able to judge similarity of chunks) and, optionally, create a lookup index of metadata/tags for each chunk) +# - `build_index` will chunk the documents into smaller pieces, embed them into numbers (to be able to judge the similarity of chunks) and, optionally, create a lookup index of metadata/tags for each chunk) # - `index` is the result of this step and it holds your chunks, embeddings, and other metadata! Just show it :) # - `airag` will # - embed your question -# - find the closest chunks in the index -# - [OPTIONAL] extract any potential tags/filters from the question and apply them to filter down the potential candidates -# - [OPTIONAL] rerank the candidate chunks -# - generate an answer from the closest chunks +# - find the closest chunks in the index (use parameters `top_k` and `minimum_similarity` to tweak the "relevant" chunks) +# - [OPTIONAL] extracts any potential tags/filters from the question and applies them to filter down the potential candidates (use `extract_metadata=true` in `build_index`, you can also provide some filters explicitly via `tag_filter`) +# - [OPTIONAL] re-ranks the candidate chunks (define and provide your own `rerank_strategy`, eg Cohere ReRank API) +# - build a context from the closest chunks (use `chunks_window_margin` to tweak if we include preceding and succeeding chunks as well, see `?build_context` for more details) +# - generate an answer from the closest chunks (use `return_context=true` to see under the hood and debug your application) -# You should save the index for later! +# You should save the index for later to avoid re-embedding / re-extracting the document chunks! serialize("examples/index.jls", index) index = deserialize("examples/index.jls") @@ -64,22 +72,13 @@ evals = JSON3.read("examples/evals.json", Vector{RT.QAEvalItem}); # ## Explore one Q&A pair # Let's explore one evals item -- it's not the best but gives you the idea! +# evals[1] -# QAEvalItem: -# source: markdown/DataFrames/comparison_with_python.txt -# context: Comparisons -# This section compares DataFrames.jl with other data manipulation frameworks in Python, R, and Stata. - -# A sample data set can be created using the following code: - -# using DataFrames -# using Statistics -# question: What frameworks are compared with DataFrames.jl? -# answer: Python, R, and Stata # ## Evaluate this Q&A pair -## Let's answer and evaluate this QA item with the judge +# Let's evaluate this QA item with a "judge model" (often GPT-4 is used as a judge). + ## Note: that we used the same question, but generated a different context and answer via `airag` msg, ctx = airag(index; evals[1].question, return_context = true); @@ -107,8 +106,8 @@ x = run_qa_evals(evals[10], ctx; # ## Evaluate the whole set -# Let's run each question&answer through our eval loop in async (we do it only for the first 10) -# See the `?airag` for which parameters you can tweak, eg, top_k +# Let's run each question & answer through our eval loop in async (we do it only for the first 10 to save time). See the `?airag` for which parameters you can tweak, eg, `top_k` + results = asyncmap(evals[1:10]) do qa_item ## Generate an answer -- often you want the model_judge to be the highest quality possible, eg, "GPT-4 Turbo" (alias "gpt4t) msg, ctx = airag(index; qa_item.question, return_context = true, @@ -118,16 +117,20 @@ results = asyncmap(evals[1:10]) do qa_item run_qa_evals(qa_item, ctx; parameters_dict = Dict(:top_k => 3), verbose = false) end ## Note that the "failed" evals can show as "nothing", so make sure to handle them. -results = filter(!isnothing, results); +results = filter(x -> !isnothing(x.answer_score), results); + +# Note: You could also use the vectorized version `results = run_qa_evals(evals)` to evaluate all items at once. ## Let's take a simple average to calculate our score -@info "RAG Evals: $(length(results)) results, Avg. score: $(round(mean(x->x.answer_score, results);digits=1)), Retrieval score: $(100*round(mean(x->x.retrieval_score,results);digits=1))%" -# [ Info: RAG Evals: 10 results, Avg. score: 4.5, Retrieval score: 70.0% +@info "RAG Evals: $(length(results)) results, Avg. score: $(round(mean(x->x.answer_score, results);digits=1)), Retrieval score: $(100*round(Int,mean(x->x.retrieval_score,results)))%" +## [ Info: RAG Evals: 10 results, Avg. score: 4.6, Retrieval score: 100% + +# Note: The retrieval score is 100% only because we have two small documents and running on 10 items only. In practice, you would have a much larger document set and a much larger eval set, which would result in a more representative retrieval score. + +# You can also analyze the results in a DataFrame: -# or you can analyze it in a DataFrame df = DataFrame(results) -# 10×8 DataFrame -# Row │ source context ... +first(df, 5) # We're done for today! @@ -141,4 +144,4 @@ df = DataFrame(results) # - Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API)`) # - Improve the question embedding (eg, rephrase it, generate hypothetical answers and use them to find better context) # -# ... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) +# ... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) \ No newline at end of file From d8456be05575044ed8625da25f2ebfd3981c0f97 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 12:46:45 +0100 Subject: [PATCH 080/251] fix docs --- docs/make.jl | 6 +++++- docs/src/reference_experimental.md | 12 ++++++++++++ docs/src/reference_ragtools.md | 9 +++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 docs/src/reference_experimental.md create mode 100644 docs/src/reference_ragtools.md diff --git a/docs/make.jl b/docs/make.jl index 08c18a617..16b4d2456 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -31,7 +31,11 @@ makedocs(; "Building RAG Application" => "examples/building_RAG.md", ], "F.A.Q." => "frequently_asked_questions.md", - "Reference" => "reference.md", + "Reference" => [ + "PromptingTools.jl" => "reference.md", + "Experimental Modules" => "reference_experimental.md", + "RAGTools" => "reference_ragtools.md", + ], ]) deploydocs(; diff --git a/docs/src/reference_experimental.md b/docs/src/reference_experimental.md new file mode 100644 index 000000000..428775406 --- /dev/null +++ b/docs/src/reference_experimental.md @@ -0,0 +1,12 @@ +# Reference for Experimental Module + +Note: This module is experimental and may change in future releases. +The intention is for the functionality to be moved to separate packages over time. + +```@index +Modules = [PromptingTools.Experimental] +``` + +```@autodocs +Modules = [PromptingTools.Experimental] +``` diff --git a/docs/src/reference_ragtools.md b/docs/src/reference_ragtools.md new file mode 100644 index 000000000..5f7bd23d9 --- /dev/null +++ b/docs/src/reference_ragtools.md @@ -0,0 +1,9 @@ +# Reference for RAGTools + +```@index +Modules = [PromptingTools.Experimental.RAGTools] +``` + +```@autodocs +Modules = [PromptingTools.Experimental.RAGTools] +``` From 9a569bc60100b297925aaba72f1db194c24d02b6 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 12:52:10 +0100 Subject: [PATCH 081/251] improve tests --- test/Experimental/RAGTools/generation.jl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/Experimental/RAGTools/generation.jl b/test/Experimental/RAGTools/generation.jl index 51d70edbb..b19d03a44 100644 --- a/test/Experimental/RAGTools/generation.jl +++ b/test/Experimental/RAGTools/generation.jl @@ -95,7 +95,8 @@ end tag_filter = ["yes"], return_context = false) @test occursin("Time?", msg.content) - # different kwargs + + ## Test different kwargs msg, ctx = airag(index; question = "Time?", model_embedding = "mock-emb", model_chat = "mock-gen", model_metadata = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)"), @@ -111,6 +112,19 @@ end @test ctx.filtered_candidates.distances == 0.5ones(2) @test ctx.reranked_candidates.positions == [2, 1] # no change @test ctx.reranked_candidates.distances == 0.5ones(2) # no change + + ## Not tag filter + msg, ctx = airag(index; question = "Time?", model_embedding = "mock-emb", + model_chat = "mock-gen", + model_metadata = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)"), + tag_filter = nothing, + return_context = true) + @test ctx.context == ["1. b\nc", "2. a\nb\nc", "3. a\nb"] + @test ctx.emb_candidates.positions == [3, 2, 1] + @test ctx.emb_candidates.distances == zeros(3) + @test ctx.tag_candidates == nothing + @test ctx.filtered_candidates.positions == [3, 2, 1] #re-sort + @test ctx.reranked_candidates.positions == [3, 2, 1] # no change # clean up close(echo_server) end From d9183b9d1d78cb0a215dfe7ae4c31a9687924f3e Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 14:11:34 +0100 Subject: [PATCH 082/251] update docs --- docs/make.jl | 1 + docs/src/examples/building_RAG.md | 2 +- docs/src/examples/working_with_custom_apis.md | 69 + docs/src/examples/working_with_ollama.md | 4102 +---------------- examples/building_RAG.jl | 2 +- examples/working_with_ollama.jl | 1 + 6 files changed, 79 insertions(+), 4098 deletions(-) create mode 100644 docs/src/examples/working_with_custom_apis.md diff --git a/docs/make.jl b/docs/make.jl index 16b4d2456..e2ca988fb 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -28,6 +28,7 @@ makedocs(; "Various examples" => "examples/readme_examples.md", "Using AITemplates" => "examples/working_with_aitemplates.md", "Local models with Ollama.ai" => "examples/working_with_ollama.md", + "Custom APIs (Mistral, Llama.cpp)" => "examples/working_with_custom_apis.md", "Building RAG Application" => "examples/building_RAG.md", ], "F.A.Q." => "frequently_asked_questions.md", diff --git a/docs/src/examples/building_RAG.md b/docs/src/examples/building_RAG.md index a14416093..9108269de 100644 --- a/docs/src/examples/building_RAG.md +++ b/docs/src/examples/building_RAG.md @@ -217,7 +217,7 @@ We're done for today! - Add filtering for semantic similarity (embedding distance) to make sure we don't pick up irrelevant chunks in the context - Use multiple indices or a hybrid index (add a simple BM25 lookup from TextAnalysis.jl) - Data processing is the most important step - properly parsed and split text could make wonders -- Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API)`) +- Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API) - Improve the question embedding (eg, rephrase it, generate hypothetical answers and use them to find better context) ... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) diff --git a/docs/src/examples/working_with_custom_apis.md b/docs/src/examples/working_with_custom_apis.md new file mode 100644 index 000000000..d6d961552 --- /dev/null +++ b/docs/src/examples/working_with_custom_apis.md @@ -0,0 +1,69 @@ +# Custom APIs + +PromptingTools allows you to use any OpenAI-compatible API (eg, MistralAI), including a locally hosted one like the server from `llama.cpp`. + +````julia +using PromptingTools +const PT = PromptingTools +```` + +## Using MistralAI + +Mistral models have long been dominating the open-source space. They are now available via their API, so you can use them with PromptingTools.jl! + +```julia +msg = aigenerate("Say hi!"; model="mistral-tiny") +# [ Info: Tokens: 114 @ Cost: $0.0 in 0.9 seconds +# AIMessage("Hello there! I'm here to help answer any questions you might have, or assist you with tasks to the best of my abilities. How can I be of service to you today? If you have a specific question, feel free to ask and I'll do my best to provide accurate and helpful information. If you're looking for general assistance, I can help you find resources or information on a variety of topics. Let me know how I can help.") +``` + +It all just works, because we have registered the models in the `PromptingTools.MODEL_REGISTRY`! There are currently 4 models available: `mistral-tiny`, `mistral-small`, `mistral-medium`, `mistral-embed`. + +Under the hood, we use a dedicated schema `MistralOpenAISchema` that leverages most of the OpenAI-specific code base, so you can always provide that explicitly as the first argument: + +```julia +const PT = PromptingTools +msg = aigenerate(PT.MistralOpenAISchema(), "Say Hi!"; model="mistral-tiny", api_key=ENV["MISTRALAI_API_KEY"]) +``` +As you can see, we can load your API key either from the ENV or via the Preferences.jl mechanism (see `?PREFERENCES` for more information). + +## Using other OpenAI-compatible APIs + +MistralAI are not the only ones who mimic the OpenAI API! +There are many other exciting providers, eg, [Perplexity.ai](https://docs.perplexity.ai/), [Fireworks.ai](https://app.fireworks.ai/). + +As long as they are compatible with the OpenAI API (eg, sending `messages` with `role` and `content` keys), you can use them with PromptingTools.jl by using `schema = CustomOpenAISchema()`: + +```julia +# Set your API key and the necessary base URL for the API +api_key = "..." +provider_url = "..." # provider API URL +prompt = "Say hi!" +msg = aigenerate(PT.CustomOpenAISchema(), prompt; model="", api_key, api_kwargs=(; url=provider_url)) +``` + +> [!TIP] +> If you register the model names with `PT.register_model!`, you won't have to keep providing the `schema` manually. + +Note: At the moment, we only support `aigenerate` and `aiembed` functions. + +## Using llama.cpp server + +In line with the above, you can also use the [`llama.cpp` server](https://github.com/ggerganov/llama.cpp/blob/master/examples/server/README.md). + +It is a bit more technically demanding because you need to "compile" `llama.cpp` first, but it will always have the latest models and it is quite fast (eg, faster than Ollama, which uses llama.cpp under the hood but has some extra overhead). + +Start your server in a command line (`-m` refers to the model file, `-c` is the context length, `-ngl` is the number of layers to offload to GPU): + +```bash +./server -m models/mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf -c 2048 -ngl 99 +``` + +Then simply access it via PromptingTools: + +```julia +msg = aigenerate(PT.CustomOpenAISchema(), "Count to 5 and say hi!"; api_kwargs=(; url="http://localhost:8080/v1")) +``` + +> [!TIP] +> If you register the model names with `PT.register_model!`, you won't have to keep providing the `schema` manually. It can be any `model` name, because the model is actually selected when you start the server in the terminal. \ No newline at end of file diff --git a/docs/src/examples/working_with_ollama.md b/docs/src/examples/working_with_ollama.md index 4f5f92d3b..51b195a4d 100644 --- a/docs/src/examples/working_with_ollama.md +++ b/docs/src/examples/working_with_ollama.md @@ -23,6 +23,10 @@ Eg, "llama2" or "openhermes2.5-mistral" (see `PT.list_registry()` and `PT.list_a Note: You must download these models prior to using them with `ollama pull ` in your Terminal. +> [!TIP] +> If you use Apple Mac M1-3, make sure to provide `api_kwargs=(; options=(; num_gpu=99))` to make sure the whole model is offloaded on your GPU. Current default is 1, which makes some models unusable. Example for running Mixtral: +> `msg = aigenerate(PT.OllamaManagedSchema(), "Count from 1 to 5 and then say hi."; model="dolphin-mixtral:8x7b-v2.5-q4_K_M", api_kwargs=(; options=(; num_gpu=99)))` + ## Text Generation with aigenerate ### Simple message @@ -148,4106 +152,12 @@ tasks = asyncmap(docs) do doc msg = aiembed(schema, doc; model) end embedding = mapreduce(x -> x.content, hcat, tasks) +size(embedding) ```` ```` 4096×2 Matrix{Float64}: - -2.45508 -2.45508 - -2.46738 -2.46738 - 0.850981 0.850981 - 0.709954 0.709954 - 0.806159 0.806159 - 1.41638 1.41638 - 1.09515 1.09515 - -1.77543 -1.77543 - 1.16811 1.16811 - 1.68691 1.68691 - -1.03008 -1.03008 - -1.27389 -1.27389 - 1.749 1.749 - 1.94468 1.94468 - 1.3547 1.3547 - 1.37603 1.37603 - -0.743637 -0.743637 - 0.805151 0.805151 - -0.108866 -0.108866 - 0.836891 0.836891 - 2.84637 2.84637 - -1.33009 -1.33009 - -1.16621 -1.16621 - -1.64225 -1.64225 - -3.23381 -3.23381 - 1.87149 1.87149 - 1.62256 1.62256 - -1.92032 -1.92032 - 1.92751 1.92751 - -0.596136 -0.596136 - 2.67425 2.67425 - -0.0169855 -0.0169855 - -0.74862 -0.74862 - 1.38466 1.38466 - -1.43319 -1.43319 - -0.0358144 -0.0358144 - -2.61001 -2.61001 - 0.979845 0.979845 - -2.50828 -2.50828 - -0.19873 -0.19873 - 1.56007 1.56007 - -2.9497 -2.9497 - 1.18661 1.18661 - -0.0459045 -0.0459045 - -1.89088 -1.89088 - 0.134144 0.134144 - 1.46773 1.46773 - 0.169044 0.169044 - 1.83185 1.83185 - 1.29546 1.29546 - -0.223334 -0.223334 - 0.248241 0.248241 - -0.143776 -0.143776 - -1.26674 -1.26674 - -2.34766 -2.34766 - -0.233582 -0.233582 - -1.32809 -1.32809 - -3.1517 -3.1517 - -1.37292 -1.37292 - -0.296508 -0.296508 - -0.730617 -0.730617 - 1.33757 1.33757 - -0.0650454 -0.0650454 - -1.69394 -1.69394 - -0.634744 -0.634744 - -1.43693 -1.43693 - 1.04292 1.04292 - 3.70178 3.70178 - 1.58817 1.58817 - -1.71241 -1.71241 - 1.28312 1.28312 - 1.28754 1.28754 - 0.079447 0.079447 - 0.0501985 0.0501985 - -2.16351 -2.16351 - 0.157908 0.157908 - 1.63121 1.63121 - -0.755454 -0.755454 - -0.490053 -0.490053 - -0.205053 -0.205053 - -2.08433 -2.08433 - 0.724874 0.724874 - 0.0719429 0.0719429 - -1.17219 -1.17219 - 0.440574 0.440574 - -0.939899 -0.939899 - -0.539892 -0.539892 - -0.378413 -0.378413 - 0.233923 0.233923 - 1.76957 1.76957 - -3.07904 -3.07904 - -2.3768 -2.3768 - -2.55772 -2.55772 - -1.41162 -1.41162 - 0.0823545 0.0823545 - 1.55678 1.55678 - 0.0568478 0.0568478 - 0.840122 0.840122 - 1.06037 1.06037 - 0.965553 0.965553 - 0.830964 0.830964 - -0.754283 -0.754283 - -0.1191 -0.1191 - 0.0280245 0.0280245 - -0.143816 -0.143816 - 1.27753 1.27753 - 0.066933 0.066933 - 0.926294 0.926294 - 0.743463 0.743463 - -0.559796 -0.559796 - 0.901292 0.901292 - 0.457328 0.457328 - -0.365485 -0.365485 - 0.218589 0.218589 - -1.26814 -1.26814 - -1.61122 -1.61122 - 1.82385 1.82385 - -0.722799 -0.722799 - -0.744419 -0.744419 - -0.369159 -0.369159 - -0.0910514 -0.0910514 - 1.38917 1.38917 - -0.176995 -0.176995 - -0.737644 -0.737644 - -1.51984 -1.51984 - 1.43837 1.43837 - -2.17809 -2.17809 - 1.78125 1.78125 - 1.61911 1.61911 - 2.56386 2.56386 - -0.265214 -0.265214 - -0.0391067 -0.0391067 - -1.15711 -1.15711 - 1.87313 1.87313 - -1.92623 -1.92623 - 3.0698 3.0698 - 3.35805 3.35805 - 0.843137 0.843137 - 2.88871 2.88871 - -0.309429 -0.309429 - 0.251267 0.251267 - 0.350675 0.350675 - -0.0698121 -0.0698121 - -2.383 -2.383 - 0.955251 0.955251 - 0.219921 0.219921 - -3.57961 -3.57961 - 0.809841 0.809841 - -0.171057 -0.171057 - 2.60038 2.60038 - 0.123726 0.123726 - 0.541886 0.541886 - -0.67232 -0.67232 - 1.65081 1.65081 - 1.2956 1.2956 - -0.936224 -0.936224 - -1.43537 -1.43537 - 1.30422 1.30422 - 1.645 1.645 - -0.59586 -0.59586 - -2.30588 -2.30588 - 0.265308 0.265308 - -1.16852 -1.16852 - 0.572658 0.572658 - -2.48747 -2.48747 - 1.75643 1.75643 - -0.208687 -0.208687 - -0.589351 -0.589351 - 1.38263 1.38263 - 0.0407013 0.0407013 - -0.478184 -0.478184 - 1.1551 1.1551 - 2.66743 2.66743 - -0.0347208 -0.0347208 - 0.258393 0.258393 - 0.376756 0.376756 - -1.91087 -1.91087 - 1.58246 1.58246 - -2.55877 -2.55877 - 0.29497 0.29497 - -0.679837 -0.679837 - 2.75608 2.75608 - 0.846178 0.846178 - 0.120587 0.120587 - -0.931332 -0.931332 - -0.460664 -0.460664 - -1.41767 -1.41767 - -0.0370647 -0.0370647 - 2.85593 2.85593 - 0.93859 0.93859 - 1.06093 1.06093 - -0.806263 -0.806263 - -0.276273 -0.276273 - 0.190741 0.190741 - 0.642197 0.642197 - -0.146854 -0.146854 - 0.235867 0.235867 - 1.53559 1.53559 - 3.23839 3.23839 - 2.51139 2.51139 - 2.69216 2.69216 - -0.808452 -0.808452 - 0.131485 0.131485 - 1.79253 1.79253 - -1.489 -1.489 - 1.36115 1.36115 - 0.689342 0.689342 - 1.93437 1.93437 - 1.1515 1.1515 - -0.267202 -0.267202 - -2.93755 -2.93755 - -0.353072 -0.353072 - 0.27407 0.27407 - -1.99885 -1.99885 - 2.30033 2.30033 - -2.08776 -2.08776 - -5.84525 -5.84525 - -0.163065 -0.163065 - -1.84699 -1.84699 - 0.272231 0.272231 - 0.178446 0.178446 - -2.07568 -2.07568 - 1.64194 1.64194 - 2.61154 2.61154 - 2.61646 2.61646 - 1.49662 1.49662 - -0.762681 -0.762681 - -1.58585 -1.58585 - -2.56334 -2.56334 - 2.90547 2.90547 - 0.907315 0.907315 - -4.51614 -4.51614 - -1.84281 -1.84281 - -0.770806 -0.770806 - -4.84157 -4.84157 - 1.12599 1.12599 - -2.35923 -2.35923 - -1.65892 -1.65892 - -1.57506 -1.57506 - 2.73717 2.73717 - -2.01069 -2.01069 - 2.1204 2.1204 - -1.01274 -1.01274 - -1.54509 -1.54509 - -1.11102 -1.11102 - 2.27304 2.27304 - -1.7203 -1.7203 - -0.141955 -0.141955 - -1.93008 -1.93008 - 4.40994 4.40994 - -0.843562 -0.843562 - 1.8763 1.8763 - -1.15709 -1.15709 - 1.93146 1.93146 - 0.0839072 0.0839072 - -1.57024 -1.57024 - -0.373742 -0.373742 - -3.81842 -3.81842 - -0.654545 -0.654545 - -0.555125 -0.555125 - 2.62493 2.62493 - -0.299043 -0.299043 - 0.572533 0.572533 - 0.93598 0.93598 - -0.831565 -0.831565 - -0.324564 -0.324564 - 0.938522 0.938522 - 1.63603 1.63603 - 1.73752 1.73752 - 3.90209 3.90209 - 2.96495 2.96495 - -1.92878 -1.92878 - -0.620347 -0.620347 - -0.617516 -0.617516 - 0.0554883 0.0554883 - 1.50602 1.50602 - 3.24908 3.24908 - -4.00377 -4.00377 - 0.611538 0.611538 - 0.395783 0.395783 - 0.840638 0.840638 - 0.6627 0.6627 - -1.06035 -1.06035 - 1.23494 1.23494 - -1.09427 -1.09427 - -0.944445 -0.944445 - -0.149559 -0.149559 - 0.364951 0.364951 - -2.74365 -2.74365 - -1.70194 -1.70194 - -0.597655 -0.597655 - 0.10956 0.10956 - 0.2841 0.2841 - -1.70202 -1.70202 - 0.815583 0.815583 - -1.3173 -1.3173 - -0.350691 -0.350691 - 0.323236 0.323236 - 0.423569 0.423569 - 0.135633 0.135633 - -0.894987 -0.894987 - 3.82014 3.82014 - -1.37686 -1.37686 - -1.11328 -1.11328 - -1.42841 -1.42841 - 0.486355 0.486355 - 1.00883 1.00883 - -2.19186 -2.19186 - 1.60446 1.60446 - 1.44481 1.44481 - 0.50944 0.50944 - 2.35551 2.35551 - -1.09942 -1.09942 - 0.449232 0.449232 - -2.47599 -2.47599 - 1.4625 1.4625 - -0.60117 -0.60117 - -4.33734 -4.33734 - 1.09776 1.09776 - -1.773 -1.773 - 0.664505 0.664505 - 0.358341 0.358341 - -2.10807 -2.10807 - -0.43586 -0.43586 - 1.39689 1.39689 - 0.276272 0.276272 - -0.500288 -0.500288 - -1.5217 -1.5217 - 0.054699 0.054699 - 1.1783 1.1783 - 0.276075 0.276075 - -3.61916 -3.61916 - 1.01459 1.01459 - 0.817065 0.817065 - 2.77712 2.77712 - 0.911377 0.911377 - -3.64983 -3.64983 - 1.20056 1.20056 - 2.43472 2.43472 - -1.40853 -1.40853 - -0.398067 -0.398067 - 0.44781 0.44781 - -0.983103 -0.983103 - -1.01117 -1.01117 - -1.36821 -1.36821 - -1.62301 -1.62301 - 1.19024 1.19024 - 0.176577 0.176577 - -0.0266686 -0.0266686 - 0.173558 0.173558 - 1.28662 1.28662 - 0.80163 0.80163 - 0.89478 0.89478 - 0.165779 0.165779 - 0.201296 0.201296 - 3.06203 3.06203 - -0.799754 -0.799754 - 0.654919 0.654919 - -0.175649 -0.175649 - 0.748995 0.748995 - -0.155388 -0.155388 - 1.37946 1.37946 - 0.286727 0.286727 - -1.56082 -1.56082 - -4.17062 -4.17062 - 1.82237 1.82237 - 1.25286 1.25286 - 0.911904 0.911904 - 0.203066 0.203066 - 2.03452 2.03452 - -0.683902 -0.683902 - -1.83145 -1.83145 - 3.99144 3.99144 - -1.61397 -1.61397 - 0.525864 0.525864 - -0.0760927 -0.0760927 - -1.51654 -1.51654 - -1.26449 -1.26449 - -1.62594 -1.62594 - -0.0456586 -0.0456586 - 1.15446 1.15446 - -4.01131 -4.01131 - -1.69864 -1.69864 - -0.626423 -0.626423 - -0.768044 -0.768044 - 1.42236 1.42236 - 0.590386 0.590386 - -0.734404 -0.734404 - -0.991776 -0.991776 - 0.846241 0.846241 - -1.58825 -1.58825 - 1.64396 1.64396 - 0.506507 0.506507 - 2.04938 2.04938 - -0.142975 -0.142975 - -0.248108 -0.248108 - -0.395868 -0.395868 - -1.09874 -1.09874 - 3.86285 3.86285 - 0.401966 0.401966 - -1.78 -1.78 - -0.142962 -0.142962 - -1.40281 -1.40281 - 1.00746 1.00746 - -4.80569 -4.80569 - -2.78505 -2.78505 - -1.13022 -1.13022 - 1.64658 1.64658 - 3.87433 3.87433 - -0.490487 -0.490487 - -1.87168 -1.87168 - 0.324756 0.324756 - -0.887698 -0.887698 - 0.527319 0.527319 - -0.819513 -0.819513 - 0.66582 0.66582 - -1.01098 -1.01098 - -1.77557 -1.77557 - -1.86591 -1.86591 - -1.66033 -1.66033 - -2.26619 -2.26619 - 0.396155 0.396155 - -1.53582 -1.53582 - -2.12857 -2.12857 - 0.127346 0.127346 - 0.198356 0.198356 - -2.24282 -2.24282 - -0.431639 -0.431639 - -0.529176 -0.529176 - -1.24013 -1.24013 - -0.00600927 -0.00600927 - -0.970315 -0.970315 - 0.825078 0.825078 - 0.158507 0.158507 - 0.126577 0.126577 - -2.27452 -2.27452 - 0.43308 0.43308 - -2.68747 -2.68747 - -0.63088 -0.63088 - -0.300087 -0.300087 - 1.62468 1.62468 - -2.08314 -2.08314 - 0.537432 0.537432 - -1.51959 -1.51959 - -1.36428 -1.36428 - -1.37559 -1.37559 - -0.270443 -0.270443 - 1.92657 1.92657 - -0.397027 -0.397027 - -1.23493 -1.23493 - -0.816994 -0.816994 - 1.4852 1.4852 - -1.34173 -1.34173 - -0.308267 -0.308267 - 2.45801 2.45801 - -1.7382 -1.7382 - -2.47404 -2.47404 - -0.791644 -0.791644 - -2.12967 -2.12967 - -1.6631 -1.6631 - 0.335476 0.335476 - 1.22216 1.22216 - -0.059939 -0.059939 - -3.55093 -3.55093 - -0.582763 -0.582763 - 2.68949 2.68949 - 2.01951 2.01951 - 0.494128 0.494128 - -0.116969 -0.116969 - 0.304021 0.304021 - -2.11024 -2.11024 - 0.47012 0.47012 - -1.61832 -1.61832 - 2.74031 2.74031 - -0.812687 -0.812687 - 2.5097 2.5097 - 1.04051 1.04051 - 0.281989 0.281989 - -2.74931 -2.74931 - 2.62989 2.62989 - -0.802226 -0.802226 - 1.62871 1.62871 - -1.06605 -1.06605 - 0.121078 0.121078 - -0.632968 -0.632968 - -0.216849 -0.216849 - -0.322418 -0.322418 - 3.07053 3.07053 - 4.40286 4.40286 - -0.872669 -0.872669 - -0.127031 -0.127031 - 0.78 0.78 - 0.0148648 0.0148648 - -0.629511 -0.629511 - -1.05654 -1.05654 - 1.60099 1.60099 - -0.0685499 -0.0685499 - 0.854439 0.854439 - -0.187685 -0.187685 - -1.08173 -1.08173 - 0.614483 0.614483 - 2.87317 2.87317 - -3.44885 -3.44885 - 0.0392022 0.0392022 - -0.472168 -0.472168 - 2.172 2.172 - 0.997629 0.997629 - -0.604507 -0.604507 - 1.74349 1.74349 - 1.26225 1.26225 - 0.45734 0.45734 - -0.0242855 -0.0242855 - -1.98687 -1.98687 - 0.0133762 0.0133762 - 1.10497 1.10497 - 1.48159 1.48159 - -0.00565977 -0.00565977 - -0.273567 -0.273567 - 0.844543 0.844543 - 0.875793 0.875793 - -2.28312 -2.28312 - -1.57407 -1.57407 - 1.17889 1.17889 - 0.00492724 0.00492724 - 0.999081 0.999081 - 3.84377 3.84377 - 1.04097 1.04097 - 1.57247 1.57247 - -0.470368 -0.470368 - 0.00163007 0.00163007 - -1.51918 -1.51918 - -0.00223237 -0.00223237 - 1.31357 1.31357 - -0.189349 -0.189349 - 1.28289 1.28289 - 2.97546 2.97546 - -2.90089 -2.90089 - 1.57488 1.57488 - 1.34312 1.34312 - 1.52898 1.52898 - -1.64515 -1.64515 - -1.15666 -1.15666 - -1.2686 -1.2686 - -1.8123 -1.8123 - 0.996818 0.996818 - 1.02553 1.02553 - -0.0632497 -0.0632497 - 1.59041 1.59041 - -0.527615 -0.527615 - -1.48674 -1.48674 - -0.620959 -0.620959 - -0.342495 -0.342495 - -1.93695 -1.93695 - -3.64679 -3.64679 - -0.15153 -0.15153 - -0.942336 -0.942336 - -0.236199 -0.236199 - 2.03071 2.03071 - 0.316865 0.316865 - -1.04498 -1.04498 - 2.27925 2.27925 - -2.82399 -2.82399 - -1.33688 -1.33688 - 0.886898 0.886898 - -0.636226 -0.636226 - -1.85773 -1.85773 - -0.0714151 -0.0714151 - 3.68168 3.68168 - 2.40809 2.40809 - 1.97447 1.97447 - 0.203908 0.203908 - 1.04359 1.04359 - -0.0644891 -0.0644891 - -0.310849 -0.310849 - -0.326476 -0.326476 - -1.17233 -1.17233 - 0.883281 0.883281 - -0.801131 -0.801131 - -0.554541 -0.554541 - 0.411139 0.411139 - -2.13628 -2.13628 - -1.85167 -1.85167 - -0.073298 -0.073298 - 3.95706 3.95706 - -1.3341 -1.3341 - 1.69608 1.69608 - 0.0962806 0.0962806 - -2.74171 -2.74171 - 0.616342 0.616342 - 3.68453 3.68453 - -4.27901 -4.27901 - -0.366352 -0.366352 - -0.951105 -0.951105 - 2.67429 2.67429 - -0.167208 -0.167208 - 0.574208 0.574208 - 0.965369 0.965369 - 0.572892 0.572892 - -0.0812238 -0.0812238 - 0.126916 0.126916 - 0.0906106 0.0906106 - -2.21247 -2.21247 - 0.330044 0.330044 - -1.31031 -1.31031 - -0.196966 -0.196966 - -0.587011 -0.587011 - 1.42082 1.42082 - 1.58975 1.58975 - 0.568703 0.568703 - 3.14349 3.14349 - 0.883494 0.883494 - 1.21594 1.21594 - 1.11572 1.11572 - -1.01462 -1.01462 - 0.0228628 0.0228628 - -0.777921 -0.777921 - -0.0185091 -0.0185091 - 2.9342 2.9342 - -1.63432 -1.63432 - -0.0897289 -0.0897289 - 0.173623 0.173623 - 1.46671 1.46671 - 0.263784 0.263784 - -0.760012 -0.760012 - 0.179174 0.179174 - 0.318366 0.318366 - -1.09259 -1.09259 - -2.41019 -2.41019 - 2.18473 2.18473 - -0.406192 -0.406192 - -1.71917 -1.71917 - 3.65316 3.65316 - 0.163596 0.163596 - -0.0996923 -0.0996923 - -1.36222 -1.36222 - -0.946804 -0.946804 - 1.19432 1.19432 - 2.71731 2.71731 - 0.0474303 0.0474303 - -2.81076 -2.81076 - -4.10022 -4.10022 - 1.54347 1.54347 - -0.870202 -0.870202 - -0.521006 -0.521006 - 0.406167 0.406167 - -0.578452 -0.578452 - -1.34979 -1.34979 - 0.307851 0.307851 - 0.0160142 0.0160142 - -0.0118573 -0.0118573 - -0.0629544 -0.0629544 - -2.30284 -2.30284 - 3.1494 3.1494 - -1.74032 -1.74032 - -0.499308 -0.499308 - -0.637685 -0.637685 - -0.458922 -0.458922 - -1.81869 -1.81869 - 3.54851 3.54851 - 1.02612 1.02612 - -0.0730208 -0.0730208 - 1.59768 1.59768 - -0.123013 -0.123013 - 1.97723 1.97723 - -0.620977 -0.620977 - -0.634377 -0.634377 - -1.66552 -1.66552 - -1.96801 -1.96801 - -2.06075 -2.06075 - 1.59643 1.59643 - 0.496652 0.496652 - 3.12764 3.12764 - 1.53881 1.53881 - -0.603705 -0.603705 - -1.67414 -1.67414 - -1.44337 -1.44337 - 0.413483 0.413483 - 0.513435 0.513435 - 1.94637 1.94637 - -1.04941 -1.04941 - -1.18587 -1.18587 - 0.0255263 0.0255263 - 3.32276 3.32276 - -1.03282 -1.03282 - -1.7708 -1.7708 - -0.139377 -0.139377 - 0.0487106 0.0487106 - 2.69087 2.69087 - 0.0466001 0.0466001 - 3.96837 3.96837 - -0.437004 -0.437004 - 0.463263 0.463263 - -0.390073 -0.390073 - 1.03124 1.03124 - 1.94068 1.94068 - 0.951059 0.951059 - 0.561724 0.561724 - -0.435045 -0.435045 - 1.02984 1.02984 - 1.7263 1.7263 - 0.538726 0.538726 - 1.56557 1.56557 - 1.15078 1.15078 - 1.68213 1.68213 - 0.16588 0.16588 - 1.85933 1.85933 - 1.12831 1.12831 - -1.91047 -1.91047 - 0.0938072 0.0938072 - 0.709282 0.709282 - -0.491057 -0.491057 - -0.708452 -0.708452 - 2.89155 2.89155 - -1.42685 -1.42685 - -0.386733 -0.386733 - -2.32159 -2.32159 - 0.751944 0.751944 - 0.314597 0.314597 - -1.90364 -1.90364 - -2.28503 -2.28503 - -0.983842 -0.983842 - 0.478018 0.478018 - -1.06256 -1.06256 - -1.27427 -1.27427 - 1.92344 1.92344 - 1.93071 1.93071 - -0.634679 -0.634679 - 1.07079 1.07079 - -2.73159 -2.73159 - 1.13198 1.13198 - -1.13381 -1.13381 - -0.275951 -0.275951 - -2.56787 -2.56787 - 2.96264 2.96264 - -1.68603 -1.68603 - 0.0565433 0.0565433 - -1.78596 -1.78596 - -1.64427 -1.64427 - -0.109207 -0.109207 - -0.123623 -0.123623 - 0.696342 0.696342 - -0.865633 -0.865633 - 1.55017 1.55017 - 3.57577 3.57577 - 0.574881 0.574881 - 0.311736 0.311736 - 1.12859 1.12859 - -0.660953 -0.660953 - 0.0603463 0.0603463 - 1.19673 1.19673 - -1.68319 -1.68319 - 0.0573934 0.0573934 - -3.0577 -3.0577 - -0.0811733 -0.0811733 - -1.35226 -1.35226 - 2.20395 2.20395 - -5.04376 -5.04376 - 0.957934 0.957934 - -2.67944 -2.67944 - 0.0583888 0.0583888 - -1.36195 -1.36195 - -1.14978 -1.14978 - 2.52851 2.52851 - -0.221427 -0.221427 - 1.39957 1.39957 - 1.97268 1.97268 - 1.58743 1.58743 - -1.25526 -1.25526 - -0.574435 -0.574435 - 0.879556 0.879556 - -1.60156 -1.60156 - 0.96617 0.96617 - -0.704615 -0.704615 - -1.13136 -1.13136 - -1.29642 -1.29642 - -0.48684 -0.48684 - -0.603142 -0.603142 - -1.55072 -1.55072 - 0.199318 0.199318 - -1.25072 -1.25072 - -0.0617283 -0.0617283 - 1.28993 1.28993 - 2.55354 2.55354 - -0.484267 -0.484267 - -0.629886 -0.629886 - 0.738121 0.738121 - 1.97734 1.97734 - -4.42185 -4.42185 - 1.38591 1.38591 - -0.403438 -0.403438 - 0.113538 0.113538 - -0.379133 -0.379133 - 1.15787 1.15787 - 2.57502 2.57502 - -2.24232 -2.24232 - 0.0712947 0.0712947 - -2.6907 -2.6907 - -2.27887 -2.27887 - -0.813216 -0.813216 - 2.34541 2.34541 - -1.11775 -1.11775 - 0.768486 0.768486 - -2.0542 -2.0542 - 1.52947 1.52947 - -4.3352 -4.3352 - -0.485814 -0.485814 - -1.67194 -1.67194 - 0.233108 0.233108 - 0.530897 0.530897 - 2.05494 2.05494 - 1.49262 1.49262 - 0.395652 0.395652 - -1.30437 -1.30437 - 0.500992 0.500992 - 1.28381 1.28381 - 0.393163 0.393163 - 3.96419 3.96419 - 0.434971 0.434971 - -0.228664 -0.228664 - 0.0615298 0.0615298 - -1.58079 -1.58079 - -1.01419 -1.01419 - -2.10373 -2.10373 - 0.0069584 0.0069584 - 0.382241 0.382241 - -0.0795624 -0.0795624 - 1.59346 1.59346 - 1.22138 1.22138 - -1.90972 -1.90972 - -0.365729 -0.365729 - -0.568999 -0.568999 - 1.42262 1.42262 - 0.282054 0.282054 - -1.06812 -1.06812 - 2.11136 2.11136 - 2.22528 2.22528 - 0.958127 0.958127 - -0.211857 -0.211857 - 0.354436 0.354436 - 6.28299 6.28299 - -1.88306 -1.88306 - 1.16816 1.16816 - -0.724852 -0.724852 - 2.05035 2.05035 - 1.1614 1.1614 - -0.0774035 -0.0774035 - -0.991458 -0.991458 - -2.224 -2.224 - -0.912683 -0.912683 - -2.33781 -2.33781 - -1.32562 -1.32562 - 2.07222 2.07222 - 1.86425 1.86425 - -1.12945 -1.12945 - -1.42154 -1.42154 - -1.83308 -1.83308 - -0.80144 -0.80144 - 0.0659904 0.0659904 - 0.46477 0.46477 - 0.516478 0.516478 - -0.714539 -0.714539 - 0.563923 0.563923 - -1.11843 -1.11843 - 0.24327 0.24327 - 1.9211 1.9211 - 1.5536 1.5536 - -1.83635 -1.83635 - 0.245734 0.245734 - 0.894795 0.894795 - 0.449533 0.449533 - 1.83397 1.83397 - 1.76346 1.76346 - 3.16185 3.16185 - 0.559932 0.559932 - -0.882377 -0.882377 - -0.717065 -0.717065 - -0.268154 -0.268154 - 1.02418 1.02418 - 2.46569 2.46569 - -0.737093 -0.737093 - 0.872719 0.872719 - 0.156786 0.156786 - -1.68971 -1.68971 - -0.762942 -0.762942 - 0.0251112 0.0251112 - 0.612489 0.612489 - -3.30371 -3.30371 - -0.698961 -0.698961 - -2.20203 -2.20203 - -1.89275 -1.89275 - 0.0291887 0.0291887 - -0.61895 -0.61895 - -2.4545 -2.4545 - 0.716685 0.716685 - -0.63556 -0.63556 - 0.957159 0.957159 - -0.273225 -0.273225 - -0.441594 -0.441594 - -0.899482 -0.899482 - -2.85088 -2.85088 - -0.660739 -0.660739 - 1.20197 1.20197 - -0.421759 -0.421759 - 2.36317 2.36317 - -2.68519 -2.68519 - 1.48605 1.48605 - -0.680547 -0.680547 - -3.61262 -3.61262 - 0.200643 0.200643 - 2.05747 2.05747 - 2.03866 2.03866 - -1.53579 -1.53579 - -0.300568 -0.300568 - 2.17815 2.17815 - 0.889736 0.889736 - -0.269451 -0.269451 - 3.85836 3.85836 - -0.0832182 -0.0832182 - 0.575183 0.575183 - -0.90211 -0.90211 - -0.352595 -0.352595 - 2.23507 2.23507 - 0.863045 0.863045 - 1.76841 1.76841 - 0.159141 0.159141 - -0.187019 -0.187019 - -0.605252 -0.605252 - 0.950606 0.950606 - 0.126904 0.126904 - -1.33489 -1.33489 - -2.65456 -2.65456 - 0.973579 0.973579 - 1.11206 1.11206 - 2.71157 2.71157 - -1.35922 -1.35922 - -2.09924 -2.09924 - -0.261339 -0.261339 - -0.71599 -0.71599 - 0.957563 0.957563 - -2.38311 -2.38311 - -1.47771 -1.47771 - 1.71136 1.71136 - -1.54574 -1.54574 - -0.66235 -0.66235 - -0.727048 -0.727048 - 1.7508 1.7508 - 3.09727 3.09727 - 2.60078 2.60078 - -1.96873 -1.96873 - 0.924622 0.924622 - 0.287636 0.287636 - 0.201588 0.201588 - 0.638481 0.638481 - 1.3465 1.3465 - 1.56972 1.56972 - 1.14971 1.14971 - -1.89176 -1.89176 - 2.74371 2.74371 - 1.23168 1.23168 - 3.05252 3.05252 - 0.608012 0.608012 - -1.21346 -1.21346 - -0.220215 -0.220215 - 1.27374 1.27374 - 1.89971 1.89971 - -0.315658 -0.315658 - -1.53046 -1.53046 - -0.815528 -0.815528 - 1.11711 1.11711 - -1.86098 -1.86098 - -2.62764 -2.62764 - -1.94186 -1.94186 - -1.01812 -1.01812 - 0.10927 0.10927 - 1.83491 1.83491 - 3.75411 3.75411 - -0.329943 -0.329943 - 1.04999 1.04999 - -0.0572795 -0.0572795 - -1.19271 -1.19271 - -0.0640939 -0.0640939 - 3.24013 3.24013 - 1.19772 1.19772 - 0.139843 0.139843 - -1.14619 -1.14619 - -0.346295 -0.346295 - 0.314854 0.314854 - -1.86965 -1.86965 - -2.5997 -2.5997 - 0.860876 0.860876 - 0.779138 0.779138 - -1.48689 -1.48689 - -0.041387 -0.041387 - 0.415675 0.415675 - -0.996154 -0.996154 - -0.210727 -0.210727 - 3.01432 3.01432 - -0.818496 -0.818496 - -1.77229 -1.77229 - -5.1652 -5.1652 - -0.430049 -0.430049 - 0.110492 0.110492 - -0.633977 -0.633977 - 5.47398 5.47398 - -0.827223 -0.827223 - 0.316164 0.316164 - -1.33416 -1.33416 - -2.58731 -2.58731 - 0.331473 0.331473 - -0.495747 -0.495747 - -0.578983 -0.578983 - 2.24558 2.24558 - -0.465562 -0.465562 - -0.88282 -0.88282 - 1.00385 1.00385 - -0.986489 -0.986489 - -0.356386 -0.356386 - 1.56415 1.56415 - -3.3696 -3.3696 - -2.39466 -2.39466 - -6.8684 -6.8684 - -0.705285 -0.705285 - -0.473808 -0.473808 - 1.34126 1.34126 - 1.17112 1.17112 - -0.638047 -0.638047 - -2.18021 -2.18021 - -0.748828 -0.748828 - 1.33976 1.33976 - 0.838517 0.838517 - 0.229636 0.229636 - -1.10193 -1.10193 - -0.713668 -0.713668 - 0.0447604 0.0447604 - -0.126929 -0.126929 - -2.65546 -2.65546 - -0.407056 -0.407056 - -0.224281 -0.224281 - -0.206263 -0.206263 - 1.59537 1.59537 - 2.12341 2.12341 - -0.165041 -0.165041 - 0.745372 0.745372 - -2.61492 -2.61492 - 0.764695 0.764695 - -0.721477 -0.721477 - 2.71105 2.71105 - 0.400047 0.400047 - -0.620564 -0.620564 - -0.676275 -0.676275 - -1.65892 -1.65892 - 0.153766 0.153766 - -0.789512 -0.789512 - 0.718952 0.718952 - -1.3161 -1.3161 - 1.3924 1.3924 - -0.752873 -0.752873 - 1.47481 1.47481 - 0.764075 0.764075 - 1.13314 1.13314 - 1.11338 1.11338 - 0.0911534 0.0911534 - 0.358637 0.358637 - 0.443527 0.443527 - -1.65442 -1.65442 - -1.18768 -1.18768 - -0.671452 -0.671452 - -0.10452 -0.10452 - 2.63571 2.63571 - -0.365343 -0.365343 - 0.989189 0.989189 - 1.016 1.016 - 0.02238 0.02238 - 2.5152 2.5152 - -0.502349 -0.502349 - 2.07812 2.07812 - -2.60912 -2.60912 - 0.039301 0.039301 - 2.21963 2.21963 - -1.25433 -1.25433 - 1.04089 1.04089 - 2.15635 2.15635 - -1.17901 -1.17901 - 0.464251 0.464251 - 0.523713 0.523713 - 2.6929 2.6929 - -3.17565 -3.17565 - -0.0704547 -0.0704547 - 0.316088 0.316088 - -0.595716 -0.595716 - -0.962247 -0.962247 - 2.07635 2.07635 - -0.707047 -0.707047 - -1.95768 -1.95768 - -1.05889 -1.05889 - -0.346038 -0.346038 - -0.519161 -0.519161 - 0.679303 0.679303 - 0.668559 0.668559 - -1.18768 -1.18768 - 1.05391 1.05391 - 0.495406 0.495406 - 0.775206 0.775206 - 1.21855 1.21855 - -0.188646 -0.188646 - 2.78981 2.78981 - 0.276702 0.276702 - -2.51357 -2.51357 - 0.893815 0.893815 - -0.600099 -0.600099 - 1.27468 1.27468 - 0.176813 0.176813 - -0.97769 -0.97769 - -0.831947 -0.831947 - -0.78568 -0.78568 - 1.54688 1.54688 - -1.66614 -1.66614 - 1.74662 1.74662 - -0.144336 -0.144336 - 3.25977 3.25977 - 0.604924 0.604924 - -1.08699 -1.08699 - 0.337396 0.337396 - 0.6638 0.6638 - -1.03335 -1.03335 - -2.12209 -2.12209 - -4.10625 -4.10625 - 0.575479 0.575479 - -0.703679 -0.703679 - 0.455751 0.455751 - -3.87142 -3.87142 - 1.82789 1.82789 - -1.17806 -1.17806 - -0.555556 -0.555556 - -3.14948 -3.14948 - 2.48156 2.48156 - 1.53411 1.53411 - -0.306063 -0.306063 - 0.182748 0.182748 - -4.68966 -4.68966 - 1.65411 1.65411 - -0.471352 -0.471352 - 2.08115 2.08115 - -1.14689 -1.14689 - 1.7511 1.7511 - 2.02855 2.02855 - -0.842776 -0.842776 - 0.708793 0.708793 - -0.663495 -0.663495 - 0.752657 0.752657 - -0.837532 -0.837532 - 1.94326 1.94326 - -1.0195 -1.0195 - -0.189062 -0.189062 - 0.443252 0.443252 - 1.53346 1.53346 - -1.06344 -1.06344 - 2.09193 2.09193 - 1.11391 1.11391 - 0.973692 0.973692 - 1.60649 1.60649 - -2.98173 -2.98173 - -1.30708 -1.30708 - 0.686241 0.686241 - 1.44115 1.44115 - -0.804022 -0.804022 - 0.128653 0.128653 - 0.148315 0.148315 - 1.05242 1.05242 - 4.81345 4.81345 - -0.0360308 -0.0360308 - -0.0977653 -0.0977653 - 2.50497 2.50497 - 0.0427317 0.0427317 - 1.07616 1.07616 - 2.6184 2.6184 - 4.16012 4.16012 - 1.94137 1.94137 - 0.168671 0.168671 - 0.249589 0.249589 - 2.29888 2.29888 - -2.58131 -2.58131 - 1.02287 1.02287 - -3.24177 -3.24177 - 0.0501678 0.0501678 - -3.08706 -3.08706 - -1.81322 -1.81322 - -3.21173 -3.21173 - -0.605546 -0.605546 - 1.80962 1.80962 - 1.1928 1.1928 - 1.16219 1.16219 - -0.502315 -0.502315 - -2.12962 -2.12962 - 1.82909 1.82909 - -0.515471 -0.515471 - 0.465682 0.465682 - -2.27668 -2.27668 - -0.794018 -0.794018 - 0.906492 0.906492 - -2.97682 -2.97682 - 1.76548 1.76548 - -2.14784 -2.14784 - -1.70079 -1.70079 - 0.575641 0.575641 - 2.18061 2.18061 - 0.721652 0.721652 - 3.34416 3.34416 - 2.92373 2.92373 - -2.27608 -2.27608 - -0.147619 -0.147619 - 2.27896 2.27896 - 3.13783 3.13783 - 0.53847 0.53847 - 1.59368 1.59368 - -2.92919 -2.92919 - -0.665617 -0.665617 - 2.21631 2.21631 - -0.132953 -0.132953 - 0.255364 0.255364 - 1.26044 1.26044 - -0.234991 -0.234991 - 0.0326571 0.0326571 - 1.95529 1.95529 - -0.147709 -0.147709 - 0.837403 0.837403 - 0.264494 0.264494 - -0.79803 -0.79803 - 1.24209 1.24209 - 4.16292 4.16292 - 2.90203 2.90203 - -1.77443 -1.77443 - -1.25963 -1.25963 - -1.97697 -1.97697 - -0.0228958 -0.0228958 - 0.961171 0.961171 - -1.98808 -1.98808 - 1.9206 1.9206 - 0.948345 0.948345 - -0.26941 -0.26941 - -0.895958 -0.895958 - -0.0969616 -0.0969616 - -1.8459 -1.8459 - 1.06631 1.06631 - 0.582305 0.582305 - 0.949072 0.949072 - -1.12433 -1.12433 - 1.75585 1.75585 - -4.27514 -4.27514 - 0.193812 0.193812 - -0.879839 -0.879839 - 0.740636 0.740636 - -1.46844 -1.46844 - 1.1903 1.1903 - 0.14461 0.14461 - -2.83538 -2.83538 - 1.90454 1.90454 - -1.91853 -1.91853 - -0.606642 -0.606642 - 2.28632 2.28632 - 1.92542 1.92542 - 2.53635 2.53635 - 1.55833 1.55833 - 1.52359 1.52359 - 0.777676 0.777676 - 3.06456 3.06456 - 0.288197 0.288197 - 1.2306 1.2306 - -1.57984 -1.57984 - -2.09081 -2.09081 - 2.21411 2.21411 - -0.0590976 -0.0590976 - 0.0967886 0.0967886 - 1.56966 1.56966 - 1.0458 1.0458 - 2.13399 2.13399 - 0.582962 0.582962 - -3.258 -3.258 - 1.31176 1.31176 - 2.12008 2.12008 - 0.420016 0.420016 - 1.16769 1.16769 - 0.322813 0.322813 - -2.08123 -2.08123 - -0.366867 -0.366867 - -0.863194 -0.863194 - 2.1781 2.1781 - -0.607621 -0.607621 - -1.46271 -1.46271 - -0.645887 -0.645887 - 2.07943 2.07943 - 1.66221 1.66221 - 1.00057 1.00057 - 1.96566 1.96566 - 1.40844 1.40844 - 1.99744 1.99744 - -1.17224 -1.17224 - 0.688052 0.688052 - -1.56755 -1.56755 - -1.1927 -1.1927 - 1.44395 1.44395 - 0.739251 0.739251 - 2.06788 2.06788 - 0.603058 0.603058 - -0.749518 -0.749518 - 0.935585 0.935585 - -0.671683 -0.671683 - -0.489752 -0.489752 - 0.984702 0.984702 - 0.571385 0.571385 - -0.386589 -0.386589 - 1.61408 1.61408 - 2.16478 2.16478 - 2.3196 2.3196 - -2.31076 -2.31076 - -0.186898 -0.186898 - 1.61571 1.61571 - -0.369523 -0.369523 - -2.21236 -2.21236 - -3.84185 -3.84185 - -0.0851651 -0.0851651 - 0.295692 0.295692 - -2.87568 -2.87568 - 2.97524 2.97524 - 0.974503 0.974503 - 2.36206 2.36206 - -1.7699 -1.7699 - 1.04331 1.04331 - -1.8445 -1.8445 - -1.58605 -1.58605 - -0.410044 -0.410044 - -0.588594 -0.588594 - -0.916676 -0.916676 - 0.493802 0.493802 - 0.0781382 0.0781382 - -0.958667 -0.958667 - 0.237942 0.237942 - 0.344577 0.344577 - -0.155517 -0.155517 - -0.0599809 -0.0599809 - 0.625756 0.625756 - 1.083 1.083 - -1.55908 -1.55908 - -0.0447221 -0.0447221 - 2.29293 2.29293 - -0.818719 -0.818719 - -0.431673 -0.431673 - 0.807989 0.807989 - -1.49538 -1.49538 - -0.415386 -0.415386 - -0.368204 -0.368204 - 0.822768 0.822768 - -1.04757 -1.04757 - 1.50985 1.50985 - 0.591814 0.591814 - 0.576176 0.576176 - 2.2397 2.2397 - -2.87292 -2.87292 - -1.05271 -1.05271 - -0.584545 -0.584545 - 23.6334 23.6334 - -1.94422 -1.94422 - -2.4793 -2.4793 - -0.461402 -0.461402 - -2.68547 -2.68547 - 1.27863 1.27863 - 0.349233 0.349233 - -1.58019 -1.58019 - 1.55179 1.55179 - -0.206459 -0.206459 - -0.985779 -0.985779 - 0.943214 0.943214 - -0.110792 -0.110792 - -0.0489029 -0.0489029 - 2.33366 2.33366 - -1.6701 -1.6701 - 0.201543 0.201543 - -0.0789827 -0.0789827 - -1.27944 -1.27944 - -1.77649 -1.77649 - 0.872661 0.872661 - 2.52858 2.52858 - 2.94238 2.94238 - 1.27241 1.27241 - -0.58158 -0.58158 - 1.37193 1.37193 - 2.03604 2.03604 - 2.76534 2.76534 - 1.36309 1.36309 - 0.910546 0.910546 - 1.2274 1.2274 - 0.501399 0.501399 - 1.69964 1.69964 - -0.0684446 -0.0684446 - -0.271516 -0.271516 - -2.72402 -2.72402 - -0.962159 -0.962159 - -0.396473 -0.396473 - 0.924897 0.924897 - -1.1069 -1.1069 - 2.62532 2.62532 - -0.393446 -0.393446 - -0.136789 -0.136789 - -0.151452 -0.151452 - -0.096789 -0.096789 - 2.1524 2.1524 - 1.68668 1.68668 - -0.730035 -0.730035 - 1.92158 1.92158 - -3.87326 -3.87326 - -0.221315 -0.221315 - -0.355481 -0.355481 - -0.775173 -0.775173 - 1.7409 1.7409 - 1.3288 1.3288 - -12.0217 -12.0217 - 0.583705 0.583705 - 0.184282 0.184282 - 0.417036 0.417036 - -0.956676 -0.956676 - 0.520674 0.520674 - -0.899164 -0.899164 - 3.50664 3.50664 - -0.636307 -0.636307 - -1.73476 -1.73476 - 1.29043 1.29043 - 0.472702 0.472702 - -1.79694 -1.79694 - -1.04534 -1.04534 - 1.10865 1.10865 - -1.43563 -1.43563 - -1.20749 -1.20749 - 1.26766 1.26766 - 0.673491 0.673491 - -0.934399 -0.934399 - 2.26572 2.26572 - -1.46043 -1.46043 - 2.209 2.209 - -1.00799 -1.00799 - -0.148943 -0.148943 - -1.1825 -1.1825 - -1.21252 -1.21252 - 0.282581 0.282581 - 0.41457 0.41457 - 2.20195 2.20195 - 0.0617071 0.0617071 - -0.550251 -0.550251 - -1.06407 -1.06407 - 0.642124 0.642124 - -2.1808 -2.1808 - 0.495721 0.495721 - 0.932684 0.932684 - 2.11964 2.11964 - 2.48226 2.48226 - 1.99288 1.99288 - -0.40534 -0.40534 - -0.830009 -0.830009 - 0.694545 0.694545 - -1.87861 -1.87861 - 0.656508 0.656508 - 1.45896 1.45896 - 0.831577 0.831577 - 1.56499 1.56499 - 0.220691 0.220691 - -1.53346 -1.53346 - 2.11808 2.11808 - -0.172959 -0.172959 - 2.30349 2.30349 - -0.415921 -0.415921 - -0.412111 -0.412111 - -1.08472 -1.08472 - 0.878408 0.878408 - -1.59121 -1.59121 - 1.27521 1.27521 - 1.49535 1.49535 - 1.01611 1.01611 - 0.826557 0.826557 - 0.413962 0.413962 - -0.418091 -0.418091 - -1.27713 -1.27713 - -1.72911 -1.72911 - -0.680069 -0.680069 - -3.18506 -3.18506 - 0.144236 0.144236 - -1.3908 -1.3908 - -1.37211 -1.37211 - -0.583177 -0.583177 - -2.50307 -2.50307 - 0.214101 0.214101 - -0.742307 -0.742307 - 0.806269 0.806269 - -0.58649 -0.58649 - -1.13406 -1.13406 - 0.589182 0.589182 - -1.41751 -1.41751 - 0.38813 0.38813 - -0.761625 -0.761625 - -1.82823 -1.82823 - -2.76721 -2.76721 - -0.555275 -0.555275 - 2.20053 2.20053 - 0.989203 0.989203 - -1.94756 -1.94756 - -3.27283 -3.27283 - 1.82038 1.82038 - 1.73285 1.73285 - -3.36146 -3.36146 - -0.0434622 -0.0434622 - -3.1662 -3.1662 - -2.27172 -2.27172 - -1.64158 -1.64158 - 1.36217 1.36217 - 0.0614365 0.0614365 - -1.51214 -1.51214 - 0.525372 0.525372 - -2.38711 -2.38711 - 0.91023 0.91023 - 0.642573 0.642573 - 0.787917 0.787917 - -0.165626 -0.165626 - -2.5386 -2.5386 - -0.0847667 -0.0847667 - -1.73854 -1.73854 - -0.629933 -0.629933 - 0.0476356 0.0476356 - -2.96352 -2.96352 - 2.43105 2.43105 - 3.07803 3.07803 - 1.61165 1.61165 - 1.80835 1.80835 - 2.6522 2.6522 - 2.42881 2.42881 - 1.95263 1.95263 - -0.627659 -0.627659 - -2.94554 -2.94554 - -0.584281 -0.584281 - 0.174889 0.174889 - 1.63772 1.63772 - -2.90996 -2.90996 - -1.19518 -1.19518 - -0.30899 -0.30899 - -1.44569 -1.44569 - 0.380009 0.380009 - -1.09017 -1.09017 - 0.401782 0.401782 - -1.01077 -1.01077 - -0.391996 -0.391996 - -0.284267 -0.284267 - -0.281839 -0.281839 - 0.0555518 0.0555518 - 1.16644 1.16644 - -2.17393 -2.17393 - -3.02707 -3.02707 - -0.744337 -0.744337 - 0.614164 0.614164 - -1.26162 -1.26162 - -0.892012 -0.892012 - -1.38321 -1.38321 - -0.690443 -0.690443 - -0.190017 -0.190017 - 0.652475 0.652475 - -0.665589 -0.665589 - -0.958746 -0.958746 - 0.211427 0.211427 - 0.172027 0.172027 - -0.183977 -0.183977 - -4.92239 -4.92239 - 0.234765 0.234765 - -0.128532 -0.128532 - -0.397255 -0.397255 - 0.637882 0.637882 - -0.665182 -0.665182 - 0.989375 0.989375 - 13.9866 13.9866 - 0.507452 0.507452 - 1.82993 1.82993 - -0.661663 -0.661663 - -0.315201 -0.315201 - 0.0298377 0.0298377 - -1.02559 -1.02559 - 1.67329 1.67329 - 0.770169 0.770169 - -0.567072 -0.567072 - -0.822084 -0.822084 - -0.695256 -0.695256 - -0.81899 -0.81899 - 0.00983914 0.00983914 - 3.47498 3.47498 - -0.0516518 -0.0516518 - 1.07953 1.07953 - -0.286307 -0.286307 - -0.901457 -0.901457 - 0.111325 0.111325 - 0.724452 0.724452 - -0.672886 -0.672886 - 0.821217 0.821217 - -3.77372 -3.77372 - 0.461435 0.461435 - -0.231648 -0.231648 - 1.41987 1.41987 - 1.03119 1.03119 - 0.93142 0.93142 - -0.913693 -0.913693 - 2.17759 2.17759 - 1.63487 1.63487 - -0.0814173 -0.0814173 - -1.10167 -1.10167 - 0.479772 0.479772 - 0.542071 0.542071 - 0.226907 0.226907 - 0.484866 0.484866 - -2.79594 -2.79594 - 1.17344 1.17344 - 0.448936 0.448936 - -0.425432 -0.425432 - -0.816027 -0.816027 - 1.11008 1.11008 - 2.16526 2.16526 - 0.574023 0.574023 - 3.34678 3.34678 - 3.05037 3.05037 - -1.49271 -1.49271 - -1.14698 -1.14698 - 0.556459 0.556459 - -1.01214 -1.01214 - -0.309133 -0.309133 - -0.536957 -0.536957 - 1.62871 1.62871 - -2.88469 -2.88469 - -0.0979174 -0.0979174 - -3.37567 -3.37567 - 2.46043 2.46043 - -0.730684 -0.730684 - -0.900807 -0.900807 - -2.45852 -2.45852 - -0.108725 -0.108725 - -1.01572 -1.01572 - -3.68841 -3.68841 - 0.290852 0.290852 - 1.68984 1.68984 - 3.35964 3.35964 - -1.31673 -1.31673 - 0.321869 0.321869 - 2.34154 2.34154 - -1.67666 -1.67666 - -0.035619 -0.035619 - -2.13567 -2.13567 - 1.3469 1.3469 - 0.153865 0.153865 - 1.14801 1.14801 - -0.713299 -0.713299 - -0.412727 -0.412727 - -1.01501 -1.01501 - -0.578133 -0.578133 - 0.684314 0.684314 - 3.48687 3.48687 - 1.21238 1.21238 - -1.95149 -1.95149 - -3.36448 -3.36448 - -0.819665 -0.819665 - 0.277849 0.277849 - 1.29952 1.29952 - -0.717767 -0.717767 - -0.0224495 -0.0224495 - -0.95711 -0.95711 - -2.57496 -2.57496 - 0.856373 0.856373 - 0.938171 0.938171 - -0.492205 -0.492205 - -1.45455 -1.45455 - 1.95963 1.95963 - -0.169591 -0.169591 - 1.93841 1.93841 - -0.0293303 -0.0293303 - -1.06165 -1.06165 - 0.0963018 0.0963018 - -0.0253168 -0.0253168 - -0.615891 -0.615891 - -1.84572 -1.84572 - 2.04114 2.04114 - 0.709038 0.709038 - -11.8503 -11.8503 - 1.30469 1.30469 - 1.29953 1.29953 - -0.0864412 -0.0864412 - -2.55483 -2.55483 - 0.0868521 0.0868521 - -0.472505 -0.472505 - -2.22982 -2.22982 - 0.675526 0.675526 - 0.903289 0.903289 - 2.1813 2.1813 - -0.731082 -0.731082 - 1.23144 1.23144 - 0.211691 0.211691 - -0.447575 -0.447575 - -0.266834 -0.266834 - 0.777422 0.777422 - 2.13011 2.13011 - 1.14053 1.14053 - -1.64879 -1.64879 - 0.253671 0.253671 - 1.37881 1.37881 - -3.47648 -3.47648 - 0.120876 0.120876 - 0.706163 0.706163 - -0.408708 -0.408708 - 1.63921 1.63921 - -0.0897068 -0.0897068 - -0.36319 -0.36319 - -2.72738 -2.72738 - 0.778532 0.778532 - -1.44104 -1.44104 - -0.477193 -0.477193 - 2.99272 2.99272 - -1.93478 -1.93478 - 0.7366 0.7366 - -0.461307 -0.461307 - 1.34756 1.34756 - 0.575798 0.575798 - 0.243039 0.243039 - 0.00184323 0.00184323 - -4.65195 -4.65195 - -0.0233881 -0.0233881 - 2.27189 2.27189 - -0.454542 -0.454542 - -0.621287 -0.621287 - -1.95901 -1.95901 - -1.59841 -1.59841 - -1.11109 -1.11109 - 0.276918 0.276918 - 0.822635 0.822635 - 0.399364 0.399364 - -3.59 -3.59 - 0.63147 0.63147 - -0.122051 -0.122051 - -2.99397 -2.99397 - -0.199531 -0.199531 - 1.70567 1.70567 - -0.802968 -0.802968 - 1.54216 1.54216 - 5.01761 5.01761 - 0.593876 0.593876 - 1.4851 1.4851 - -0.16419 -0.16419 - -2.67396 -2.67396 - -1.4692 -1.4692 - 3.53628 3.53628 - 0.842369 0.842369 - -1.10273 -1.10273 - 1.38417 1.38417 - -0.654059 -0.654059 - 3.03409 3.03409 - -0.599392 -0.599392 - -2.90447 -2.90447 - -1.42843 -1.42843 - 1.0605 1.0605 - 0.648235 0.648235 - -1.72383 -1.72383 - -0.356193 -0.356193 - -0.538367 -0.538367 - -3.79779 -3.79779 - 3.63493 3.63493 - -0.464008 -0.464008 - 0.347418 0.347418 - -0.467898 -0.467898 - 0.672538 0.672538 - -0.160574 -0.160574 - -3.60299 -3.60299 - 2.72608 2.72608 - 0.337715 0.337715 - -0.859081 -0.859081 - 4.38695 4.38695 - 2.18067 2.18067 - -1.61338 -1.61338 - -0.448975 -0.448975 - -1.84397 -1.84397 - 0.0152013 0.0152013 - 0.667211 0.667211 - -0.0943334 -0.0943334 - 0.860503 0.860503 - -0.115682 -0.115682 - -0.284645 -0.284645 - 3.97725 3.97725 - 1.98285 1.98285 - 0.576249 0.576249 - -1.60365 -1.60365 - 6.83394 6.83394 - 3.17895 3.17895 - 2.29932 2.29932 - 0.077594 0.077594 - -0.862484 -0.862484 - -0.455567 -0.455567 - 1.0918 1.0918 - -1.05352 -1.05352 - -0.503157 -0.503157 - -1.28519 -1.28519 - -2.55219 -2.55219 - -0.511736 -0.511736 - -0.601855 -0.601855 - 1.38829 1.38829 - 1.72822 1.72822 - 2.44367 2.44367 - -0.876967 -0.876967 - 1.32143 1.32143 - -0.345428 -0.345428 - 0.158257 0.158257 - -0.502065 -0.502065 - 0.602091 0.602091 - 0.546799 0.546799 - 0.144201 0.144201 - 1.28249 1.28249 - -0.766982 -0.766982 - -2.91678 -2.91678 - -0.420817 -0.420817 - 0.282388 0.282388 - -2.35922 -2.35922 - -0.959736 -0.959736 - 0.970576 0.970576 - 1.28214 1.28214 - -1.38422 -1.38422 - -1.28912 -1.28912 - -1.55508 -1.55508 - 2.60786 2.60786 - -0.174129 -0.174129 - -4.97013 -4.97013 - -3.13894 -3.13894 - -1.0113 -1.0113 - 0.29012 0.29012 - 0.301534 0.301534 - -1.36587 -1.36587 - 0.592223 0.592223 - -0.393313 -0.393313 - 1.54523 1.54523 - 0.489536 0.489536 - 0.431823 0.431823 - -0.698724 -0.698724 - 0.597885 0.597885 - -1.28348 -1.28348 - 9.20118 9.20118 - 0.523845 0.523845 - 1.64891 1.64891 - 1.26397 1.26397 - -0.0614047 -0.0614047 - 0.0327276 0.0327276 - -0.4776 -0.4776 - -2.24761 -2.24761 - 1.49423 1.49423 - -1.70954 -1.70954 - -0.174842 -0.174842 - 0.35175 0.35175 - 0.549776 0.549776 - -1.97708 -1.97708 - -0.797726 -0.797726 - -0.880683 -0.880683 - 1.92272 1.92272 - -1.05489 -1.05489 - -2.33809 -2.33809 - -1.59936 -1.59936 - -0.840616 -0.840616 - 2.6777 2.6777 - 0.86962 0.86962 - -1.01078 -1.01078 - -0.936281 -0.936281 - -1.24079 -1.24079 - -2.68717 -2.68717 - 0.22939 0.22939 - 0.75135 0.75135 - 1.07189 1.07189 - -0.0904491 -0.0904491 - -2.72094 -2.72094 - 0.259405 0.259405 - 0.801323 0.801323 - 1.79821 1.79821 - 0.13453 0.13453 - -0.294662 -0.294662 - 1.39746 1.39746 - -0.876874 -0.876874 - 0.411162 0.411162 - -1.2239 -1.2239 - -0.85717 -0.85717 - 0.50512 0.50512 - 1.59327 1.59327 - -1.76924 -1.76924 - 2.85557 2.85557 - -2.69548 -2.69548 - -1.5875 -1.5875 - -0.796946 -0.796946 - -0.335648 -0.335648 - -0.512417 -0.512417 - 2.13108 2.13108 - -1.85765 -1.85765 - -2.89643 -2.89643 - 1.76352 1.76352 - 0.575773 0.575773 - 1.59789 1.59789 - 1.42267 1.42267 - 0.497862 0.497862 - -0.567461 -0.567461 - 1.36287 1.36287 - 2.10155 2.10155 - -1.18077 -1.18077 - 2.49273 2.49273 - -1.94928 -1.94928 - 0.324793 0.324793 - 0.140937 0.140937 - 2.33415 2.33415 - -0.0985955 -0.0985955 - 1.71064 1.71064 - -0.274678 -0.274678 - 0.82142 0.82142 - 1.97483 1.97483 - 1.33147 1.33147 - 1.25382 1.25382 - -1.92684 -1.92684 - 0.633816 0.633816 - -1.74887 -1.74887 - -0.935251 -0.935251 - 0.406894 0.406894 - 0.630239 0.630239 - 1.49081 1.49081 - -1.7665 -1.7665 - 0.468253 0.468253 - -0.829623 -0.829623 - 0.683713 0.683713 - 0.343987 0.343987 - 1.87329 1.87329 - -0.46613 -0.46613 - 2.83798 2.83798 - 1.64433 1.64433 - 0.303556 0.303556 - -0.810505 -0.810505 - -0.548167 -0.548167 - 1.77963 1.77963 - 0.0793537 0.0793537 - -3.29027 -3.29027 - -0.183904 -0.183904 - -1.61751 -1.61751 - 0.860805 0.860805 - -0.722543 -0.722543 - 0.0263393 0.0263393 - -1.53128 -1.53128 - -0.242737 -0.242737 - 0.319961 0.319961 - 2.17752 2.17752 - 0.845624 0.845624 - 1.91994 1.91994 - 0.258224 0.258224 - 0.256254 0.256254 - -0.64679 -0.64679 - 1.51706 1.51706 - 1.2567 1.2567 - -0.0595054 -0.0595054 - 3.78386 3.78386 - 0.0964969 0.0964969 - 0.822442 0.822442 - 2.36122 2.36122 - 1.19663 1.19663 - 1.45557 1.45557 - -0.487415 -0.487415 - 2.80363 2.80363 - 0.354397 0.354397 - 2.064 2.064 - -1.60732 -1.60732 - 0.152285 0.152285 - -1.06767 -1.06767 - -2.89226 -2.89226 - -0.923556 -0.923556 - 0.129633 0.129633 - 0.414714 0.414714 - 0.313557 0.313557 - -0.63598 -0.63598 - -1.24145 -1.24145 - 0.445823 0.445823 - -1.18621 -1.18621 - -0.824399 -0.824399 - 0.0348621 0.0348621 - 1.72359 1.72359 - 3.23846 3.23846 - -0.982701 -0.982701 - -2.8631 -2.8631 - -0.247987 -0.247987 - 1.10508 1.10508 - -0.335101 -0.335101 - 1.59819 1.59819 - 0.363252 0.363252 - -1.22112 -1.22112 - 2.40608 2.40608 - 2.14135 2.14135 - 0.926073 0.926073 - -0.616568 -0.616568 - 0.829974 0.829974 - 4.0788 4.0788 - 1.97497 1.97497 - -0.943884 -0.943884 - 0.770483 0.770483 - 1.58248 1.58248 - 2.10963 2.10963 - -3.33186 -3.33186 - 1.52246 1.52246 - -1.38448 -1.38448 - 0.178628 0.178628 - 2.35971 2.35971 - 1.35345 1.35345 - 2.2217 2.2217 - -1.6579 -1.6579 - 0.752211 0.752211 - 1.60611 1.60611 - 0.224136 0.224136 - 0.8147 0.8147 - -6.54244 -6.54244 - 2.03158 2.03158 - 0.334614 0.334614 - -0.849204 -0.849204 - 0.573662 0.573662 - 0.834273 0.834273 - 1.98059 1.98059 - 4.59366 4.59366 - -0.0696824 -0.0696824 - -1.12577 -1.12577 - -0.392295 -0.392295 - 0.608493 0.608493 - 1.57322 1.57322 - 0.724246 0.724246 - -2.54194 -2.54194 - -0.61659 -0.61659 - 1.52939 1.52939 - 0.628167 0.628167 - 0.153008 0.153008 - 0.729473 0.729473 - 1.80753 1.80753 - -0.257529 -0.257529 - 2.30209 2.30209 - -1.07183 -1.07183 - -2.34549 -2.34549 - 1.7575 1.7575 - -1.09788 -1.09788 - -0.733157 -0.733157 - -1.15178 -1.15178 - -1.38334 -1.38334 - -0.282802 -0.282802 - -1.64302 -1.64302 - -8.14652 -8.14652 - 0.230736 0.230736 - 1.49834 1.49834 - -3.92113 -3.92113 - 1.26241 1.26241 - -0.867555 -0.867555 - 0.385275 0.385275 - -1.52048 -1.52048 - -3.58632 -3.58632 - 0.216633 0.216633 - -0.626523 -0.626523 - 0.0498179 0.0498179 - -0.736738 -0.736738 - -1.96551 -1.96551 - -0.886975 -0.886975 - -4.19118 -4.19118 - 4.69049 4.69049 - 1.24464 1.24464 - 0.559333 0.559333 - 0.365196 0.365196 - 2.05931 2.05931 - -1.61866 -1.61866 - -0.937082 -0.937082 - -2.28156 -2.28156 - 0.480478 0.480478 - 0.0375252 0.0375252 - 1.16423 1.16423 - 1.06467 1.06467 - -1.51476 -1.51476 - 0.639421 0.639421 - 0.835906 0.835906 - 0.0678173 0.0678173 - 0.911222 0.911222 - 0.178712 0.178712 - 0.16706 0.16706 - -0.338281 -0.338281 - 1.13458 1.13458 - 2.369 2.369 - 0.104902 0.104902 - 3.74878 3.74878 - -0.66965 -0.66965 - -0.539803 -0.539803 - 8.81148 8.81148 - 2.01163 2.01163 - -2.44142 -2.44142 - 1.71428 1.71428 - 0.195195 0.195195 - 0.886484 0.886484 - -1.61999 -1.61999 - 1.38916 1.38916 - -0.182804 -0.182804 - -2.12762 -2.12762 - 1.32879 1.32879 - 2.93628 2.93628 - 0.585458 0.585458 - 0.759949 0.759949 - 0.200884 0.200884 - 0.434392 0.434392 - 0.592809 0.592809 - -0.972697 -0.972697 - -0.30606 -0.30606 - 2.75682 2.75682 - 1.06405 1.06405 - 0.535445 0.535445 - 0.245727 0.245727 - -0.881514 -0.881514 - -0.90398 -0.90398 - 1.34242 1.34242 - 0.974607 0.974607 - -0.23557 -0.23557 - -2.36657 -2.36657 - 2.20975 2.20975 - 0.372629 0.372629 - -0.7674 -0.7674 - 0.503398 0.503398 - -0.39968 -0.39968 - 4.73687 4.73687 - 1.83971 1.83971 - -0.59006 -0.59006 - -3.12939 -3.12939 - 1.67781 1.67781 - -0.346719 -0.346719 - -1.20046 -1.20046 - -0.543011 -0.543011 - 2.00497 2.00497 - -2.29529 -2.29529 - 1.46115 1.46115 - -1.93647 -1.93647 - 0.346798 0.346798 - -0.781626 -0.781626 - -3.73517 -3.73517 - 0.403249 0.403249 - 0.147234 0.147234 - 0.581328 0.581328 - -1.56541 -1.56541 - -0.0382613 -0.0382613 - -2.69999 -2.69999 - 2.53761 2.53761 - 0.485891 0.485891 - 2.04661 2.04661 - -1.34907 -1.34907 - -2.77599 -2.77599 - -1.9941 -1.9941 - 2.26706 2.26706 - -0.841907 -0.841907 - 2.89779 2.89779 - 1.34194 1.34194 - 1.72031 1.72031 - 0.35742 0.35742 - -0.673142 -0.673142 - -1.97963 -1.97963 - 2.99822 2.99822 - 0.84135 0.84135 - 0.467833 0.467833 - -1.6184 -1.6184 - 0.545393 0.545393 - 0.0865121 0.0865121 - 1.38347 1.38347 - 2.3421 2.3421 - 2.6215 2.6215 - -1.48313 -1.48313 - 2.29728 2.29728 - -0.629032 -0.629032 - -0.053791 -0.053791 - 0.827546 0.827546 - -2.01859 -2.01859 - -0.151077 -0.151077 - -4.74001 -4.74001 - -2.07772 -2.07772 - 0.964427 0.964427 - 1.76579 1.76579 - -2.11515 -2.11515 - 2.99605 2.99605 - 1.38556 1.38556 - -2.60586 -2.60586 - -3.05364 -3.05364 - -1.22284 -1.22284 - 0.499165 0.499165 - 1.22502 1.22502 - -0.887611 -0.887611 - 1.1518 1.1518 - -2.7606 -2.7606 - 0.106263 0.106263 - 0.92196 0.92196 - -0.542352 -0.542352 - 0.772226 0.772226 - -0.409383 -0.409383 - -0.480159 -0.480159 - 0.434618 0.434618 - -1.93489 -1.93489 - 1.0037 1.0037 - 0.760285 0.760285 - 0.380404 0.380404 - 2.45566 2.45566 - -1.03099 -1.03099 - 0.652107 0.652107 - -1.64881 -1.64881 - -1.38051 -1.38051 - 1.43146 1.43146 - -0.366772 -0.366772 - 1.06121 1.06121 - 0.20026 0.20026 - 0.646821 0.646821 - 0.361296 0.361296 - 0.547448 0.547448 - 1.46293 1.46293 - 1.10425 1.10425 - -1.98927 -1.98927 - 1.33293 1.33293 - 0.702344 0.702344 - -0.294478 -0.294478 - 2.57426 2.57426 - 0.972237 0.972237 - -1.8532 -1.8532 - -1.52489 -1.52489 - -1.92059 -1.92059 - 0.4566 0.4566 - 0.0146066 0.0146066 - 1.15567 1.15567 - -2.15427 -2.15427 - -0.896258 -0.896258 - -0.579245 -0.579245 - 4.10418 4.10418 - 0.147275 0.147275 - -2.60135 -2.60135 - 1.13637 1.13637 - 4.39171 4.39171 - -3.93541 -3.93541 - -2.25406 -2.25406 - -0.766558 -0.766558 - 1.32684 1.32684 - 0.701722 0.701722 - 1.07565 1.07565 - -0.881368 -0.881368 - -1.79475 -1.79475 - -0.12303 -0.12303 - 0.939797 0.939797 - -1.63244 -1.63244 - -0.561795 -0.561795 - 0.668111 0.668111 - 2.28615 2.28615 - 1.6027 1.6027 - 0.210892 0.210892 - 1.10496 1.10496 - -3.5233 -3.5233 - 0.315028 0.315028 - -2.40724 -2.40724 - -0.865286 -0.865286 - -0.178629 -0.178629 - 4.42947 4.42947 - 2.0842 2.0842 - -1.17052 -1.17052 - -0.573684 -0.573684 - -0.230861 -0.230861 - -0.00536986 -0.00536986 - 1.18437 1.18437 - 0.954435 0.954435 - 0.475723 0.475723 - -1.45097 -1.45097 - -0.946094 -0.946094 - -0.371655 -0.371655 - -1.73344 -1.73344 - 0.25137 0.25137 - -0.904498 -0.904498 - 1.72233 1.72233 - -0.0630691 -0.0630691 - 1.20918 1.20918 - -1.58876 -1.58876 - -3.08633 -3.08633 - 0.469343 0.469343 - 0.334824 0.334824 - -0.420134 -0.420134 - -1.39986 -1.39986 - -0.65878 -0.65878 - -0.680645 -0.680645 - -1.46235 -1.46235 - -1.7763 -1.7763 - 0.061792 0.061792 - 2.10605 2.10605 - -0.198492 -0.198492 - 0.437371 0.437371 - -1.539 -1.539 - -0.615466 -0.615466 - 0.739653 0.739653 - -1.62828 -1.62828 - -3.99554 -3.99554 - -0.0523548 -0.0523548 - 1.77712 1.77712 - -0.756514 -0.756514 - 1.43585 1.43585 - -0.730324 -0.730324 - -0.714939 -0.714939 - -0.208195 -0.208195 - 3.00343 3.00343 - -0.780319 -0.780319 - -1.67158 -1.67158 - -4.72578 -4.72578 - -0.409396 -0.409396 - -0.182756 -0.182756 - 0.131891 0.131891 - -0.0622689 -0.0622689 - 0.504642 0.504642 - 1.05712 1.05712 - 0.603898 0.603898 - -1.08295 -1.08295 - 3.50826 3.50826 - -1.21709 -1.21709 - 2.62074 2.62074 - 2.27406 2.27406 - -0.444599 -0.444599 - -1.23079 -1.23079 - 2.01351 2.01351 - -0.771506 -0.771506 - -1.25143 -1.25143 - -2.42808 -2.42808 - 0.618042 0.618042 - -0.139276 -0.139276 - -1.15122 -1.15122 - -1.00705 -1.00705 - -0.513695 -0.513695 - 1.49666 1.49666 - 2.08012 2.08012 - -0.0331592 -0.0331592 - -1.27368 -1.27368 - 0.237436 0.237436 - -1.92915 -1.92915 - -3.38558 -3.38558 - 0.656076 0.656076 - 4.89326 4.89326 - -0.696932 -0.696932 - 1.81671 1.81671 - -0.163389 -0.163389 - -0.640608 -0.640608 - -0.466348 -0.466348 - 0.790762 0.790762 - -0.15507 -0.15507 - -0.387485 -0.387485 - 3.22853 3.22853 - -0.361262 -0.361262 - 0.838794 0.838794 - -1.80534 -1.80534 - -0.514814 -0.514814 - -1.96885 -1.96885 - -1.32784 -1.32784 - 0.420068 0.420068 - -0.720913 -0.720913 - -0.340982 -0.340982 - 0.496067 0.496067 - -0.921807 -0.921807 - -1.92175 -1.92175 - 0.59455 0.59455 - 0.827755 0.827755 - -1.90727 -1.90727 - 1.51996 1.51996 - 1.79573 1.79573 - -2.10208 -2.10208 - 4.78008 4.78008 - 0.767953 0.767953 - 0.240847 0.240847 - 1.97272 1.97272 - -1.59962 -1.59962 - -1.47237 -1.47237 - -0.580724 -0.580724 - 1.75388 1.75388 - -1.67413 -1.67413 - -0.636017 -0.636017 - -1.35096 -1.35096 - 0.722983 0.722983 - -0.585923 -0.585923 - -0.529864 -0.529864 - -0.91566 -0.91566 - 0.242429 0.242429 - 0.236836 0.236836 - -1.78323 -1.78323 - 0.716162 0.716162 - 0.166974 0.166974 - -0.00891157 -0.00891157 - 0.936227 0.936227 - 0.992144 0.992144 - 0.00694193 0.00694193 - -3.58995 -3.58995 - -2.90908 -2.90908 - 2.30937 2.30937 - -0.504803 -0.504803 - 0.870025 0.870025 - 1.0472 1.0472 - -0.165569 -0.165569 - -0.110656 -0.110656 - -0.240613 -0.240613 - 1.47692 1.47692 - 0.153659 0.153659 - 0.237772 0.237772 - -0.427641 -0.427641 - -1.6352 -1.6352 - -0.329495 -0.329495 - 1.89567 1.89567 - 0.500036 0.500036 - 2.60672 2.60672 - 1.08705 1.08705 - 0.234073 0.234073 - 1.80469 1.80469 - 0.859386 0.859386 - 1.59161 1.59161 - -1.25096 -1.25096 - 0.693032 0.693032 - -0.208542 -0.208542 - 2.9791 2.9791 - 0.0637262 0.0637262 - -0.329288 -0.329288 - 0.0765908 0.0765908 - -0.975783 -0.975783 - 0.44728 0.44728 - 0.158878 0.158878 - -1.23836 -1.23836 - 0.670039 0.670039 - -0.0279593 -0.0279593 - -1.08503 -1.08503 - -1.61416 -1.61416 - -1.68594 -1.68594 - 0.91421 0.91421 - 0.164412 0.164412 - -0.452083 -0.452083 - -3.85434 -3.85434 - -1.17754 -1.17754 - -0.824006 -0.824006 - 2.34023 2.34023 - -2.09985 -2.09985 - 0.881987 0.881987 - -1.37035 -1.37035 - 0.642559 0.642559 - -0.17144 -0.17144 - 1.61998 1.61998 - -0.621223 -0.621223 - -3.74722 -3.74722 - 1.14967 1.14967 - 0.718755 0.718755 - 0.675293 0.675293 - -1.53459 -1.53459 - 0.521371 0.521371 - -0.0142472 -0.0142472 - -1.10933 -1.10933 - -1.60977 -1.60977 - -0.152616 -0.152616 - 0.402871 0.402871 - 0.929587 0.929587 - 1.69781 1.69781 - -1.8187 -1.8187 - 2.04684 2.04684 - -1.29619 -1.29619 - -0.212775 -0.212775 - 1.52843 1.52843 - -1.69828 -1.69828 - -1.05251 -1.05251 - 3.80258 3.80258 - 1.77294 1.77294 - 1.20703 1.20703 - 0.81561 0.81561 - 0.748058 0.748058 - 1.45736 1.45736 - 1.33881 1.33881 - -0.86026 -0.86026 - -0.00694993 -0.00694993 - -1.67171 -1.67171 - -2.64284 -2.64284 - -0.617916 -0.617916 - -0.658362 -0.658362 - -0.302764 -0.302764 - -0.218727 -0.218727 - 2.68447 2.68447 - 1.91834 1.91834 - -0.630305 -0.630305 - -0.483942 -0.483942 - -0.205622 -0.205622 - 0.230539 0.230539 - -2.42183 -2.42183 - 0.000632207 0.000632207 - 1.22722 1.22722 - 0.499288 0.499288 - -0.461535 -0.461535 - -2.41778 -2.41778 - -6.43109 -6.43109 - 1.87164 1.87164 - -0.128426 -0.128426 - 1.70185 1.70185 - 0.321244 0.321244 - -2.8774 -2.8774 - -0.973839 -0.973839 - -1.78429 -1.78429 - -1.42296 -1.42296 - -2.18451 -2.18451 - -0.997205 -0.997205 - -2.09179 -2.09179 - -0.418012 -0.418012 - -3.86836 -3.86836 - -1.08119 -1.08119 - -2.73167 -2.73167 - -1.83625 -1.83625 - 0.048819 0.048819 - 1.82894 1.82894 - 0.828514 0.828514 - 0.392871 0.392871 - 1.13363 1.13363 - -1.79231 -1.79231 - 0.163824 0.163824 - -1.27652 -1.27652 - 2.06888 2.06888 - -0.63393 -0.63393 - 0.684518 0.684518 - -0.906544 -0.906544 - -0.987848 -0.987848 - -0.478984 -0.478984 - -1.2593 -1.2593 - 2.07364 2.07364 - 0.162523 0.162523 - -0.799031 -0.799031 - -0.299117 -0.299117 - 0.0988399 0.0988399 - -0.238386 -0.238386 - 3.0895 3.0895 - 0.415509 0.415509 - 1.50283 1.50283 - -0.561321 -0.561321 - 2.24325 2.24325 - -0.490123 -0.490123 - -1.27889 -1.27889 - 0.10889 0.10889 - 0.581722 0.581722 - -0.657735 -0.657735 - -0.722435 -0.722435 - -1.3811 -1.3811 - 1.5928 1.5928 - -9.72189 -9.72189 - -1.07349 -1.07349 - -1.25083 -1.25083 - 0.919328 0.919328 - -1.9068 -1.9068 - -1.23178 -1.23178 - -0.782403 -0.782403 - -0.689571 -0.689571 - -2.07093 -2.07093 - -0.0676932 -0.0676932 - -1.47628 -1.47628 - -1.79765 -1.79765 - 1.79051 1.79051 - 0.410585 0.410585 - -1.41706 -1.41706 - 1.64883 1.64883 - -0.528457 -0.528457 - 0.193134 0.193134 - -0.989156 -0.989156 - 3.3787 3.3787 - 0.734755 0.734755 - 1.20919 1.20919 - 0.142022 0.142022 - 0.950151 0.950151 - -2.14197 -2.14197 - 2.91323 2.91323 - -0.216118 -0.216118 - 1.18072 1.18072 - -5.00813 -5.00813 - 1.22234 1.22234 - -2.40273 -2.40273 - -1.93511 -1.93511 - 1.02775 1.02775 - -0.407362 -0.407362 - -0.033977 -0.033977 - -0.673432 -0.673432 - -0.395374 -0.395374 - -0.0686072 -0.0686072 - 2.26745 2.26745 - -4.0298 -4.0298 - 0.30017 0.30017 - 0.576736 0.576736 - 1.84158 1.84158 - -0.427059 -0.427059 - -3.47072 -3.47072 - -0.207896 -0.207896 - 1.07928 1.07928 - -0.424236 -0.424236 - -0.809626 -0.809626 - 2.48986 2.48986 - -0.808405 -0.808405 - -0.0510026 -0.0510026 - -0.919838 -0.919838 - 1.44427 1.44427 - 2.71946 2.71946 - 0.0683191 0.0683191 - -1.08351 -1.08351 - -1.56937 -1.56937 - 3.01879 3.01879 - -1.56172 -1.56172 - 2.06467 2.06467 - 2.03327 2.03327 - -0.81693 -0.81693 - 0.922913 0.922913 - 0.95788 0.95788 - -0.778203 -0.778203 - 1.08222 1.08222 - 1.44635 1.44635 - -1.75929 -1.75929 - 0.112181 0.112181 - 0.681292 0.681292 - -0.432171 -0.432171 - 1.06316 1.06316 - -1.7268 -1.7268 - -3.37779 -3.37779 - -2.89964 -2.89964 - 0.901763 0.901763 - -1.2757 -1.2757 - -12.5488 -12.5488 - 2.55937 2.55937 - -1.79624 -1.79624 - -2.36511 -2.36511 - 0.755598 0.755598 - -1.59443 -1.59443 - -3.05103 -3.05103 - -1.29157 -1.29157 - 0.254057 0.254057 - -0.628874 -0.628874 - -0.119316 -0.119316 - 4.96649 4.96649 - -1.36958 -1.36958 - -0.258365 -0.258365 - 0.207744 0.207744 - -0.166113 -0.166113 - -1.22071 -1.22071 - -3.68573 -3.68573 - -3.16103 -3.16103 - -1.89154 -1.89154 - 0.98744 0.98744 - -0.585739 -0.585739 - -1.99954 -1.99954 - 2.03071 2.03071 - 0.471045 0.471045 - -1.93214 -1.93214 - -0.825462 -0.825462 - 0.497266 0.497266 - -0.563979 -0.563979 - 1.69555 1.69555 - 3.04967 3.04967 - -0.302795 -0.302795 - 1.60455 1.60455 - 2.28473 2.28473 - -1.28218 -1.28218 - -0.559678 -0.559678 - -0.759997 -0.759997 - 0.809902 0.809902 - -0.451467 -0.451467 - -1.21982 -1.21982 - 0.325713 0.325713 - -3.28885 -3.28885 - -0.124363 -0.124363 - -0.698127 -0.698127 - -0.213705 -0.213705 - -2.03161 -2.03161 - 1.82803 1.82803 - 0.644781 0.644781 - 0.20483 0.20483 - 4.57695 4.57695 - -2.63731 -2.63731 - -2.18961 -2.18961 - -2.48184 -2.48184 - 0.0614645 0.0614645 - 0.377704 0.377704 - 2.34104 2.34104 - 0.657159 0.657159 - -1.25309 -1.25309 - -2.45413 -2.45413 - 1.88619 1.88619 - 1.74094 1.74094 - 2.3351 2.3351 - 0.212613 0.212613 - -1.16212 -1.16212 - 1.02027 1.02027 - 0.0929927 0.0929927 - 0.260815 0.260815 - -1.58322 -1.58322 - -1.5689 -1.5689 - 1.90734 1.90734 - -0.398199 -0.398199 - 1.1045 1.1045 - -1.90117 -1.90117 - -1.35457 -1.35457 - -0.904438 -0.904438 - -2.4261 -2.4261 - 2.24749 2.24749 - 0.293511 0.293511 - -1.96554 -1.96554 - -0.310607 -0.310607 - 0.399931 0.399931 - 1.06214 1.06214 - 1.14559 1.14559 - -1.29189 -1.29189 - 1.25441 1.25441 - 2.97262 2.97262 - -1.02313 -1.02313 - 1.05321 1.05321 - -0.804898 -0.804898 - 3.63862 3.63862 - 0.527294 0.527294 - -0.681347 -0.681347 - 1.11035 1.11035 - 2.40787 2.40787 - 1.19095 1.19095 - -0.380581 -0.380581 - 0.0582491 0.0582491 - 0.264974 0.264974 - 0.169384 0.169384 - 1.4731 1.4731 - 2.23289 2.23289 - 0.191901 0.191901 - -2.60952 -2.60952 - -2.62244 -2.62244 - 0.784122 0.784122 - 2.74349 2.74349 - -0.155218 -0.155218 - 0.967558 0.967558 - 0.186159 0.186159 - -0.204567 -0.204567 - -0.606278 -0.606278 - -0.011704 -0.011704 - 1.08489 1.08489 - -0.277437 -0.277437 - 0.0090858 0.0090858 - 0.258057 0.258057 - -0.373131 -0.373131 - -1.44058 -1.44058 - -1.01993 -1.01993 - -2.4798 -2.4798 - 0.271599 0.271599 - -1.84189 -1.84189 - -0.490777 -0.490777 - -0.414488 -0.414488 - -1.07757 -1.07757 - 1.23712 1.23712 - -2.02542 -2.02542 - -2.07667 -2.07667 - 2.89555 2.89555 - -2.61436 -2.61436 - -0.469252 -0.469252 - -1.29202 -1.29202 - -0.646527 -0.646527 - 0.00517958 0.00517958 - 0.5503 0.5503 - 1.06083 1.06083 - -0.324516 -0.324516 - -1.84483 -1.84483 - -0.574072 -0.574072 - -0.211102 -0.211102 - -0.585931 -0.585931 - 0.854918 0.854918 - -1.33179 -1.33179 - -1.87131 -1.87131 - 1.63465 1.63465 - 0.904734 0.904734 - -1.27273 -1.27273 - -2.41944 -2.41944 - 0.626935 0.626935 - -1.74987 -1.74987 - -3.06298 -3.06298 - -0.714794 -0.714794 - -1.00437 -1.00437 - 0.197116 0.197116 - 0.276584 0.276584 - 0.163489 0.163489 - 1.74588 1.74588 - -1.48997 -1.48997 - 0.561946 0.561946 - -0.164713 -0.164713 - -1.79587 -1.79587 - 0.681301 0.681301 - -0.303574 -0.303574 - 0.695659 0.695659 - 2.46479 2.46479 - 2.98348 2.98348 - 1.50751 1.50751 - 0.531878 0.531878 - -1.00327 -1.00327 - 1.42955 1.42955 - 1.41544 1.41544 - 0.631301 0.631301 - 0.887182 0.887182 - -0.689095 -0.689095 - 1.0004 1.0004 - 1.73016 1.73016 - 3.52498 3.52498 - 2.04091 2.04091 - 1.25617 1.25617 - -0.769574 -0.769574 - -0.439135 -0.439135 - -0.320473 -0.320473 - 1.88776 1.88776 - -3.08042 -3.08042 - 0.427227 0.427227 - 1.32274 1.32274 - -2.11399 -2.11399 - 3.3764 3.3764 - -0.943091 -0.943091 - 2.07374 2.07374 - -1.59418 -1.59418 - -4.37112 -4.37112 - 0.620227 0.620227 - -0.322782 -0.322782 - 0.447005 0.447005 - 0.280967 0.280967 - 1.68907 1.68907 - -1.60102 -1.60102 - 2.9697 2.9697 - -0.0128504 -0.0128504 - -0.782678 -0.782678 - 0.397837 0.397837 - -0.159312 -0.159312 - 1.14526 1.14526 - 0.0428197 0.0428197 - 1.77001 1.77001 - 1.08919 1.08919 - 0.55684 0.55684 - 0.398568 0.398568 - 1.27518 1.27518 - -1.85676 -1.85676 - -2.56394 -2.56394 - -0.357036 -0.357036 - 2.1851 2.1851 - -1.03051 -1.03051 - 0.922326 0.922326 - 0.654941 0.654941 - -0.902982 -0.902982 - -0.90051 -0.90051 - -0.127011 -0.127011 - 1.76007 1.76007 - 0.424547 0.424547 - -1.13683 -1.13683 - 1.83884 1.83884 - -0.918215 -0.918215 - 0.658109 0.658109 - -0.365433 -0.365433 - -2.18908 -2.18908 - -1.93223 -1.93223 - 0.408743 0.408743 - -1.92079 -1.92079 - 0.0976185 0.0976185 - 4.67569 4.67569 - -1.49021 -1.49021 - 0.176745 0.176745 - -1.03095 -1.03095 - -2.1574 -2.1574 - -0.593898 -0.593898 - 1.01641 1.01641 - -0.28545 -0.28545 - -2.17595 -2.17595 - 1.00962 1.00962 - 0.592646 0.592646 - -0.343818 -0.343818 - -0.440034 -0.440034 - 1.38711 1.38711 - -1.43068 -1.43068 - 0.681063 0.681063 - -1.94052 -1.94052 - -2.38281 -2.38281 - -0.789486 -0.789486 - -1.35709 -1.35709 - -1.69084 -1.69084 - -1.62863 -1.62863 - -0.284129 -0.284129 - -2.83546 -2.83546 - -0.915304 -0.915304 - 0.257879 0.257879 - 0.745803 0.745803 - -0.134261 -0.134261 - -0.00297398 -0.00297398 - -0.452233 -0.452233 - -2.36279 -2.36279 - 1.21461 1.21461 - -1.13373 -1.13373 - -0.329279 -0.329279 - -1.64147 -1.64147 - -0.579568 -0.579568 - -0.783701 -0.783701 - 0.606064 0.606064 - 0.249187 0.249187 - -0.266901 -0.266901 - 5.9043 5.9043 - -0.630335 -0.630335 - -0.37167 -0.37167 - 2.66605 2.66605 - -1.36786 -1.36786 - 0.452445 0.452445 - -1.28971 -1.28971 - 2.37668 2.37668 - 2.52488 2.52488 - 0.142467 0.142467 - 1.68463 1.68463 - 1.30605 1.30605 - -3.62793 -3.62793 - 2.48566 2.48566 - -1.13421 -1.13421 - 0.0926649 0.0926649 - -3.63194 -3.63194 - 0.108222 0.108222 - -1.92097 -1.92097 - -0.609085 -0.609085 - -2.23882 -2.23882 - 1.30409 1.30409 - 0.251002 0.251002 - 0.820516 0.820516 - 1.14979 1.14979 - 4.24577 4.24577 - 0.195344 0.195344 - -0.322737 -0.322737 - 0.93393 0.93393 - 2.81175 2.81175 - 0.446439 0.446439 - -1.38935 -1.38935 - 0.386624 0.386624 - 0.464916 0.464916 - -1.58662 -1.58662 - -1.12035 -1.12035 - 1.79443 1.79443 - 1.03373 1.03373 - 1.24398 1.24398 - -0.343199 -0.343199 - 0.778172 0.778172 - 3.7689 3.7689 - 1.15726 1.15726 - 0.0548858 0.0548858 - 1.46606 1.46606 - -0.0390753 -0.0390753 - 0.525417 0.525417 - -0.399366 -0.399366 - -0.389525 -0.389525 - 1.53548 1.53548 - -0.660596 -0.660596 - 1.02893 1.02893 - 5.36388 5.36388 - -0.158376 -0.158376 - -0.00158735 -0.00158735 - 1.48366 1.48366 - 1.58287 1.58287 - -1.32445 -1.32445 - 1.18222 1.18222 - 0.859016 0.859016 - -0.366681 -0.366681 - -1.97608 -1.97608 - -1.60289 -1.60289 - 0.801427 0.801427 - 0.734912 0.734912 - -0.655545 -0.655545 - -3.4244 -3.4244 - 1.79295 1.79295 - 1.64987 1.64987 - -0.372111 -0.372111 - -0.818163 -0.818163 - -1.73224 -1.73224 - 1.07321 1.07321 - -1.09122 -1.09122 - -2.31512 -2.31512 - -1.51542 -1.51542 - -0.366266 -0.366266 - 2.78864 2.78864 - -1.79099 -1.79099 - -0.527644 -0.527644 - 0.617675 0.617675 - 0.817341 0.817341 - -1.59739 -1.59739 - -0.942853 -0.942853 - -0.024404 -0.024404 - 0.838706 0.838706 - 2.69883 2.69883 - -1.84305 -1.84305 - 0.197916 0.197916 - 1.40902 1.40902 - 0.780807 0.780807 - -0.447145 -0.447145 - 0.39841 0.39841 - -0.568937 -0.568937 - 3.35843 3.35843 - 0.244485 0.244485 - -0.409614 -0.409614 - 1.79979 1.79979 - -0.114665 -0.114665 - 2.12529 2.12529 - -1.1328 -1.1328 - -1.32568 -1.32568 - -1.33014 -1.33014 - 0.61469 0.61469 - 0.23576 0.23576 - -1.58669 -1.58669 - 0.882018 0.882018 - -0.199537 -0.199537 - 0.0213817 0.0213817 - -0.0526741 -0.0526741 - -1.90557 -1.90557 - -0.861437 -0.861437 - -1.73529 -1.73529 - -1.20074 -1.20074 - 1.52766 1.52766 - 0.229589 0.229589 - 0.371437 0.371437 - -2.27666 -2.27666 - 0.500965 0.500965 - 0.342153 0.342153 - 1.54886 1.54886 - -2.35151 -2.35151 - 0.369683 0.369683 - 0.537004 0.537004 - 0.462267 0.462267 - 0.104486 0.104486 - 1.76568 1.76568 - 0.66308 0.66308 - -0.610417 -0.610417 - -1.19933 -1.19933 - 0.458763 0.458763 - 1.12317 1.12317 - 0.639384 0.639384 - 0.420247 0.420247 - -0.516877 -0.516877 - 1.85253 1.85253 - -1.28899 -1.28899 - -1.2008 -1.2008 - 0.55433 0.55433 - 0.209396 0.209396 - -1.13624 -1.13624 - -1.29488 -1.29488 - -0.990851 -0.990851 - -1.82754 -1.82754 - -1.10815 -1.10815 - 0.268715 0.268715 - -1.8783 -1.8783 - 1.46547 1.46547 - 2.47695 2.47695 - -2.0124 -2.0124 - -2.23708 -2.23708 - 2.61164 2.61164 - 0.260522 0.260522 - 0.570767 0.570767 - 2.82558 2.82558 - 1.21898 1.21898 - -2.1909 -2.1909 - 1.95577 1.95577 - -0.334229 -0.334229 - 0.490568 0.490568 - 0.476326 0.476326 - -2.85789 -2.85789 - -1.67129 -1.67129 - -3.44224 -3.44224 - 1.39022 1.39022 - 1.13294 1.13294 - 1.12937 1.12937 - 1.55696 1.55696 - 0.641922 0.641922 - 3.02277 3.02277 - 0.727429 0.727429 - 2.29113 2.29113 - -0.345511 -0.345511 - -0.963097 -0.963097 - 2.04846 2.04846 - 1.82306 1.82306 - -0.10738 -0.10738 - 1.47278 1.47278 - 0.44212 0.44212 - -0.664988 -0.664988 - -0.522717 -0.522717 - 0.015111 0.015111 - -1.17036 -1.17036 - 0.533411 0.533411 - -0.755567 -0.755567 - 1.3441 1.3441 - 2.34532 2.34532 - 1.54432 1.54432 - 0.346052 0.346052 - -0.317292 -0.317292 - -0.419647 -0.419647 - 0.778187 0.778187 - 2.52819 2.52819 - 1.71307 1.71307 - 0.108402 0.108402 - -0.286356 -0.286356 - -0.184754 -0.184754 - -2.0023 -2.0023 - -1.88134 -1.88134 - -0.401823 -0.401823 - -0.103521 -0.103521 - 0.753005 0.753005 - -0.803658 -0.803658 - 2.44367 2.44367 - 1.58612 1.58612 - -0.94574 -0.94574 - 0.269329 0.269329 - 2.60832 2.60832 - -0.113991 -0.113991 - 2.23152 2.23152 - -1.14749 -1.14749 - -1.86227 -1.86227 - -2.06391 -2.06391 - -0.960271 -0.960271 - -0.953305 -0.953305 - -0.162597 -0.162597 - 0.405977 0.405977 - 1.18975 1.18975 - -2.68021 -2.68021 - 0.557723 0.557723 - 0.1416 0.1416 - -2.27623 -2.27623 - -0.91957 -0.91957 - 1.4132 1.4132 - 2.40127 2.40127 - 0.364167 0.364167 - 1.67016 1.67016 - 0.345176 0.345176 - -0.885645 -0.885645 - -1.92508 -1.92508 - -0.527327 -0.527327 - 0.59694 0.59694 - -0.928777 -0.928777 - -2.45285 -2.45285 - 0.093849 0.093849 - 0.875405 0.875405 - 0.734934 0.734934 - -1.58165 -1.58165 - -0.43735 -0.43735 - 0.103331 0.103331 - 1.57271 1.57271 - -0.670804 -0.670804 - 3.11503 3.11503 - 2.03147 2.03147 - -1.12063 -1.12063 - 1.76581 1.76581 - -1.91044 -1.91044 - -0.334989 -0.334989 - -0.115597 -0.115597 - 2.82 2.82 - 2.8363 2.8363 - -2.12753 -2.12753 - 0.551957 0.551957 - 0.778678 0.778678 - 0.26151 0.26151 - 0.527825 0.527825 - -1.24336 -1.24336 - -1.28699 -1.28699 - -0.227357 -0.227357 - 4.99396 4.99396 - -1.90422 -1.90422 - -0.883407 -0.883407 - 3.10147 3.10147 - 0.33776 0.33776 - 0.12595 0.12595 - -0.147078 -0.147078 - 0.40136 0.40136 - 0.092439 0.092439 - -0.903678 -0.903678 - 0.786246 0.786246 - 1.43764 1.43764 - -4.39428 -4.39428 - 1.17994 1.17994 - 0.485781 0.485781 - 1.63216 1.63216 - -1.36006 -1.36006 - 0.773674 0.773674 - 0.618503 0.618503 - -0.0300672 -0.0300672 - 0.484393 0.484393 - -0.874462 -0.874462 - -2.0849 -2.0849 - -0.576099 -0.576099 - 0.765223 0.765223 - -3.83117 -3.83117 - 1.46556 1.46556 - 1.88278 1.88278 - 3.49482 3.49482 - 2.57193 2.57193 - -3.45378 -3.45378 - 0.101699 0.101699 - -0.67995 -0.67995 - -1.76783 -1.76783 - 1.16307 1.16307 - -0.258105 -0.258105 - 1.91705 1.91705 - -2.29152 -2.29152 - 0.348873 0.348873 - -0.74269 -0.74269 - 0.158894 0.158894 - -0.454844 -0.454844 - 2.23608 2.23608 - -0.210747 -0.210747 - -0.544031 -0.544031 - 0.656622 0.656622 - -1.47574 -1.47574 - 0.0299864 0.0299864 - 1.77173 1.77173 - -2.23061 -2.23061 - -1.00651 -1.00651 - -0.0842836 -0.0842836 - -1.17457 -1.17457 - 3.36571 3.36571 - 0.656163 0.656163 - -0.0818359 -0.0818359 - 2.36086 2.36086 - -2.1491 -2.1491 - 1.45799 1.45799 - -1.09112 -1.09112 - 1.1336 1.1336 - 0.414929 0.414929 - 1.73283 1.73283 - 0.333332 0.333332 - 0.612909 0.612909 - 2.27707 2.27707 - -17.3366 -17.3366 - -0.706549 -0.706549 - 0.569747 0.569747 - -2.88274 -2.88274 - -0.39159 -0.39159 - -1.272 -1.272 - 1.8522 1.8522 - 2.84343 2.84343 - -0.497109 -0.497109 - 0.712927 0.712927 - -0.454025 -0.454025 - 0.26183 0.26183 - 1.70567 1.70567 - 2.65185 2.65185 - 0.0631512 0.0631512 - 1.55346 1.55346 - 1.88528 1.88528 - 0.550265 0.550265 - -0.663944 -0.663944 - 0.796127 0.796127 - 1.5298 1.5298 - -0.619118 -0.619118 - 0.718367 0.718367 - -1.1251 -1.1251 - -0.162817 -0.162817 - 0.158565 0.158565 - -0.479435 -0.479435 - 0.39778 0.39778 - -0.0668922 -0.0668922 - -0.554059 -0.554059 - -0.112442 -0.112442 - 0.723476 0.723476 - 0.585585 0.585585 - -23.8354 -23.8354 - 2.23005 2.23005 - 3.19716 3.19716 - 0.878244 0.878244 - 1.24065 1.24065 - 2.02788 2.02788 - 1.33889 1.33889 - -0.0959704 -0.0959704 - 0.819301 0.819301 - 1.13624 1.13624 - -3.73236 -3.73236 - -0.228297 -0.228297 - 0.300764 0.300764 - 2.44121 2.44121 - 1.64459 1.64459 - -2.9741 -2.9741 - -0.0767092 -0.0767092 - 1.18346 1.18346 - 0.766311 0.766311 - -2.34982 -2.34982 - -1.19871 -1.19871 - -1.12327 -1.12327 - -1.51106 -1.51106 - -0.954545 -0.954545 - 1.09042 1.09042 - 1.78073 1.78073 - -1.6752 -1.6752 - 0.032185 0.032185 - -0.67565 -0.67565 - -0.318844 -0.318844 - -2.8893 -2.8893 - -0.0509838 -0.0509838 - -1.21507 -1.21507 - 2.30929 2.30929 - 0.216374 0.216374 - 1.31134 1.31134 - -0.0998794 -0.0998794 - 1.20712 1.20712 - -0.650083 -0.650083 - 0.399312 0.399312 - -0.983942 -0.983942 - -1.13459 -1.13459 - -1.05734 -1.05734 - -1.5015 -1.5015 - -1.77513 -1.77513 - -1.40363 -1.40363 - 1.33786 1.33786 - -0.826259 -0.826259 - 1.47633 1.47633 - -0.943677 -0.943677 - -1.46965 -1.46965 - -0.526938 -0.526938 - 0.191906 0.191906 - -0.886405 -0.886405 - 0.449917 0.449917 - 0.515804 0.515804 - -1.39774 -1.39774 - -0.654256 -0.654256 - -3.35005 -3.35005 - 2.06083 2.06083 - -0.704538 -0.704538 - -0.462466 -0.462466 - -2.67366 -2.67366 - -2.18423 -2.18423 - -2.64621 -2.64621 - 0.763978 0.763978 - -2.66418 -2.66418 - -1.60911 -1.60911 - 2.16377 2.16377 - 0.0968811 0.0968811 - 1.1338 1.1338 - 1.00793 1.00793 - 1.46003 1.46003 - -1.83402 -1.83402 - 0.91668 0.91668 - 0.614564 0.614564 - 0.635087 0.635087 - -1.34331 -1.34331 - -0.972611 -0.972611 - 0.674227 0.674227 - -0.667037 -0.667037 - -2.53443 -2.53443 - -1.53923 -1.53923 - -0.503973 -0.503973 - -3.74964 -3.74964 - -2.319 -2.319 - 0.634011 0.634011 - -0.232782 -0.232782 - 0.494076 0.494076 - -1.22967 -1.22967 - 2.0448 2.0448 - -2.49856 -2.49856 - 0.234543 0.234543 - 0.926016 0.926016 - 0.556234 0.556234 - 1.40825 1.40825 - -0.0225873 -0.0225873 - 0.584709 0.584709 - 0.750481 0.750481 - 0.548451 0.548451 - -0.50962 -0.50962 - 1.37736 1.37736 - -0.205268 -0.205268 - -2.18919 -2.18919 - -1.09548 -1.09548 - 2.88677 2.88677 - 0.36593 0.36593 - 1.59942 1.59942 - 2.36871 2.36871 - -0.360048 -0.360048 - 0.462762 0.462762 - -2.94594 -2.94594 - -1.04354 -1.04354 - 0.866209 0.866209 - 1.5211 1.5211 - 1.06147 1.06147 - 0.900548 0.900548 - -2.17463 -2.17463 - 2.6977 2.6977 - -0.854358 -0.854358 - 1.45336 1.45336 - 0.700988 0.700988 - -1.89129 -1.89129 - 0.346364 0.346364 - 1.09519 1.09519 - 1.19629 1.19629 - -0.193888 -0.193888 - 3.33354 3.33354 - -0.986782 -0.986782 - -0.328471 -0.328471 - 1.29318 1.29318 - 0.463316 0.463316 - -1.3053 -1.3053 - -0.247824 -0.247824 - 0.770688 0.770688 - -0.471133 -0.471133 - 0.613311 0.613311 - -0.394463 -0.394463 - -1.93519 -1.93519 - -1.45159 -1.45159 - 0.868683 0.868683 - 2.57851 2.57851 - -2.39077 -2.39077 - 1.63547 1.63547 - -0.0639398 -0.0639398 - 1.19782 1.19782 - -1.83348 -1.83348 - -0.473202 -0.473202 - 0.124168 0.124168 - 1.16798 1.16798 - 1.21296 1.21296 - -0.527453 -0.527453 - -1.20688 -1.20688 - -0.713952 -0.713952 - 0.262652 0.262652 - -2.18811 -2.18811 - 4.102 4.102 - 0.494994 0.494994 - 2.77783 2.77783 - 1.07731 1.07731 - 0.198336 0.198336 - 9.57229 9.57229 - 0.349448 0.349448 - -2.29562 -2.29562 - -0.644837 -0.644837 - -0.217495 -0.217495 - -1.45496 -1.45496 - -2.09866 -2.09866 - -0.0384889 -0.0384889 - 0.745561 0.745561 - 0.14508 0.14508 - -1.51823 -1.51823 - -3.26751 -3.26751 - 0.0523259 0.0523259 - 1.55301 1.55301 - -3.45208 -3.45208 - 0.572187 0.572187 - -2.49679 -2.49679 - 2.37328 2.37328 - 0.765015 0.765015 - -0.27352 -0.27352 - 0.0308149 0.0308149 - -0.593623 -0.593623 - -1.2373 -1.2373 - -0.656835 -0.656835 - -2.13627 -2.13627 - 0.352383 0.352383 - -0.406136 -0.406136 - 1.4126 1.4126 - 1.98515 1.98515 - 0.606976 0.606976 - 0.788446 0.788446 - 0.16519 0.16519 - 0.596765 0.596765 - -0.873611 -0.873611 - -1.43965 -1.43965 - 0.0971001 0.0971001 - 3.33849 3.33849 - 4.49406 4.49406 - 1.91075 1.91075 - 2.4465 2.4465 - 2.35879 2.35879 - -2.22234 -2.22234 - 2.1624 2.1624 - 0.037742 0.037742 - -1.28695 -1.28695 - 0.341673 0.341673 - 1.11316 1.11316 - 1.57437 1.57437 - -0.778719 -0.778719 - -2.35793 -2.35793 - -0.95091 -0.95091 - -1.59879 -1.59879 - -1.65049 -1.65049 - 1.05626 1.05626 - 1.14966 1.14966 - -2.68582 -2.68582 - -1.96644 -1.96644 - 0.950004 0.950004 - 3.03491 3.03491 - -1.64747 -1.64747 - -2.14413 -2.14413 - -0.804942 -0.804942 - 2.19513 2.19513 - 0.789969 0.789969 - -0.899571 -0.899571 - -0.190668 -0.190668 - 0.600747 0.600747 - 1.16949 1.16949 - -1.37309 -1.37309 - -1.43457 -1.43457 - 0.462702 0.462702 - 1.40818 1.40818 - -0.665746 -0.665746 - -2.68812 -2.68812 - -0.0352648 -0.0352648 - -1.20922 -1.20922 - -0.577064 -0.577064 - -1.71517 -1.71517 - 1.35121 1.35121 - 1.4034 1.4034 - -4.70048 -4.70048 - -4.44732 -4.44732 - 0.136732 0.136732 - 2.41848 2.41848 - -4.30087 -4.30087 - -2.46922 -2.46922 - 0.376919 0.376919 - 1.9388 1.9388 - 1.19049 1.19049 - 1.97452 1.97452 - -0.172176 -0.172176 - -2.31517 -2.31517 - 7.36023 7.36023 - -0.738473 -0.738473 - 0.0146472 0.0146472 - 0.14857 0.14857 - 2.47231 2.47231 - 2.18251 2.18251 - -2.32425 -2.32425 - -2.17348 -2.17348 - 7.06046 7.06046 - 1.42047 1.42047 - -1.54418 -1.54418 - -0.469984 -0.469984 - 1.63989 1.63989 - 0.346456 0.346456 - 1.15453 1.15453 - -0.441885 -0.441885 - 0.183807 0.183807 - 3.1971 3.1971 - -0.647559 -0.647559 - 0.00128156 0.00128156 - -0.288304 -0.288304 - 2.59287 2.59287 - 1.19027 1.19027 - 0.797827 0.797827 - 1.09197 1.09197 - 0.43136 0.43136 - 0.380913 0.380913 - -1.74969 -1.74969 - 1.14465 1.14465 - -0.386246 -0.386246 - 0.566953 0.566953 - 0.00563241 0.00563241 - 0.841304 0.841304 - 1.16119 1.16119 - -0.473218 -0.473218 - 0.567447 0.567447 - -2.51315 -2.51315 - 0.857171 0.857171 - -1.81643 -1.81643 - 1.81088 1.81088 - 1.49892 1.49892 - -0.900041 -0.900041 - -0.0992927 -0.0992927 - 0.635308 0.635308 - 0.0198956 0.0198956 - 2.37949 2.37949 - -1.77301 -1.77301 - -0.716384 -0.716384 - 0.226013 0.226013 - -3.16016 -3.16016 - 0.470524 0.470524 - 0.11595 0.11595 - -2.40913 -2.40913 - -0.394387 -0.394387 - -0.712704 -0.712704 - 0.539429 0.539429 - 2.49816 2.49816 - -0.611206 -0.611206 - -0.562752 -0.562752 - -0.835585 -0.835585 - 3.25424 3.25424 - -0.480187 -0.480187 - 5.29497 5.29497 - -2.8016 -2.8016 - 0.0118055 0.0118055 - 1.86542 1.86542 - 0.101613 0.101613 - -2.00855 -2.00855 - 0.771721 0.771721 - -0.0705836 -0.0705836 - -0.722898 -0.722898 - 0.11892 0.11892 - -1.12107 -1.12107 - 0.23499 0.23499 - -3.2635 -3.2635 - 0.774997 0.774997 - 3.05413 3.05413 - 2.52668 2.52668 - -0.733611 -0.733611 - -0.465726 -0.465726 - 1.61462 1.61462 - -0.297293 -0.297293 - 0.247443 0.247443 - 0.451474 0.451474 - -2.18135 -2.18135 - -0.253176 -0.253176 - -1.77935 -1.77935 - 1.96681 1.96681 - -2.6561 -2.6561 - -0.98605 -0.98605 - 3.74766 3.74766 - -1.86272 -1.86272 - -1.6491 -1.6491 - 1.43297 1.43297 - 3.00856 3.00856 - -2.91703 -2.91703 - 2.31349 2.31349 - 0.612495 0.612495 - 0.303318 0.303318 - 0.96081 0.96081 - 0.401881 0.401881 - -0.130979 -0.130979 - -0.660971 -0.660971 - 0.39712 0.39712 - -1.30811 -1.30811 - 1.66804 1.66804 - -0.362844 -0.362844 - -3.37322 -3.37322 - -0.898031 -0.898031 - -2.35764 -2.35764 - 0.234687 0.234687 - 0.776599 0.776599 - -0.030942 -0.030942 - -3.49072 -3.49072 - -0.354064 -0.354064 - 0.254806 0.254806 - 0.413767 0.413767 - -2.23448 -2.23448 - -3.00986 -3.00986 - -3.03087 -3.03087 - 1.22426 1.22426 - -0.00528415 -0.00528415 - 1.63329 1.63329 - -3.00948 -3.00948 - 0.544208 0.544208 - -2.26614 -2.26614 - 0.202507 0.202507 - 1.10992 1.10992 - -1.14457 -1.14457 - 0.223205 0.223205 - -2.11668 -2.11668 - 0.939281 0.939281 - -1.74119 -1.74119 - 1.36972 1.36972 - -1.02231 -1.02231 - 0.760286 0.760286 - 1.4837 1.4837 - 2.29908 2.29908 - 1.30551 1.30551 - -0.294187 -0.294187 - -1.49058 -1.49058 - 0.292385 0.292385 - -1.13357 -1.13357 - -0.90415 -0.90415 - 0.874832 0.874832 - 1.10395 1.10395 - 2.24293 2.24293 - -1.39277 -1.39277 - -1.87169 -1.87169 - -0.0884342 -0.0884342 - 0.167855 0.167855 - 1.65614 1.65614 - 1.8828 1.8828 - 0.115504 0.115504 - 4.08686 4.08686 - -0.626715 -0.626715 - -0.0646496 -0.0646496 - 0.178995 0.178995 - 1.50054 1.50054 - 1.85495 1.85495 - 1.94429 1.94429 - 0.0459594 0.0459594 - -1.9626 -1.9626 - -1.70859 -1.70859 - -1.05544 -1.05544 - 0.331346 0.331346 - 0.698268 0.698268 - -1.06962 -1.06962 - -2.22214 -2.22214 - 1.438 1.438 - 1.93536 1.93536 - -0.429009 -0.429009 - -1.04787 -1.04787 - -1.8681 -1.8681 - -1.46553 -1.46553 - -0.704678 -0.704678 - -1.80182 -1.80182 - -0.954295 -0.954295 - -2.80916 -2.80916 - -1.25241 -1.25241 - -0.581477 -0.581477 - 1.11401 1.11401 - -0.206182 -0.206182 - -0.0337333 -0.0337333 - -0.192489 -0.192489 - -0.602529 -0.602529 - -0.465761 -0.465761 - 0.886887 0.886887 - -1.47977 -1.47977 - 0.81157 0.81157 - -4.08164 -4.08164 - -0.650256 -0.650256 - -0.239842 -0.239842 - -0.376812 -0.376812 - 0.608583 0.608583 - 3.24043 3.24043 - -2.40365 -2.40365 - -2.12369 -2.12369 - -2.85674 -2.85674 - -1.11819 -1.11819 - -0.183936 -0.183936 - -1.38699 -1.38699 - 0.63657 0.63657 - 3.24116 3.24116 - -0.0696033 -0.0696033 - 1.67356 1.67356 - -0.111105 -0.111105 - 0.928557 0.928557 - -1.21476 -1.21476 - 1.87918 1.87918 - -0.538827 -0.538827 - -1.96541 -1.96541 - -2.25371 -2.25371 - 1.95748 1.95748 - -0.19679 -0.19679 - -0.139908 -0.139908 - 0.621337 0.621337 - -0.019154 -0.019154 - 0.701629 0.701629 - 0.374446 0.374446 - -1.25937 -1.25937 - 1.34433 1.34433 - -0.922861 -0.922861 - 2.90885 2.90885 - -2.56409 -2.56409 - -0.256198 -0.256198 - -0.460552 -0.460552 - 0.178514 0.178514 - 0.182276 0.182276 - 0.646096 0.646096 - -1.0186 -1.0186 - 0.979231 0.979231 - 1.84611 1.84611 - 2.91017 2.91017 - 0.327271 0.327271 - 0.454179 0.454179 - 1.09265 1.09265 - -0.143546 -0.143546 - 0.846766 0.846766 - 0.719947 0.719947 - -1.76553 -1.76553 - 3.30387 3.30387 - 0.321026 0.321026 - 0.201893 0.201893 - 2.04676 2.04676 - 0.304647 0.304647 - -0.753991 -0.753991 - -1.40742 -1.40742 - -0.77855 -0.77855 - 0.29947 0.29947 - -0.920328 -0.920328 - -0.463037 -0.463037 - -1.81857 -1.81857 - -0.305488 -0.305488 - 1.80501 1.80501 - -2.63795 -2.63795 - -1.81647 -1.81647 - -2.82112 -2.82112 - 0.773168 0.773168 - 0.275974 0.275974 - 0.0240539 0.0240539 - -0.764899 -0.764899 - 1.05992 1.05992 - -0.84197 -0.84197 - 1.27771 1.27771 - 0.71536 0.71536 - -0.970194 -0.970194 - -2.22678 -2.22678 - -1.18751 -1.18751 - 2.08652 2.08652 - -1.77634 -1.77634 - 0.330371 0.330371 - -0.459388 -0.459388 - 3.26156 3.26156 - -0.801958 -0.801958 - 0.507587 0.507587 - 0.0705198 0.0705198 - -4.27573 -4.27573 - 1.35474 1.35474 - -0.563817 -0.563817 - -0.036168 -0.036168 - 0.373159 0.373159 - 0.146304 0.146304 - 0.540254 0.540254 - -1.32898 -1.32898 - 0.0550983 0.0550983 - -0.894857 -0.894857 - 0.879887 0.879887 - -1.01703 -1.01703 - 0.262536 0.262536 - 0.605519 0.605519 - -0.682249 -0.682249 - -1.76663 -1.76663 - 0.425809 0.425809 - -1.77275 -1.77275 - -2.40044 -2.40044 - -1.28476 -1.28476 - -1.80539 -1.80539 - 3.42497 3.42497 - -0.737508 -0.737508 - 1.09345 1.09345 - 0.473303 0.473303 - 0.569697 0.569697 - 0.1682 0.1682 - 1.14317 1.14317 - -0.305234 -0.305234 - -0.0553584 -0.0553584 - 0.553831 0.553831 - 2.72079 2.72079 - -1.11628 -1.11628 - -0.897232 -0.897232 - 1.24584 1.24584 - 0.407171 0.407171 - -0.548786 -0.548786 - -0.799434 -0.799434 - -0.337201 -0.337201 - 1.38839 1.38839 - -2.03675 -2.03675 - -0.451153 -0.451153 - -1.51334 -1.51334 - 0.99722 0.99722 - -0.952379 -0.952379 - -1.8534 -1.8534 - 0.427123 0.427123 - -0.813837 -0.813837 - 1.34476 1.34476 - -0.0132453 -0.0132453 - 1.59654 1.59654 - 1.28669 1.28669 - -0.262967 -0.262967 - 0.319661 0.319661 - 0.514152 0.514152 - 8.72845 8.72845 - 0.981584 0.981584 - 2.1889 2.1889 - 2.59352 2.59352 - 4.8238 4.8238 - -1.98665 -1.98665 - -3.87038 -3.87038 - 0.451746 0.451746 - -2.04906 -2.04906 - 0.192938 0.192938 - 1.08472 1.08472 - -0.548332 -0.548332 - -0.871136 -0.871136 - 2.97136 2.97136 - 0.822154 0.822154 - -1.43775 -1.43775 - -1.05529 -1.05529 - 0.332036 0.332036 - -1.01064 -1.01064 - 1.01202 1.01202 - 0.603861 0.603861 - 1.44141 1.44141 - -0.0347525 -0.0347525 - -2.645 -2.645 - -2.77208 -2.77208 - 2.51425 2.51425 - -1.59414 -1.59414 - 2.38812 2.38812 - -0.515258 -0.515258 - -1.70386 -1.70386 - 0.740403 0.740403 - -0.519196 -0.519196 - 3.56872 3.56872 - 2.61736 2.61736 - 0.921465 0.921465 - 0.435321 0.435321 - -0.620441 -0.620441 - 2.59352 2.59352 - -1.05395 -1.05395 - 2.66575 2.66575 - -0.144386 -0.144386 - 1.83629 1.83629 - -0.66284 -0.66284 - 0.456305 0.456305 - 2.47313 2.47313 - -0.947016 -0.947016 - 0.201143 0.201143 - 0.489325 0.489325 - 1.87943 1.87943 - 0.317216 0.317216 - 1.68371 1.68371 - -1.48571 -1.48571 - 1.8859 1.8859 - 0.964108 0.964108 - 3.81978 3.81978 - 1.57192 1.57192 - 3.67288 3.67288 - 0.946176 0.946176 - 1.40396 1.40396 - 0.7311 0.7311 - 1.83264 1.83264 - 1.88672 1.88672 - -1.65002 -1.65002 - -1.27921 -1.27921 - 0.328562 0.328562 - -1.67421 -1.67421 - 1.5893 1.5893 - -0.871527 -0.871527 - 2.71922 2.71922 - 2.10276 2.10276 - -0.53672 -0.53672 - 1.90948 1.90948 - -1.04531 -1.04531 - 0.536989 0.536989 - 0.0668934 0.0668934 - 2.15979 2.15979 - -1.34284 -1.34284 - 2.87173 2.87173 - -1.23698 -1.23698 - 1.12051 1.12051 - -0.241109 -0.241109 - -1.84701 -1.84701 - -1.19951 -1.19951 - -1.08058 -1.08058 - 0.412929 0.412929 - -0.620347 -0.620347 - 3.21218 3.21218 - 0.266479 0.266479 - 0.999923 0.999923 - 0.151579 0.151579 - 0.0457301 0.0457301 - -0.863071 -0.863071 - 0.650015 0.650015 - -0.724167 -0.724167 - 0.943086 0.943086 - -2.67036 -2.67036 - 0.592203 0.592203 - -1.73829 -1.73829 - -0.439297 -0.439297 - 0.35609 0.35609 - -0.415771 -0.415771 - 0.226092 0.226092 - 1.98817 1.98817 - -2.98191 -2.98191 - -0.157906 -0.157906 - -0.0408938 -0.0408938 - 1.32982 1.32982 - -1.98093 -1.98093 - 1.68608 1.68608 - 1.42987 1.42987 - -0.506031 -0.506031 - -3.44969 -3.44969 - -0.670397 -0.670397 - 0.00317784 0.00317784 - -2.44821 -2.44821 - -0.973505 -0.973505 - -2.01111 -2.01111 - 0.0882928 0.0882928 - -1.30306 -1.30306 - 0.606925 0.606925 - -2.90382 -2.90382 - 2.34368 2.34368 - 1.06742 1.06742 - -0.147726 -0.147726 - -0.644699 -0.644699 - 2.38565 2.38565 - 0.423195 0.423195 - -2.39995 -2.39995 - 3.07238 3.07238 - 1.89833 1.89833 - 2.6225 2.6225 - -0.520012 -0.520012 - 2.15609 2.15609 - -3.67348 -3.67348 - -0.0497026 -0.0497026 - 2.84596 2.84596 - 0.620075 0.620075 - -0.554731 -0.554731 - 2.15651 2.15651 - -0.754285 -0.754285 - -0.205464 -0.205464 - -0.530061 -0.530061 - 1.06215 1.06215 - -0.484595 -0.484595 - 0.527476 0.527476 - 2.19013 2.19013 - -2.27807 -2.27807 - -0.523498 -0.523498 - 1.80502 1.80502 - 2.36156 2.36156 - -0.482968 -0.482968 - -1.094 -1.094 - -1.03241 -1.03241 - 1.35803 1.35803 - 2.60347 2.60347 - 0.971085 0.971085 - 0.552662 0.552662 - -0.776221 -0.776221 - 0.828938 0.828938 - 1.69376 1.69376 - -0.0488535 -0.0488535 - 0.585078 0.585078 - -0.783814 -0.783814 - 3.78798 3.78798 - -0.977169 -0.977169 - -0.699405 -0.699405 - -1.00392 -1.00392 - 1.78081 1.78081 - 0.846007 0.846007 - -1.53142 -1.53142 - -0.278166 -0.278166 - -1.7533 -1.7533 - -1.5105 -1.5105 - -0.911813 -0.911813 - -2.28061 -2.28061 - 1.55691 1.55691 - 0.119324 0.119324 - -1.75206 -1.75206 - 1.31127 1.31127 - -1.10018 -1.10018 - 1.50646 1.50646 - 0.671974 0.671974 - 0.408163 0.408163 - -2.99704 -2.99704 - -0.397517 -0.397517 - 1.10579 1.10579 - 0.309227 0.309227 - 0.211714 0.211714 - -1.84012 -1.84012 - -0.727436 -0.727436 - 2.11426 2.11426 - 1.31798 1.31798 - -0.253154 -0.253154 - 1.98683 1.98683 - -0.523705 -0.523705 - 0.645383 0.645383 - 4.59945 4.59945 - -0.753718 -0.753718 - -2.0014 -2.0014 - -2.86775 -2.86775 - -0.51614 -0.51614 - 3.9091 3.9091 - 2.32844 2.32844 - -1.42046 -1.42046 - 2.29523 2.29523 - 0.966474 0.966474 - 0.938259 0.938259 - 0.314803 0.314803 - -0.965298 -0.965298 - 1.36128 1.36128 - 2.32066 2.32066 - 1.15654 1.15654 - 0.208872 0.208872 - -0.40363 -0.40363 - -2.79805 -2.79805 - -1.10689 -1.10689 - 1.06074 1.06074 - 2.46081 2.46081 - -3.64489 -3.64489 - 0.0728799 0.0728799 - 1.73179 1.73179 - -0.529045 -0.529045 - 0.661595 0.661595 - -1.01312 -1.01312 - 1.3391 1.3391 - 0.30467 0.30467 - 2.72219 2.72219 - 4.78742 4.78742 - -3.13914 -3.13914 - 1.88923 1.88923 - 2.43035 2.43035 - 0.547254 0.547254 - 0.406615 0.406615 - -3.88793 -3.88793 - -0.369601 -0.369601 - -0.868166 -0.868166 - -1.47208 -1.47208 - -0.294463 -0.294463 - -1.52052 -1.52052 - -1.18209 -1.18209 - 0.554649 0.554649 - 0.517766 0.517766 - -1.10023 -1.10023 - 1.65443 1.65443 - 2.07565 2.07565 - -0.788139 -0.788139 - 1.2133 1.2133 - -0.364007 -0.364007 - 1.45819 1.45819 - 2.62594 2.62594 - -3.35007 -3.35007 - -0.143461 -0.143461 - 0.21478 0.21478 - 0.800875 0.800875 - -0.329759 -0.329759 - 1.33073 1.33073 - 0.147415 0.147415 - -1.57766 -1.57766 - 0.151905 0.151905 - 1.63538 1.63538 - -0.0241539 -0.0241539 - 0.674812 0.674812 - 0.0364426 0.0364426 - 0.470479 0.470479 - 0.617235 0.617235 - 0.941854 0.941854 - -0.39178 -0.39178 - -3.72198 -3.72198 - 0.249689 0.249689 - 0.790793 0.790793 +... ```` ### Using postprocessing function diff --git a/examples/building_RAG.jl b/examples/building_RAG.jl index d868830f1..fa55c307a 100644 --- a/examples/building_RAG.jl +++ b/examples/building_RAG.jl @@ -141,7 +141,7 @@ first(df, 5) # - Add filtering for semantic similarity (embedding distance) to make sure we don't pick up irrelevant chunks in the context # - Use multiple indices or a hybrid index (add a simple BM25 lookup from TextAnalysis.jl) # - Data processing is the most important step - properly parsed and split text could make wonders -# - Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API)`) +# - Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API) # - Improve the question embedding (eg, rephrase it, generate hypothetical answers and use them to find better context) # # ... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) \ No newline at end of file diff --git a/examples/working_with_ollama.jl b/examples/working_with_ollama.jl index a8ca77aa2..9bf5892b4 100644 --- a/examples/working_with_ollama.jl +++ b/examples/working_with_ollama.jl @@ -70,6 +70,7 @@ tasks = asyncmap(docs) do doc msg = aiembed(schema, doc; model) end embedding = mapreduce(x -> x.content, hcat, tasks) +size(embedding) # ### Using postprocessing function # Add normalization as postprocessing function to normalize embeddings on reception (for easy cosine similarity later) From 9805bb0849f335a31e2451d7a92cef704d838271 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 14:16:38 +0100 Subject: [PATCH 083/251] push version change --- CHANGELOG.md | 7 +++++++ Project.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b62c4688..7202c07cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.5.0] + ### Added - Experimental sub-module RAGTools providing basic Retrieval-Augmented Generation functionality. See `?RAGTools` for more information. It's all nested inside of `PromptingTools.Experimental.RAGTools` to signify that it might change in the future. Key functions are `build_index` and `airag`, but it also provides a suite to make evaluation easier (see `?build_qa_evals` and `?run_qa_evals` or just see the example `examples/building_RAG.jl`) @@ -13,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Stricter code parsing in `AICode` to avoid false positives (code blocks must end with "```\n" to catch comments inside text) - Introduced an option `skip_invalid=true` for `AICode`, which allows you to include only code blocks that parse successfully (useful when the code definition is good, but the subsequent examples are not), and an option `capture_stdout=false` to avoid capturing stdout if you want to evaluate `AICode` in parallel (`Pipe()` that we use is NOT thread-safe) - `OllamaManagedSchema` was passing an incorrect model name to the Ollama server, often serving the default llama2 model instead of the requested model. This is now fixed. +- Fixed a bug in kwarg `model` handling when leveraging PT.MODEL_REGISTRY ## [0.4.0] diff --git a/Project.toml b/Project.toml index dd79425ea..28099aeae 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.5.0-DEV" +version = "0.5.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From 62b3a958be571ebc40d5722d480df6816769b8fb Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 14:38:14 +0100 Subject: [PATCH 084/251] update params --- src/Experimental/RAGTools/evaluation.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Experimental/RAGTools/evaluation.jl b/src/Experimental/RAGTools/evaluation.jl index fe25b32f1..1f0da8329 100644 --- a/src/Experimental/RAGTools/evaluation.jl +++ b/src/Experimental/RAGTools/evaluation.jl @@ -144,8 +144,8 @@ end """ run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; verbose::Bool = true, - parameters_dict::Dict{Symbol, Any}, judge_template::Symbol = :RAGJudgeAnswerFromContext, - model_judge::AbstractString,api_kwargs::NamedTuple = NamedTuple()) -> QAEvalResult + parameters_dict::Dict{Symbol, <:Any}, judge_template::Symbol = :RAGJudgeAnswerFromContext, + model_judge::AbstractString, api_kwargs::NamedTuple = NamedTuple()) -> QAEvalResult Evaluates a single `QAEvalItem` using a RAG context (`RAGContext`) and returns a `QAEvalResult` structure. This function assesses the relevance and accuracy of the answers generated in a QA evaluation context. @@ -180,7 +180,7 @@ eval_result = run_qa_evals(qa_item, ctx, parameters_dict=parameters_dict, model_ ``` """ function run_qa_evals(qa_item::QAEvalItem, ctx::RAGContext; - verbose::Bool = true, parameters_dict::Dict{Symbol, Any} = Dict{Symbol, Any}(), + verbose::Bool = true, parameters_dict::Dict{Symbol, <:Any} = Dict{Symbol, Any}(), judge_template::Symbol = :RAGJudgeAnswerFromContextShort, model_judge::AbstractString = PT.MODEL_CHAT, api_kwargs::NamedTuple = NamedTuple()) @@ -223,7 +223,7 @@ end api_kwargs::NamedTuple = NamedTuple(), airag_kwargs::NamedTuple = NamedTuple(), qa_evals_kwargs::NamedTuple = NamedTuple(), - verbose::Bool = true, parameters_dict::Dict{Symbol, Any} = Dict{Symbol, Any}()) + verbose::Bool = true, parameters_dict::Dict{Symbol, <:Any} = Dict{Symbol, Any}()) Evaluates a vector of `QAEvalItem`s and returns a vector `QAEvalResult`. This function assesses the relevance and accuracy of the answers generated in a QA evaluation context. @@ -262,7 +262,7 @@ function run_qa_evals(index::AbstractChunkIndex, qa_items::AbstractVector{<:QAEv api_kwargs::NamedTuple = NamedTuple(), airag_kwargs::NamedTuple = NamedTuple(), qa_evals_kwargs::NamedTuple = NamedTuple(), - verbose::Bool = true, parameters_dict::Dict{Symbol, Any} = Dict{Symbol, Any}()) + verbose::Bool = true, parameters_dict::Dict{Symbol, <:Any} = Dict{Symbol, Any}()) # Run evaluations in parallel results = asyncmap(qa_items) do qa_item # Generate an answer -- often you want the model_judge to be the highest quality possible, eg, "GPT-4 Turbo" (alias "gpt4t) From b754261b1138e78e24af6f30be20e4b892fc5935 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 23 Dec 2023 17:12:46 +0100 Subject: [PATCH 085/251] remove CreateQAFromContext --- CHANGELOG.md | 1 + Project.toml | 2 +- templates/RAG/CreateQAFromContext.json | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 templates/RAG/CreateQAFromContext.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7202c07cb..2e2bcef98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Fixed +- Removed template `RAG/CreateQAFromContext` because it's a duplicate of `RAG/RAGCreateQAFromContext` ## [0.5.0] diff --git a/Project.toml b/Project.toml index 28099aeae..dd79425ea 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.5.0" +version = "0.5.0-DEV" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/templates/RAG/CreateQAFromContext.json b/templates/RAG/CreateQAFromContext.json deleted file mode 100644 index 83e900ba1..000000000 --- a/templates/RAG/CreateQAFromContext.json +++ /dev/null @@ -1 +0,0 @@ -[{"content":"Template Metadata","description":"For RAG applications. Generate Question and Answer from the provided Context.If you don't have any special instructions, provide `instructions=\"None.\"`. Placeholders: `context`, `instructions`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You are a world-class teacher preparing contextual Question & Answer sets for evaluating AI systems.\"),\n\n**Instructions for Question Generation:**\n1. Analyze the provided Context chunk thoroughly.\n2. Formulate a question that:\n - Is specific and directly related to the information in the context chunk.\n - Is not too short or generic; it should require detailed understanding of the context to answer.\n - Can only be answered using the information from the provided context, without needing external information.\n\n**Instructions for Reference Answer Creation:**\n1. Based on the generated question, compose a reference answer that:\n - Directly and comprehensively answers the question.\n - Stays strictly within the bounds of the provided context chunk.\n - Is clear, concise, and to the point, avoiding unnecessary elaboration or repetition.\n\n**Example 1:**\n- Context Chunk: \"In 1928, Alexander Fleming discovered penicillin, which marked the beginning of modern antibiotics.\"\n- Generated Question: \"What was the significant discovery made by Alexander Fleming in 1928 and its impact?\"\n- Reference Answer: \"Alexander Fleming discovered penicillin in 1928, which led to the development of modern antibiotics.\"\n\nIf the user provides special instructions, prioritize these over the general instructions.\n","variables":[],"_type":"systemmessage"},{"content":"# Context Information\n---\n{{context}}\n---\n\n\n# Special Instructions\n\n{{instructions}}\n","variables":["context","instructions"],"_type":"usermessage"}] \ No newline at end of file From 9a9ff5f3de526c0515656fae143ed3651114d252 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 24 Dec 2023 11:00:35 +0100 Subject: [PATCH 086/251] enable conversations --- CHANGELOG.md | 1 + README.md | 9 +++- src/PromptingTools.jl | 2 +- src/macros.jl | 100 ++++++++++++++++++++++++++++++++++++++-- src/precompilation.jl | 5 ++ src/user_preferences.jl | 30 +++++++++++- src/utils.jl | 74 ++++++++++++++++++++++++++++- test/macros.jl | 89 +++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/utils.jl | 30 ++++++++++++ 10 files changed, 332 insertions(+), 9 deletions(-) create mode 100644 test/macros.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e2bcef98..930cf6784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `@ai_str` macros now support multi-turn conversations. The `ai"something"` call will automatically remember the last conversation, so you can simply reply with `ai!"my-reply"`. If you send another message with `ai""`, you'll start a new conversation. Same for the asynchronous versions `aai""` and `aai!""`. ### Fixed - Removed template `RAG/CreateQAFromContext` because it's a duplicate of `RAG/RAGCreateQAFromContext` diff --git a/README.md b/README.md index 532e260ce..cbb3a33d1 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,12 @@ ai"What is the capital of France?" The returned object is a light wrapper with a generated message in the field `:content` (eg, `ans.content`) for additional downstream processing. +> [!TIP] +> If you want to reply to the previous message, or simply continue the conversation, use `@ai!_str` (notice the bang `!`): +> ```julia +> ai!"And what is the population of it?" +> ``` + You can easily inject any variables with string interpolation: ```julia country = "Spain" @@ -46,6 +52,7 @@ ai"What is the capital of \$(country)?" > [!TIP] > Use after-string-flags to select the model to be called, eg, `ai"What is the capital of France?"gpt4` (use `gpt4t` for the new GPT-4 Turbo model). Great for those extra hard questions! + For more complex prompt templates, you can use handlebars-style templating and provide variables as keyword arguments: ```julia @@ -58,7 +65,7 @@ msg = aigenerate("What is the capital of {{country}}? Is the population larger t > Use `asyncmap` to run multiple AI-powered tasks concurrently. > [!TIP] -> If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` +> If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` (similarly `@ai!_str` -> `@aai!_str` for multi-turn conversations). For more practical examples, see the `examples/` folder and the [Advanced Examples](#advanced-examples) section below. diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index a137bd866..80c792e63 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -62,7 +62,7 @@ include("llm_openai.jl") include("llm_ollama_managed.jl") ## Convenience utils -export @ai_str, @aai_str +export @ai_str, @aai_str, @ai!_str, @aai!_str include("macros.jl") ## Experimental modules diff --git a/src/macros.jl b/src/macros.jl index c3b6e1ea5..a2fe4cefa 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -3,6 +3,8 @@ The `ai""` string macro generates an AI response to a given prompt by using `aigenerate` under the hood. +See also `ai!""` if you want to reply to the provided message / continue the conversation. + ## Arguments - `user_prompt` (String): The input prompt for the AI model. - `model_alias` (optional, any): Provide model alias of the AI model (see `MODEL_ALIASES`). @@ -25,15 +27,72 @@ result = ai"What is `\$a+\$a`?" If you want to use a different model, eg, GPT-4, you can provide its alias as a flag: ```julia -result = ai"What is `1.23 * 100 + 1`?"gpt4 +result = ai"What is `1.23 * 100 + 1`?"gpt4t # AIMessage("The answer is 124.") ``` """ macro ai_str(user_prompt, flags...) + global CONV_HISTORY, MAX_HISTORY_LENGTH + model = isempty(flags) ? MODEL_CHAT : only(flags) + prompt = Meta.parse("\"$(escape_string(user_prompt))\"") + quote + conv = aigenerate($(esc(prompt)); model = $(esc(model)), return_all = true) + push_conversation!($(esc(CONV_HISTORY)), conv, $(esc(MAX_HISTORY_LENGTH))) + last(conv) + end +end + +""" + ai!"user_prompt"[model_alias] -> AIMessage + +The `ai!""` string macro is used to continue a previous conversation with the AI model. + +It appends the new user prompt to the last conversation in the tracked history (in `PromptingTools.CONV_HISTORY`) and generates a response based on the entire conversation context. +If you want to see the previous conversation, you can access it via `PromptingTools.CONV_HISTORY`, which keeps at most last `PromptingTools.MAX_HISTORY_LENGTH` conversations. + +## Arguments +- `user_prompt` (String): The new input prompt to be added to the existing conversation. +- `model_alias` (optional, any): Specify the model alias of the AI model to be used (see `MODEL_ALIASES`). If not provided, the default model is used. + +## Returns +`AIMessage` corresponding to the new user prompt, considering the entire conversation history. + +## Example +To continue a conversation: +```julia +# start conversation as normal +ai"Say hi." + +# ... wait for reply and then react to it: + +# continue the conversation (notice that you can change the model, eg, to more powerful one for better answer) +ai!"What do you think about that?"gpt4t +# AIMessage("Considering our previous discussion, I think that...") +``` + +## Usage Notes +- This macro should be used when you want to maintain the context of an ongoing conversation (ie, the last `ai""` message). +- It automatically accesses and updates the global conversation history. +- If no conversation history is found, it raises an assertion error, suggesting to initiate a new conversation using `ai""` instead. + +## Important +Ensure that the conversation history is not too long to maintain relevancy and coherence in the AI's responses. The history length is managed by `MAX_HISTORY_LENGTH`. +""" +macro ai!_str(user_prompt, flags...) + global CONV_HISTORY model = isempty(flags) ? MODEL_CHAT : only(flags) prompt = Meta.parse("\"$(escape_string(user_prompt))\"") quote - aigenerate($(esc(prompt)); model = $(esc(model))) + @assert !isempty($(esc(CONV_HISTORY))) "No conversation history found. Please use `ai\"\"` instead." + # grab the last conversation + old_conv = $(esc(CONV_HISTORY))[end] + conv = aigenerate(vcat(old_conv, [UserMessage($(esc(prompt)))]); + model = $(esc(model)), + return_all = true) + # replace the last conversation with the new one + $(esc(CONV_HISTORY))[end] = conv + # + last(conv) end end @@ -42,6 +101,8 @@ end Asynchronous version of `@ai_str` macro, which will log the result once it's ready. +See also `aai!""` if you want an asynchronous reply to the provided message / continue the conversation. + # Example Send asynchronous request to GPT-4, so we don't have to wait for the response: @@ -54,13 +115,42 @@ m = aai"Say Hi!"gpt4; # [ Info: AIMessage> Hello! How can I assist you today? """ macro aai_str(user_prompt, flags...) + global CONV_HISTORY, MAX_HISTORY_LENGTH, CONV_HISTORY_LOCK model = isempty(flags) ? MODEL_CHAT : only(flags) prompt = Meta.parse("\"$(escape_string(user_prompt))\"") quote Threads.@spawn begin - m = aigenerate($(esc(prompt)); model = $(esc(model))) - @info "AIMessage> $(m.content)" # display the result once it's ready - m + conv = aigenerate($(esc(prompt)); model = $(esc(model)), return_all = true) + lock($(esc(CONV_HISTORY_LOCK))) do + push_conversation!($(esc(CONV_HISTORY)), conv, $(esc(MAX_HISTORY_LENGTH))) + end + @info "AIMessage> $(last(conv).content)" # display the result once it's ready + last(conv) + end + end +end + +macro aai!_str(user_prompt, flags...) + global CONV_HISTORY, CONV_HISTORY_LOCK + model = isempty(flags) ? MODEL_CHAT : only(flags) + prompt = Meta.parse("\"$(escape_string(user_prompt))\"") + quote + @assert !isempty($(esc(CONV_HISTORY))) "No conversation history found. Please use `aai\"\"` instead." + Threads.@spawn begin + # grab the last conversation + old_conv = $(esc(CONV_HISTORY))[end] + + # send to AI + conv = aigenerate(vcat(old_conv, [UserMessage($(esc(prompt)))]); + model = $(esc(model)), + return_all = true) + + # replace the last conversation with the new one + lock($(esc(CONV_HISTORY_LOCK))) do + $(esc(CONV_HISTORY))[end] = conv + end + @info "AIMessage> $(last(conv).content)" # display the result once it's ready + last(conv) end end end diff --git a/src/precompilation.jl b/src/precompilation.jl index 0a540344a..a1d823cc6 100644 --- a/src/precompilation.jl +++ b/src/precompilation.jl @@ -24,6 +24,11 @@ msg = aiextract(schema, "I want to ask {{it}}"; it = "Is this correct?", return_ image_url = "some_mock_url" msg = aiscan(schema, "Describe the image"; image_url) +# macro calls +ai"Hello"echo +ai!"Hello again"echo +empty!(CONV_HISTORY) + # Use of Templates template_name = :JudgeIsItTrue msg = aigenerate(schema, template_name; it = "Is this correct?") diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 2c33bf563..71456c4a6 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -19,6 +19,8 @@ Check your preferences by calling `get_preferences(key::String)`. See `PROMPT_SCHEMA` for more information. - `MODEL_ALIASES`: A dictionary of model aliases (`alias => full_model_name`). Aliases are used to refer to models by their aliases instead of their full names to make it more convenient to use them. See `MODEL_ALIASES` for more information. +- `MAX_HISTORY_LENGTH`: The maximum length of the conversation history. Defaults to 5. Set to `nothing` to disable history. + See `CONV_HISTORY` for more information. At the moment it is not possible to persist changes to `MODEL_REGISTRY` across sessions. Define your `register_model!()` calls in your `startup.jl` file to make them available across sessions or put them at the top of your script. @@ -55,6 +57,7 @@ function set_preferences!(pairs::Pair{String, <:Any}...) "MODEL_EMBEDDING", "MODEL_ALIASES", "PROMPT_SCHEMA", + "MAX_HISTORY_LENGTH", ] for (key, value) in pairs @assert key in allowed_preferences "Unknown preference '$key'! (Allowed preferences: $(join(allowed_preferences,", "))" @@ -110,6 +113,22 @@ isempty(OPENAI_API_KEY) && const MISTRALAI_API_KEY::String = @load_preference("MISTRALAI_API_KEY", default=get(ENV, "MISTRALAI_API_KEY", "")); +## CONVERSATION HISTORY +""" + CONV_HISTORY + +Tracks the most recent conversations through the `ai_str macros`. + +Preference available: MAX_HISTORY_LENGTH, which sets how many last messages should be remembered. + +See also: `push_conversation!`, `resize_conversation!` + +""" +const CONV_HISTORY = Vector{Vector{<:Any}}() +const CONV_HISTORY_LOCK = ReentrantLock() +const MAX_HISTORY_LENGTH = @load_preference("MAX_HISTORY_LENGTH", + default=5)::Union{Int, Nothing} + ## Model registry # A dictionary of model names and their specs (ie, name, costs per token, etc.) # Model specs are saved in ModelSpec struct (see below) @@ -288,7 +307,16 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", MistralOpenAISchema(), 1.08e-7, 0.0, - "Mistral AI's hosted model for embeddings.")) + "Mistral AI's hosted model for embeddings."), + "echo" => ModelSpec("echo", + TestEchoOpenAISchema(; + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, + :prompt_tokens => 2, + :completion_tokens => 1)), status = 200), + 0.0, + 0.0, + "Echo is only for testing. It always responds with 'Hello!'")) ### Model Registry Structure @kwdef mutable struct ModelRegistry diff --git a/src/utils.jl b/src/utils.jl index a0f6b996b..4592bd6d6 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -251,4 +251,76 @@ _encode_local_image(::Nothing) = String[] # Used for image_url in aiscan to provided consistent output type _string_to_vector(s::AbstractString) = [s] -_string_to_vector(v::Vector{<:AbstractString}) = v \ No newline at end of file +_string_to_vector(v::Vector{<:AbstractString}) = v + +### Conversation Management + +""" + push_conversation!(conv_history, conversation::AbstractVector, max_history::Union{Int, Nothing}) + +Add a new conversation to the conversation history and resize the history if necessary. + +This function appends a conversation to the `conv_history`, which is a vector of conversations. Each conversation is represented as a vector of `AbstractMessage` objects. After adding the new conversation, the history is resized according to the `max_history` parameter to ensure that the size of the history does not exceed the specified limit. + +## Arguments +- `conv_history`: A vector that stores the history of conversations. Typically, this is `PT.CONV_HISTORY`. +- `conversation`: The new conversation to be added. It should be a vector of `AbstractMessage` objects. +- `max_history`: The maximum number of conversations to retain in the history. If `Nothing`, the history is not resized. + +## Returns +The updated conversation history. + +## Example +```julia +new_conversation = aigenerate("Hello World"; return_all = true) +push_conversation!(PT.CONV_HISTORY, new_conversation, 10) +``` + +This is done automatically by the ai"" macros. +""" +function push_conversation!(conv_history::Vector{<:Vector{<:Any}}, + conversation::AbstractVector, + max_history::Union{Int, Nothing}) + if isnothing(max_history) + return + end + push!(conv_history, conversation) + resize_conversation!(conv_history, max_history) + return conv_history +end + +""" + resize_conversation!(conv_history, max_history::Union{Int, Nothing}) + +Resize the conversation history to a specified maximum length. + +This function trims the `conv_history` to ensure that its size does not exceed `max_history`. It removes the oldest conversations first if the length of `conv_history` is greater than `max_history`. + +## Arguments +- `conv_history`: A vector that stores the history of conversations. Typically, this is `PT.CONV_HISTORY`. +- `max_history`: The maximum number of conversations to retain in the history. If `Nothing`, the history is not resized. + +## Returns +The resized conversation history. + +## Example +```julia +resize_conversation!(PT.CONV_HISTORY, PT.MAX_HISTORY_LENGTH) +``` + +After the function call, `conv_history` will contain only the 10 most recent conversations. + +This is done automatically by the ai"" macros. + +""" +function resize_conversation!(conv_history, + max_history::Union{Int, Nothing}) + if isnothing(max_history) + return + end + + while length(conv_history) > max_history + popfirst!(conv_history) + end + return conv_history +end \ No newline at end of file diff --git a/test/macros.jl b/test/macros.jl new file mode 100644 index 000000000..aa3addaed --- /dev/null +++ b/test/macros.jl @@ -0,0 +1,89 @@ +using PromptingTools: @ai_str, @aai_str, @ai!_str, @aai!_str +using PromptingTools: TestEchoOpenAISchema, push_conversation!, CONV_HISTORY, UserMessage + +# Develop the test for all ai"" macros... +# eg, ai"Hello echo"echo0 will send it to our echo model + +# Global variables for conversation history and max length for testing purposes + +@testset "ai_str,ai!_str" begin + ## Setup echo + # corresponds to OpenAI API v1 + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + PT.register_model!(; + name = "echo0", + schema = TestEchoOpenAISchema(; response, status = 200)) + + # Test generation of AI response using the basic macro with no model alias (default model) + response = ai"Hello, how are you?"echo0 # simple call using the default model + @test response.content == "Hello!" + schema_ref = PT.MODEL_REGISTRY["echo0"].schema + @test schema_ref.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant") + Dict("role" => "user", "content" => "Hello, how are you?")] + + # Test the macro with string interpolation + a = 1 + response = ai"What is `$a+$a`?"echo0 # Test with interpolated variable + schema_ref = PT.MODEL_REGISTRY["echo0"].schema + @test schema_ref.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant") + Dict("role" => "user", "content" => "What is `1+1`?")] + + # ai!_str_macro" begin + # Prepopulate conversation history + push_conversation!(CONV_HISTORY, [AIMessage("Say hi.")], 999) + + # Test if it continues the conversation as expected + response = ai!"Hi again!"echo0 # continue the conversation + schema_ref = PT.MODEL_REGISTRY["echo0"].schema + @test schema_ref.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "assistant", "content" => "Say hi."), + Dict("role" => "user", "content" => "Hi again!")] + + @test CONV_HISTORY[end][end].content == "Hello!" + + # Test an assertion that there is conversation history + empty!(CONV_HISTORY) + @test_throws AssertionError ai!"Where are you located?" + + # clean up + empty!(CONV_HISTORY) +end + +@testset "aai_str,aai!_str" begin + ## Setup echo + # corresponds to OpenAI API v1 + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + PT.register_model!(; + name = "echo0", + schema = TestEchoOpenAISchema(; response, status = 200)) + + # default test + response = aai"Hello, how are you?"echo0 # simple call using the default model + wait(response) # Wait for the task to complete + @test fetch(response).content == "Hello!" + schema_ref = PT.MODEL_REGISTRY["echo0"].schema + @test schema_ref.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant") + Dict("role" => "user", "content" => "Hello, how are you?")] + @test CONV_HISTORY[end][end].content == "Hello!" + + # continue conversation + push_conversation!(CONV_HISTORY, [AIMessage("Say hi.")], 999) + response = aai!"Hi again!"echo0 # continue the conversation + wait(response) # Wait for the task to complete + schema_ref = PT.MODEL_REGISTRY["echo0"].schema + @test schema_ref.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "assistant", "content" => "Say hi."), + Dict("role" => "user", "content" => "Hi again!")] + + @test CONV_HISTORY[end][end].content == "Hello!" + + # clean up + empty!(CONV_HISTORY) +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index d8bc24dad..1d413fa3c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,6 +18,7 @@ end include("llm_shared.jl") include("llm_openai.jl") include("llm_ollama_managed.jl") + include("macros.jl") include("templates.jl") include("serialization.jl") include("code_generation.jl") diff --git a/test/utils.jl b/test/utils.jl index ceabdd5e7..1af4c3166 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -2,6 +2,7 @@ using PromptingTools: split_by_length, replace_words using PromptingTools: _extract_handlebar_variables, call_cost, _report_stats using PromptingTools: _string_to_vector, _encode_local_image using PromptingTools: DataMessage, AIMessage +using PromptingTools: push_conversation!, resize_conversation! @testset "replace_words" begin words = ["Disney", "Snow White", "Mickey Mouse"] @@ -142,3 +143,32 @@ end @test output2[1] == output2[2] == output @test_throws AssertionError _encode_local_image("not an path") end + +### Conversation Management +@testset "push_conversation!,resize_conversation!" begin + # Test 1: Adding to Conversation History + conv_history = Vector{Vector{<:Any}}() + conversation = [AIMessage("Test message")] + push_conversation!(conv_history, conversation, 5) + @test length(conv_history) == 1 + @test conv_history[end] === conversation + + # Test 2: History Resize on Addition + max_history = 5 + conv_history = [[AIMessage("Test message")] for i in 1:max_history] + new_conversation = [AIMessage("Test message")] + push_conversation!(conv_history, new_conversation, max_history) + @test length(conv_history) == max_history + @test conv_history[end] === new_conversation + + # Test 3: Manual Resize + max_history = 5 + conv_history = [[AIMessage("Test message")] for i in 1:(max_history + 2)] + resize_conversation!(conv_history, max_history) + @test length(conv_history) == max_history + + # Test 4: No Resize with Nothing + conv_history = [[AIMessage("Test message")] for i in 1:7] + resize_conversation!(conv_history, nothing) + @test length(conv_history) == 7 +end \ No newline at end of file From 7bc74f61929593d3342505569789111dc6a77f30 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 24 Dec 2023 11:02:42 +0100 Subject: [PATCH 087/251] update docs --- docs/src/getting_started.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index 27245ac8f..bb25e667c 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -58,6 +58,11 @@ AIMessage("The capital of France is Paris.") Returned object is a light wrapper with generated message in field `:content` (eg, `ans.content`) for additional downstream processing. +If you want to reply to the previous message, or simply continue the conversation, use `@ai!_str` (notice the bang `!`): +```julia +ai!"And what is the population of it?" +``` + You can easily inject any variables with string interpolation: ```julia country = "Spain" @@ -86,6 +91,6 @@ AIMessage("The capital of Spain is Madrid. And yes, the population of Madrid is Pro tip: Use `asyncmap` to run multiple AI-powered tasks concurrently. -Pro tip: If you use slow models (like GPT-4), you can use async version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` +Pro tip: If you use slow models (like GPT-4), you can use the asynchronous version of `@ai_str` -> `@aai_str` to avoid blocking the REPL, eg, `aai"Say hi but slowly!"gpt4` (similarly `@ai!_str` -> `@aai!_str` for multi-turn conversations). For more practical examples, see the [Various Examples](@ref) section. \ No newline at end of file From cb00a53c8fb79c352e1aad6660e4fd0ab16f98e1 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 24 Dec 2023 11:11:35 +0100 Subject: [PATCH 088/251] update tests --- src/utils.jl | 3 --- test/utils.jl | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 4592bd6d6..70741ac44 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -281,9 +281,6 @@ This is done automatically by the ai"" macros. function push_conversation!(conv_history::Vector{<:Vector{<:Any}}, conversation::AbstractVector, max_history::Union{Int, Nothing}) - if isnothing(max_history) - return - end push!(conv_history, conversation) resize_conversation!(conv_history, max_history) return conv_history diff --git a/test/utils.jl b/test/utils.jl index 1af4c3166..eb705ca11 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -160,6 +160,11 @@ end push_conversation!(conv_history, new_conversation, max_history) @test length(conv_history) == max_history @test conv_history[end] === new_conversation + push_conversation!(conv_history, new_conversation, nothing) + push_conversation!(conv_history, new_conversation, nothing) + push_conversation!(conv_history, new_conversation, nothing) + @test length(conv_history) > max_history + @test conv_history[end] === new_conversation # Test 3: Manual Resize max_history = 5 From 227c213f429357b777901c08773f004689a565eb Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 24 Dec 2023 17:59:41 +0000 Subject: [PATCH 089/251] Add new OllamaSchema --- CHANGELOG.md | 1 + README.md | 10 +- docs/src/examples/working_with_ollama.md | 24 +- examples/working_with_ollama.jl | 18 +- src/PromptingTools.jl | 1 + src/llm_interface.jl | 22 ++ src/llm_ollama.jl | 347 +++++++++++++++++++++++ src/llm_ollama_managed.jl | 31 +- src/messages.jl | 25 +- src/user_preferences.jl | 17 +- src/utils.jl | 15 +- test/llm_ollama.jl | 145 ++++++++++ test/llm_ollama_managed.jl | 31 ++ test/runtests.jl | 1 + test/utils.jl | 6 + 15 files changed, 656 insertions(+), 38 deletions(-) create mode 100644 src/llm_ollama.jl create mode 100644 test/llm_ollama.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 930cf6784..a23158594 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `@ai_str` macros now support multi-turn conversations. The `ai"something"` call will automatically remember the last conversation, so you can simply reply with `ai!"my-reply"`. If you send another message with `ai""`, you'll start a new conversation. Same for the asynchronous versions `aai""` and `aai!""`. +- Created a new default schema for Ollama models `OllamaSchema` (replacing `OllamaManagedSchema`), which allows multi-turn conversations and conversations with images (eg, with Llava and Bakllava models). `OllamaManagedSchema` has been kept for compatibility and as an example of a schema where one provides prompt as a string (not dictionaries like OpenAI API). ### Fixed - Removed template `RAG/CreateQAFromContext` because it's a duplicate of `RAG/RAGCreateQAFromContext` diff --git a/README.md b/README.md index cbb3a33d1..9b4860d21 100644 --- a/README.md +++ b/README.md @@ -384,13 +384,19 @@ We can use it with the `aigenerate` function: ```julia const PT = PromptingTools -schema = PT.OllamaManagedSchema() # notice the different schema! +schema = PT.OllamaSchema() # notice the different schema! msg = aigenerate(schema, "Say hi!"; model="openhermes2.5-mistral") # [ Info: Tokens: 69 in 0.9 seconds # AIMessage("Hello! How can I assist you today?") ``` +For common models that have been registered (see `?PT.MODEL_REGISTRY`), you do not need to provide the schema explicitly: + +```julia +msg = aigenerate("Say hi!"; model="openhermes2.5-mistral") +``` + And we can also use the `aiembed` function: ```julia @@ -401,6 +407,8 @@ msg = aiembed(schema, ["Embed me", "Embed me"]; model="openhermes2.5-mistral") msg.content # 4096×2 Matrix{Float64}: ``` +You can now also use `aiscan` to provide images to Ollama models! See the docs for more information. + If you're getting errors, check that Ollama is running - see the [Setup Guide for Ollama](#setup-guide-for-ollama) section below. ### Using MistralAI API and other OpenAI-compatible APIs diff --git a/docs/src/examples/working_with_ollama.md b/docs/src/examples/working_with_ollama.md index 51b195a4d..a9204aa1a 100644 --- a/docs/src/examples/working_with_ollama.md +++ b/docs/src/examples/working_with_ollama.md @@ -25,7 +25,7 @@ Note: You must download these models prior to using them with `ollama pull [!TIP] > If you use Apple Mac M1-3, make sure to provide `api_kwargs=(; options=(; num_gpu=99))` to make sure the whole model is offloaded on your GPU. Current default is 1, which makes some models unusable. Example for running Mixtral: -> `msg = aigenerate(PT.OllamaManagedSchema(), "Count from 1 to 5 and then say hi."; model="dolphin-mixtral:8x7b-v2.5-q4_K_M", api_kwargs=(; options=(; num_gpu=99)))` +> `msg = aigenerate(PT.OllamaSchema(), "Count from 1 to 5 and then say hi."; model="dolphin-mixtral:8x7b-v2.5-q4_K_M", api_kwargs=(; options=(; num_gpu=99)))` ## Text Generation with aigenerate @@ -86,7 +86,7 @@ If you're using some model that is not in the registry, you can either add it: ````julia PT.register_model!(; name = "llama123", - schema = PT.OllamaManagedSchema(), + schema = PT.OllamaSchema(), description = "Some model") PT.MODEL_ALIASES["l123"] = "llama123" # set an alias you like for it ```` @@ -98,7 +98,7 @@ PT.MODEL_ALIASES["l123"] = "llama123" # set an alias you like for it OR define the schema explicitly (to avoid dispatch on global `PT.PROMPT_SCHEMA`): ````julia -schema = PT.OllamaManagedSchema() +schema = PT.OllamaSchema() aigenerate(schema, "Say hi!"; model = "llama2") ```` @@ -106,11 +106,23 @@ aigenerate(schema, "Say hi!"; model = "llama2") AIMessage("Hello there! *smiling face* It's nice to meet you! I'm here to help you with any questions or tasks you may have, so feel free to ask me anything. Is there something specific you need assistance with today? 😊") ```` -Note: If you only use Ollama, you can change the default schema to `PT.OllamaManagedSchema()` -via `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaManagedSchema", "MODEL_CHAT"=>"llama2")` +Note: If you only use Ollama, you can change the default schema to `PT.OllamaSchema()` +via `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaSchema", "MODEL_CHAT"=>"llama2")` Restart your session and run `aigenerate("Say hi!")` to test it. +! Note that in version 0.6, we've introduced `OllamaSchema`, which superseded `OllamaManagedSchema` and allows multi-turn conversations and conversations with images (eg, with Llava and Bakllava models). `OllamaManagedSchema` has been kept for compatibility and as an example of a schema where one provides a prompt as a string (not dictionaries like OpenAI API). + +## Providing Images with aiscan + +It's as simple as providing a local image path (keyword `image_path`). You can provide one or more images: + +````julia +msg = aiscan("Describe the image"; image_path=["julia.png","python.png"] model="bakllava") +```` + +`image_url` keyword is not supported at the moment (use `Downloads.download` to download the image locally). + ## Embeddings with aiembed ### Simple embedding for one document @@ -165,7 +177,7 @@ Add normalization as postprocessing function to normalize embeddings on receptio ````julia using LinearAlgebra -schema = PT.OllamaManagedSchema() +schema = PT.OllamaSchema() msg = aiembed(schema, ["embed me", "and me too"], diff --git a/examples/working_with_ollama.jl b/examples/working_with_ollama.jl index 9bf5892b4..78b23f67c 100644 --- a/examples/working_with_ollama.jl +++ b/examples/working_with_ollama.jl @@ -39,19 +39,27 @@ msg = aigenerate(conversation; model) # If you're using some model that is not in the registry, you can either add it: PT.register_model!(; name = "llama123", - schema = PT.OllamaManagedSchema(), + schema = PT.OllamaSchema(), description = "Some model") PT.MODEL_ALIASES["l123"] = "llama123" # set an alias you like for it # OR define the schema explicitly (to avoid dispatch on global `PT.PROMPT_SCHEMA`): -schema = PT.OllamaManagedSchema() +schema = PT.OllamaSchema() aigenerate(schema, "Say hi!"; model = "llama2") -# Note: If you only use Ollama, you can change the default schema to `PT.OllamaManagedSchema()` -# via `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaManagedSchema", "MODEL_CHAT"=>"llama2")` +# Note: If you only use Ollama, you can change the default schema to `PT.OllamaSchema()` +# via `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaSchema", "MODEL_CHAT"=>"llama2")` # # Restart your session and run `aigenerate("Say hi!")` to test it. +# ! Note that in version 0.6, we've introduced `OllamaSchema`, which superseded `OllamaManagedSchema` and allows multi-turn conversations and conversations with images (eg, with Llava and Bakllava models). `OllamaManagedSchema` has been kept for compatibility and as an example of a schema where one provides a prompt as a string (not dictionaries like OpenAI API). + +# ## Providing Images with aiscan + +# It's as simple as providing an image URL (keyword `image_url`) or a local path (keyword `image_path`). You can provide one or more images: + +msg = aiscan("Describe the image"; image_path = ["/test/data/julia.png"]model = "bakllava") + # ## Embeddings with aiembed # ### Simple embedding for one document @@ -75,7 +83,7 @@ size(embedding) # ### Using postprocessing function # Add normalization as postprocessing function to normalize embeddings on reception (for easy cosine similarity later) using LinearAlgebra -schema = PT.OllamaManagedSchema() +schema = PT.OllamaSchema() msg = aiembed(schema, ["embed me", "and me too"], diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 80c792e63..c48c28257 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -60,6 +60,7 @@ include("code_generation.jl") include("llm_shared.jl") include("llm_openai.jl") include("llm_ollama_managed.jl") +include("llm_ollama.jl") ## Convenience utils export @ai_str, @aai_str, @ai!_str, @aai!_str diff --git a/src/llm_interface.jl b/src/llm_interface.jl index aead1cf77..17d7c51c8 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -86,6 +86,28 @@ See `?PREFERENCES` for more details on how to set your API key permanently. """ struct MistralOpenAISchema <: AbstractOpenAISchema end +abstract type AbstractOllamaSchema <: AbstractPromptSchema end + +""" +OllamaSchema is the default schema for Olama models. + +It uses the following conversation template: +``` +[Dict(role="system",content="..."),Dict(role="user",content="..."),Dict(role="assistant",content="...")] +``` + +It's very similar to OpenAISchema, but it appends images differently. +""" +struct OllamaSchema <: AbstractOllamaSchema end + +"Echoes the user's input back to them. Used for testing the implementation" +@kwdef mutable struct TestEchoOllamaSchema <: AbstractOllamaSchema + response::AbstractDict + status::Integer + model_id::String = "" + inputs::Any = nothing +end + abstract type AbstractChatMLSchema <: AbstractPromptSchema end """ ChatMLSchema is used by many open-source chatbots, by OpenAI models (under the hood) and by several models and inferfaces (eg, Ollama, vLLM) diff --git a/src/llm_ollama.jl b/src/llm_ollama.jl new file mode 100644 index 000000000..241af35d2 --- /dev/null +++ b/src/llm_ollama.jl @@ -0,0 +1,347 @@ +## Ollama Chat API +# - llm_olama.jl works by providing messages format to /api/chat +# - llm_managed_olama.jl works by providing 1 system prompt and 1 user prompt /api/generate +# +## Schema dedicated to [Ollama's models](https://ollama.ai/), which also managed the prompt templates +# +## Rendering of converation history for the Ollama API (similar to OpenAI but not for the images) +""" + render(schema::AbstractOllamaSchema, + messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + kwargs...) + +Builds a history of the conversation to provide the prompt to the API. All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. + +# Keyword Arguments +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. + +""" +function render(schema::AbstractOllamaSchema, + messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + kwargs...) + ## + ## First pass: keep the message types but make the replacements provided in `kwargs` + messages_replaced = render(NoSchema(), messages; conversation, kwargs...) + + ## Second pass: convert to the OpenAI schema + conversation = Dict{String, Any}[] + + # replace any handlebar variables in the messages + for msg in messages_replaced + role = if msg isa SystemMessage + "system" + elseif msg isa UserMessage || msg isa UserMessageWithImages + "user" + elseif msg isa AIMessage + "assistant" + end + new_message = Dict{String, Any}("role" => role, "content" => msg.content) + ## Special case for images + if msg isa UserMessageWithImages + new_message["images"] = msg.image_url + end + push!(conversation, new_message) + end + + return conversation +end + +## Ollama back-end +# uses ollama_api defined in src/llm_ollama_managed.jl +function ollama_api(prompt_schema::TestEchoOllamaSchema, + prompt::Union{AbstractString, Nothing} = nothing; + system::Union{Nothing, AbstractString} = nothing, + messages = [], + endpoint::String = "generate", + model::String = "llama2", kwargs...) + prompt_schema.model_id = model + prompt_schema.inputs = messages + return prompt_schema +end + +## User-Facing API +""" + aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE; verbose::Bool = true, + api_key::String = "", model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + +Generate an AI response based on a given prompt using the OpenAI API. + +# Arguments +- `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema` not `AbstractManagedSchema`) +- `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` +- `verbose`: A boolean indicating whether to print additional information. +- `api_key`: Provided for interface consistency. Not needed for locally hosted Ollama. +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). +- `conversation::AbstractVector{<:AbstractMessage}=[]`: Not allowed for this schema. Provided only for compatibility. +- `http_kwargs::NamedTuple`: Additional keyword arguments for the HTTP request. Defaults to empty `NamedTuple`. +- `api_kwargs::NamedTuple`: Additional keyword arguments for the Ollama API. Defaults to an empty `NamedTuple`. +- `kwargs`: Prompt variables to be used to fill the prompt/template + +# Returns +- `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. + Use `msg.content` to access the extracted string. + +See also: `ai_str`, `aai_str`, `aiembed` + +# Example + +Simple hello world to test the API: +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() # We need to explicit if we want Ollama, OpenAISchema is the default + +msg = aigenerate(schema, "Say hi!"; model="openhermes2.5-mistral") +# [ Info: Tokens: 69 in 0.9 seconds +# AIMessage("Hello! How can I assist you today?") +``` + +`msg` is an `AIMessage` object. Access the generated string via `content` property: +```julia +typeof(msg) # AIMessage{SubString{String}} +propertynames(msg) # (:content, :status, :tokens, :elapsed +msg.content # "Hello! How can I assist you today?" +``` + +Note: We need to be explicit about the schema we want to use. If we don't, it will default to `OpenAISchema` (=`PT.DEFAULT_SCHEMA`) +___ +You can use string interpolation: +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() +a = 1 +msg=aigenerate(schema, "What is `\$a+\$a`?"; model="openhermes2.5-mistral") +msg.content # "The result of `1+1` is `2`." +``` +___ +You can provide the whole conversation or more intricate prompts as a `Vector{AbstractMessage}`: +```julia +const PT = PromptingTools +schema = PT.OllamaManagedSchema() + +conversation = [ + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] + +msg = aigenerate(schema, conversation; model="openhermes2.5-mistral") +# [ Info: Tokens: 111 in 2.1 seconds +# AIMessage("Strong the attachment is, it leads to suffering it may. Focus on the force within you must, ...") +``` + +Note: Managed Ollama currently supports at most 1 User Message and 1 System Message given the API limitations. If you want more, you need to use the `ChatMLSchema`. +""" +function aigenerate(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_TYPE; + verbose::Bool = true, + api_key::String = "", + model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + http_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + ## + global MODEL_ALIASES + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) + + if !dry_run + time = @elapsed resp = ollama_api(prompt_schema, nothing; + system = nothing, messages = conv_rendered, endpoint = "chat", model = model_id, + http_kwargs, + api_kwargs...) + + msg = AIMessage(; content = resp.response[:message][:content] |> strip, + status = Int(resp.status), + tokens = (resp.response[:prompt_eval_count], + resp.response[:eval_count]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id) + else + msg = nothing + end + + ## Select what to return + output = finalize_outputs(prompt, + conv_rendered, + msg; + conversation, + return_all, + dry_run, + kwargs...) + return output +end + +function aiembed(prompt_schema::AbstractOllamaSchema, args...; kwargs...) + aiembed(OllamaManagedSchema(), args...; kwargs...) +end + +""" +aiscan([prompt_schema::AbstractOllamaSchema,] prompt::ALLOWED_PROMPT_TYPE; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + attach_to_latest::Bool = true, + verbose::Bool = true, api_key::String = OPENAI_API_KEY, + model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + http_kwargs::NamedTuple = (; + retry_non_idempotent = true, + retries = 5, + readtimeout = 120), + api_kwargs::NamedTuple = = (; max_tokens = 2500), + kwargs...) + +Scans the provided image (`image_url` or `image_path`) with the goal provided in the `prompt`. + +Can be used for many multi-modal tasks, such as: OCR (transcribe text in the image), image captioning, image classification, etc. + +It's effectively a light wrapper around `aigenerate` call, which uses additional keyword arguments `image_url`, `image_path`, `image_detail` to be provided. + At least one image source (url or path) must be provided. + +# Arguments +- `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema`) +- `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` +- `image_url`: A string or vector of strings representing the URL(s) of the image(s) to scan. +- `image_path`: A string or vector of strings representing the path(s) of the image(s) to scan. +- `image_detail`: A string representing the level of detail to include for images. Can be `"auto"`, `"high"`, or `"low"`. See [OpenAI Vision Guide](https://platform.openai.com/docs/guides/vision) for more details. +- `attach_to_latest`: A boolean how to handle if a conversation with multiple `UserMessage` is provided. When `true`, the images are attached to the latest `UserMessage`. +- `verbose`: A boolean indicating whether to print additional information. +- `api_key`: A string representing the API key for accessing the OpenAI API. +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. +- `http_kwargs`: A named tuple of HTTP keyword arguments. +- `api_kwargs`: A named tuple of API keyword arguments. +- `kwargs`: Prompt variables to be used to fill the prompt/template + +# Returns +If `return_all=false` (default): +- `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. + Use `msg.content` to access the extracted string. + +If `return_all=true`: +- `conversation`: A vector of `AbstractMessage` objects representing the full conversation history, including the response from the AI model (`AIMessage`). + +See also: `ai_str`, `aai_str`, `aigenerate`, `aiembed`, `aiclassify`, `aiextract`, `aitemplates` + +# Notes + +- All examples below use model "gpt4v", which is an alias for model ID "gpt-4-vision-preview" +- `max_tokens` in the `api_kwargs` is preset to 2500, otherwise OpenAI enforces a default of only a few hundred tokens (~300). If your output is truncated, increase this value + +# Example + +Describe the provided image: +```julia +msg = aiscan("Describe the image"; image_path="julia.png", model="bakllava") +# [ Info: Tokens: 1141 @ Cost: \$0.0117 in 2.2 seconds +# AIMessage("The image shows a logo consisting of the word "julia" written in lowercase") +``` + +You can provide multiple images at once as a vector and ask for "low" level of detail (cheaper): +```julia +msg = aiscan("Describe the image"; image_path=["julia.png","python.png"] model="bakllava") +``` + +You can use this function as a nice and quick OCR (transcribe text in the image) with a template `:OCRTask`. +Let's transcribe some SQL code from a screenshot (no more re-typing!): + +```julia +using Downloads +# Screenshot of some SQL code -- we cannot use image_url directly, so we need to download it first +image_url = "https://www.sqlservercentral.com/wp-content/uploads/legacy/8755f69180b7ac7ee76a69ae68ec36872a116ad4/24622.png" +image_path = Downloads.download(image_url) +msg = aiscan(:OCRTask; image_path, model="bakllava", task="Transcribe the SQL code in the image.", api_kwargs=(; max_tokens=2500)) + +# AIMessage("```sql +# update Orders + +# You can add syntax highlighting of the outputs via Markdown +using Markdown +msg.content |> Markdown.parse +``` + +Local models cannot handle image URLs directly (`image_url`), so you need to download the image first and provide it as `image_path`: + +```julia +using Downloads +image_path = Downloads.download(image_url) +``` + +Notice that we set `max_tokens = 2500`. If your outputs seem truncated, it might be because the default maximum tokens on the server is set too low! + +""" +function aiscan(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_TYPE; + image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + attach_to_latest::Bool = true, + verbose::Bool = true, + api_key::String = OPENAI_API_KEY, + model::String = MODEL_CHAT, + return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = (; max_tokens = 2500), + kwargs...) + ## Checks + @assert isnothing(image_url) "Keyword `image_url` currently is not allowed for local models. Please download the file locally first with `image_path = Downloads.download(image_url)` and provide it as an `image_path`." + ## + global MODEL_ALIASES + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + ## Vision-specific functionality + msgs = attach_images_to_user_message(prompt; + image_url, + image_path, + attach_to_latest, + base64_only = true) + + ## Build the conversation, pass what image detail is required (if provided) + conv_rendered = render(prompt_schema, msgs; conversation, kwargs...) + if !dry_run + ## Model call + time = @elapsed resp = ollama_api(prompt_schema, nothing; + system = nothing, messages = conv_rendered, endpoint = "chat", model = model_id, + http_kwargs, + api_kwargs...) + msg = AIMessage(; content = resp.response[:message][:content] |> strip, + status = Int(resp.status), + tokens = (resp.response[:prompt_eval_count], + resp.response[:eval_count]), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id) + else + msg = nothing + end + + ## Select what to return // input `msgs` to preserve the image attachments + output = finalize_outputs(msgs, + conv_rendered, + msg; + conversation, + return_all, + dry_run, + kwargs...) + + return output +end + +function aiclassify(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_TYPE; + kwargs...) + error("Managed schema does not support aiclassify. Please use OpenAISchema instead.") +end +function aiextract(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_TYPE; + kwargs...) + error("Managed schema does not support aiextract. Please use OpenAISchema instead.") +end diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index a68d6f27d..e6d12df80 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -1,3 +1,7 @@ +## Ollama Generation API +# - llm_olama.jl works by providing messages format to /api/chat +# - llm_managed_olama.jl works by providing 1 system prompt and 1 user prompt /api/generate +# ## Schema dedicated to [Ollama's managed models](https://ollama.ai/), which also managed the prompts ## It's limited to 2 messages (system and user), because there are only two slots for `system` and `prompt` ## @@ -51,9 +55,11 @@ end ## Model-calling """ - ollama_api(prompt_schema::AbstractOllamaManagedSchema, prompt::AbstractString, + ollama_api(prompt_schema::Union{AbstractOllamaManagedSchema, AbstractOllamaSchema}, + prompt::Union{AbstractString, Nothing} = nothing; system::Union{Nothing, AbstractString} = nothing, - endpoint::String = "generate"; + messages::Vector{<:AbstractMessage} = AbstractMessage[], + endpoint::String = "generate", model::String = "llama2", http_kwargs::NamedTuple = NamedTuple(), stream::Bool = false, url::String = "localhost", port::Int = 11434, @@ -73,16 +79,24 @@ Simple wrapper for a call to Ollama API. - `port`: The port of the Ollama API. Defaults to 11434. - `kwargs`: Prompt variables to be used to fill the prompt/template """ -function ollama_api(prompt_schema::AbstractOllamaManagedSchema, prompt::AbstractString; +function ollama_api(prompt_schema::Union{AbstractOllamaManagedSchema, AbstractOllamaSchema}, + prompt::Union{AbstractString, Nothing} = nothing; system::Union{Nothing, AbstractString} = nothing, + messages::Vector{<:AbstractDict{String, <:Any}} = Vector{Dict{String, Any}}(), endpoint::String = "generate", model::String = "llama2", http_kwargs::NamedTuple = NamedTuple(), stream::Bool = false, url::String = "localhost", port::Int = 11434, kwargs...) - @assert endpoint in ["generate", "embeddings"] "Only 'generate' and 'embeddings' Ollama endpoints are supported." + @assert endpoint in ["chat", "generate", "embeddings"] "Only 'chat', 'generate' and 'embeddings' Ollama endpoints are supported." ## - body = Dict("model" => model, "stream" => stream, "prompt" => prompt, kwargs...) + body = if !isnothing(prompt) + Dict("model" => model, "stream" => stream, "prompt" => prompt, kwargs...) + elseif !isempty(messages) + Dict("model" => model, "stream" => stream, "messages" => messages, kwargs...) + else + error("No prompt or messages provided! Stopping.") + end if !isnothing(system) body["system"] = system end @@ -95,8 +109,11 @@ function ollama_api(prompt_schema::AbstractOllamaManagedSchema, prompt::Abstract return (; response = body, resp.status) end # For testing -function ollama_api(prompt_schema::TestEchoOllamaManagedSchema, prompt::AbstractString; - system::Union{Nothing, AbstractString} = nothing, endpoint::String = "generate", +function ollama_api(prompt_schema::TestEchoOllamaManagedSchema, + prompt::Union{AbstractString, Nothing} = nothing; + system::Union{Nothing, AbstractString} = nothing, + messages = [], + endpoint::String = "generate", model::String = "llama2", kwargs...) prompt_schema.model_id = model prompt_schema.inputs = (; system, prompt) diff --git a/src/messages.jl b/src/messages.jl index 0a2400cc1..63d153b19 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -94,11 +94,13 @@ end "Construct `UserMessageWithImages` with 1 or more images. Images can be either URLs or local paths." function UserMessageWithImages(prompt::AbstractString; image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, - image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing) + image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + base64_only::Bool = false) @assert !(isnothing(image_url) && isnothing(image_path)) "At least one of `image_url` and `image_path` must be provided." url1 = !isnothing(image_url) ? _string_to_vector(image_url) : String[] # Process local image - url2 = !isnothing(image_path) ? _string_to_vector(_encode_local_image(image_path)) : + url2 = !isnothing(image_path) ? + _string_to_vector(_encode_local_image(image_path; base64_only)) : String[] return UserMessageWithImages(; content = prompt, @@ -111,8 +113,9 @@ end function attach_images_to_user_message(prompt::AbstractString; image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + base64_only::Bool = false, kwargs...) - UserMessageWithImages(prompt; image_url, image_path) + UserMessageWithImages(prompt; image_url, image_path, base64_only) end function attach_images_to_user_message(msg::UserMessageWithImages; kwargs...) throw(AssertionError("Cannot attach additional images to UserMessageWithImages.")) @@ -120,13 +123,15 @@ end function attach_images_to_user_message(msg::UserMessage; image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + base64_only::Bool = false, kwargs...) - UserMessageWithImages(msg.content; image_url, image_path) + UserMessageWithImages(msg.content; image_url, image_path, base64_only) end # automatically attach images to the latest user message, if not allowed, throw an error if more than 2 user messages provided function attach_images_to_user_message(msgs::Vector{T}; image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, + base64_only::Bool = false, attach_to_latest::Bool = true) where {T <: AbstractChatMessage} # Check how to add images to UserMessage count_user_msgs = count(isusermessage, msgs) @@ -136,7 +141,7 @@ function attach_images_to_user_message(msgs::Vector{T}; idx = findlast(isusermessage, msgs) # re-type to accept UserMessageWithImages type msgs = convert(Vector{typejoin(UserMessageWithImages, T)}, msgs) - msgs[idx] = attach_images_to_user_message(msgs[idx]; image_url, image_path) + msgs[idx] = attach_images_to_user_message(msgs[idx]; image_url, image_path, base64_only) return msgs end @@ -171,11 +176,11 @@ function Base.show(io::IO, ::MIME"text/plain", m::AbstractDataMessage) end ## Dispatch for render -function render(schema::AbstractPromptSchema, - messages::Vector{<:AbstractMessage}; - kwargs...) - render(schema, messages; kwargs...) -end +# function render(schema::AbstractPromptSchema, +# messages::Vector{<:AbstractMessage}; +# kwargs...) +# render(schema, messages; kwargs...) +# end function render(schema::AbstractPromptSchema, msg::AbstractMessage; kwargs...) render(schema, [msg]; kwargs...) end diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 71456c4a6..a0fb21872 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -269,25 +269,34 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", 0.0, "Text Embedding Ada is a 1.75T parameter model and the largest model available on the OpenAI API."), "llama2" => ModelSpec("llama2", - OllamaManagedSchema(), + OllamaSchema(), 0.0, 0.0, "LLAMA2 is a 7B parameter model from Meta."), "openhermes2.5-mistral" => ModelSpec("openhermes2.5-mistral", - OllamaManagedSchema(), + OllamaSchema(), 0.0, 0.0, "OpenHermes 2.5 Mistral is a 7B parameter model finetuned by X on top of base model from Mistral AI."), "starling-lm" => ModelSpec("starling-lm", - OllamaManagedSchema(), + OllamaSchema(), 0.0, 0.0, "Starling LM is a 7B parameter model finetuned by X on top of base model from Starling AI."), "yi:34b-chat" => ModelSpec("yi:34b-chat", - OllamaManagedSchema(), + OllamaSchema(), 0.0, 0.0, "Yi is a 34B parameter model finetuned by X on top of base model from Starling AI."), + "llava" => ModelSpec("llava", + OllamaSchema(), + 0.0, + 0.0, + "A novel end-to-end trained large multimodal model that combines a vision encoder and Vicuna for general-purpose visual and language understanding."), + "bakllava" => ModelSpec("bakllava", + OllamaSchema(), + 0.0, 0.0, + "BakLLaVA is a multimodal model consisting of the Mistral 7B base model augmented with the LLaVA architecture."), "mistral-tiny" => ModelSpec("mistral-tiny", MistralOpenAISchema(), 1.4e-7, diff --git a/src/utils.jl b/src/utils.jl index 70741ac44..37a89a28a 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -235,17 +235,22 @@ function _report_stats(msg, return "Tokens: $(sum(msg.tokens))$(cost_str) in $(round(msg.elapsed;digits=1)) seconds" end # Loads and encodes the provided image path as a base64 string -function _encode_local_image(image_path::AbstractString) +function _encode_local_image(image_path::AbstractString; base64_only::Bool = false) @assert isfile(image_path) "`image_path` must be a valid path to an image file. File: $image_path not found." base64_image = open(image_path, "r") do image_bytes base64encode(image_bytes) end - image_suffix = split(image_path, ".")[end] - image_url = "data:image/$image_suffix;base64,$(base64_image)" + if base64_only + return base64_image + else + image_suffix = split(image_path, ".")[end] + image_url = "data:image/$image_suffix;base64,$(base64_image)" + end return image_url end -function _encode_local_image(image_path::Vector{<:AbstractString}) - return _encode_local_image.(image_path) +function _encode_local_image(image_path::Vector{<:AbstractString}; + base64_only::Bool = false) + return _encode_local_image.(image_path; base64_only) end _encode_local_image(::Nothing) = String[] diff --git a/test/llm_ollama.jl b/test/llm_ollama.jl new file mode 100644 index 000000000..232fc65e9 --- /dev/null +++ b/test/llm_ollama.jl @@ -0,0 +1,145 @@ +using PromptingTools: TestEchoOllamaSchema, render, OllamaSchema, ollama_api +using PromptingTools: AIMessage, SystemMessage, AbstractMessage +using PromptingTools: UserMessage, UserMessageWithImages, DataMessage, _encode_local_image + +@testset "render-Ollama" begin + schema = OllamaSchema() + # Test simple message rendering + messages = [UserMessage("Hello there!")] + expected_output = [ + Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "user", "content" => "Hello there!")] + @test render(schema, messages) == expected_output + + # Test message rendering with handlebar variables + messages = [UserMessage("I am {{name}}")] + expected_output = [ + Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "user", "content" => "I am John Doe"), + ] + @test render(schema, messages; name = "John Doe") == expected_output + + # Test message rendering with system and user messages + messages = [ + SystemMessage("This is a system generated message."), + UserMessage("A user generated reply."), + ] + expected_output = [ + Dict("role" => "system", "content" => "This is a system generated message."), + Dict("role" => "user", "content" => "A user generated reply."), + ] + @test render(schema, messages) == expected_output + + # Test message rendering with images + messages = [ + UserMessageWithImages("User message with an image"; + image_url = ["https://example.com/image.jpg"]), + ] + expected_output = [ + Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "user", + "content" => "User message with an image", + "images" => ["https://example.com/image.jpg"]), + ] + @test render(schema, messages) == expected_output + # Test message with local image + messages = [ + UserMessageWithImages("User message with an image"; + image_path = joinpath(@__DIR__, "data", "julia.png"), base64_only = true), + ] + raw_img = _encode_local_image(joinpath(@__DIR__, "data", "julia.png"); + base64_only = true) + expected_output = [ + Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "user", + "content" => "User message with an image", + "images" => [raw_img]), + ] + @test render(schema, messages) == expected_output +end + +@testset "ollama_api-echo" begin + # corresponds to standard Ollama response format on api/chat endpoint + response = Dict(:message => Dict(:content => "Prompt message"), + :prompt_eval_count => 2, + :eval_count => 1) + schema = TestEchoOllamaSchema(; response, status = 200) + msg = ollama_api(schema, + nothing; + endpoint = "chat", + messages = [SystemMessage("Hi from system.")]) + @test msg.response == response + @test msg.status == 200 + @test schema.inputs == [SystemMessage("Hi from system.")] +end + +@testset "aigenerate-OllamaSchema" begin + response = Dict(:message => Dict(:content => "Prompt message"), + :prompt_eval_count => 2, + :eval_count => 1) + schema = TestEchoOllamaSchema(; response, status = 200) + + # Test aigenerate with a simple UserMessage + prompt = UserMessage("Say hi!") + # Mock dry run without actual API call should return nothing + @test aigenerate(schema, prompt; dry_run = true) === nothing + + # Return the entire conversation (assume mocked result from full conversation from API) + conversation = aigenerate(schema, "hi"; return_all = true) + @test last(conversation).content == "Prompt message" + @test schema.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "user", "content" => "hi")] + conversation = aigenerate(schema, "hi"; return_all = true, conversation) + @test length(conversation) == 5 + @test last(conversation).content == "Prompt message" + @test schema.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "user", "content" => "hi"), + Dict("role" => "assistant", "content" => "Prompt message"), + Dict("role" => "user", "content" => "hi")] + + # Test aigenerate handling kwargs for template replacement + conversation = [SystemMessage("Today's weather is {{weather}}.")] + # Mock dry run replacing the template variable + expected_convo_output = [ + SystemMessage(; content = "Today's weather is sunny.", variables = [:weather]), + ] + @test aigenerate(schema, + conversation; + weather = "sunny", + return_all = true)[1] == expected_convo_output[1] +end + +# @testset "aiembed-ollama" begin +# not testing, it just forwards to previous aiembed which is already tested +# end + +@testset "aiscan-OllamaSchema" begin + response = Dict(:message => Dict(:content => "Prompt message"), + :prompt_eval_count => 2, + :eval_count => 1) + schema = TestEchoOllamaSchema(; response, status = 200) + + conversation = aiscan(schema, + "hi"; + return_all = true, + image_path = joinpath(@__DIR__, "data", "julia.png")) + @test last(conversation).content == "Prompt message" + @test schema.inputs == + [Dict("role" => "system", "content" => "Act as a helpful AI assistant"), + Dict("role" => "user", + "content" => "hi", + "images" => [ + _encode_local_image(joinpath(@__DIR__, "data", "julia.png"), + base64_only = true), + ])] + @test_throws AssertionError aiscan(schema, + "hi"; + return_all = true, + image_url = "not-allowed-url") +end +@testset "not implemented ai* functions" begin + @test_throws ErrorException aiextract(OllamaSchema(), "prompt") + @test_throws ErrorException aiclassify(OllamaSchema(), "prompt") +end \ No newline at end of file diff --git a/test/llm_ollama_managed.jl b/test/llm_ollama_managed.jl index 0ddb7f93f..8901a2169 100644 --- a/test/llm_ollama_managed.jl +++ b/test/llm_ollama_managed.jl @@ -75,6 +75,37 @@ end @test msg.response == response @test msg.status == 200 @test schema.inputs == (; system, prompt) + ## Assert + @test_throws Exception ollama_api(OllamaManagedSchema(), nothing) + @test_throws AssertionError ollama_api(OllamaManagedSchema(), + "x"; + endpoint = "wrong-endpoint") + + ## Run mock server + PORT = rand(1000:2000) + echo_server = HTTP.serve!(PORT, verbose = -1) do req + content = JSON3.read(req.body) + response = Dict(:response => content[:prompt], + :model => content[:model], + :prompt_eval_count => 1, :eval_count => 1) + + return HTTP.Response(200, JSON3.write(response)) + end + + resp = ollama_api(OllamaManagedSchema(), + "test"; + system = "-", + model = "xyz", + url = "localhost", + port = PORT) + @test resp.status == 200 + @test resp.response == Dict(:response => "test", + :model => "xyz", + :prompt_eval_count => 1, :eval_count => 1) + prompt = "Say Hi!" + + # clean up + close(echo_server) end @testset "aigenerate-ollama" begin diff --git a/test/runtests.jl b/test/runtests.jl index 1d413fa3c..f063201a5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,6 +18,7 @@ end include("llm_shared.jl") include("llm_openai.jl") include("llm_ollama_managed.jl") + include("llm_ollama.jl") include("macros.jl") include("templates.jl") include("serialization.jl") diff --git a/test/utils.jl b/test/utils.jl index eb705ca11..8046016bc 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -142,6 +142,12 @@ end @test output2 isa Vector @test output2[1] == output2[2] == output @test_throws AssertionError _encode_local_image("not an path") + ## Test with base64_only = true + output3 = _encode_local_image(image_path; base64_only = true) + @test !occursin("data:image/png;base64,", output3) + @test "data:image/png;base64," * output3 == output + # Nothing + @test _encode_local_image(nothing) == String[] end ### Conversation Management From 3bc83708794f545200f86e5fa26ee816d3c84856 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 24 Dec 2023 18:06:36 +0000 Subject: [PATCH 090/251] Tag new version --- CHANGELOG.md | 8 +++++++- Project.toml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a23158594..c0927f804 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.6.0] + ### Added - `@ai_str` macros now support multi-turn conversations. The `ai"something"` call will automatically remember the last conversation, so you can simply reply with `ai!"my-reply"`. If you send another message with `ai""`, you'll start a new conversation. Same for the asynchronous versions `aai""` and `aai!""`. -- Created a new default schema for Ollama models `OllamaSchema` (replacing `OllamaManagedSchema`), which allows multi-turn conversations and conversations with images (eg, with Llava and Bakllava models). `OllamaManagedSchema` has been kept for compatibility and as an example of a schema where one provides prompt as a string (not dictionaries like OpenAI API). +- Created a new default schema for Ollama models `OllamaSchema` (replacing `OllamaManagedSchema`), which allows multi-turn conversations and conversations with images (eg, with Llava and Bakllava models). `OllamaManagedSchema` has been kept for compatibility and as an example of a schema where one provides the prompt as a string (not dictionaries like OpenAI API). ### Fixed - Removed template `RAG/CreateQAFromContext` because it's a duplicate of `RAG/RAGCreateQAFromContext` diff --git a/Project.toml b/Project.toml index dd79425ea..b03a17e20 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.5.0-DEV" +version = "0.6.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From 140a2dceb8d20a221104b8d9925d62600c4db662 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 24 Dec 2023 18:32:07 +0000 Subject: [PATCH 091/251] Shift port range in test suite --- test/llm_ollama_managed.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/llm_ollama_managed.jl b/test/llm_ollama_managed.jl index 8901a2169..35a3461c0 100644 --- a/test/llm_ollama_managed.jl +++ b/test/llm_ollama_managed.jl @@ -82,7 +82,7 @@ end endpoint = "wrong-endpoint") ## Run mock server - PORT = rand(1000:2000) + PORT = rand(2000:3000) echo_server = HTTP.serve!(PORT, verbose = -1) do req content = JSON3.read(req.body) response = Dict(:response => content[:prompt], From 80da8c4dfde5e762719c00621695faa0aec379e7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 4 Jan 2024 07:39:29 +0000 Subject: [PATCH 092/251] add AICodeFixer, AgentTools (#44) --- .github/workflows/CI.yml | 1 + CHANGELOG.md | 4 + Project.toml | 7 +- docs/Project.toml | 1 + docs/make.jl | 10 +- docs/src/reference_agenttools.md | 9 + ext/MarkdownPromptingToolsExt.jl | 95 ++++ src/Experimental/AgentTools/AgentTools.jl | 22 + src/Experimental/AgentTools/code_feedback.jl | 125 ++++++ src/Experimental/AgentTools/lazy_types.jl | 407 ++++++++++++++++++ src/Experimental/AgentTools/utils.jl | 50 +++ src/Experimental/Experimental.jl | 4 + src/code_generation.jl | 123 ++++-- src/templates.jl | 5 + src/utils.jl | 35 +- .../agents/code-fixing/CodeFixerRCI.json | 1 + .../agents/code-fixing/CodeFixerShort.json | 1 + .../agents/code-fixing/CodeFixerTiny.json | 1 + test/Experimental/AgentTools/code_feedback.jl | 69 +++ test/Experimental/AgentTools/lazy_types.jl | 165 +++++++ test/Experimental/AgentTools/runtests.jl | 10 + test/Experimental/AgentTools/utils.jl | 58 +++ test/code_generation.jl | 97 +++++ test/runtests.jl | 3 +- test/templates.jl | 1 + test/utils.jl | 37 +- 26 files changed, 1300 insertions(+), 41 deletions(-) create mode 100644 docs/src/reference_agenttools.md create mode 100644 ext/MarkdownPromptingToolsExt.jl create mode 100644 src/Experimental/AgentTools/AgentTools.jl create mode 100644 src/Experimental/AgentTools/code_feedback.jl create mode 100644 src/Experimental/AgentTools/lazy_types.jl create mode 100644 src/Experimental/AgentTools/utils.jl create mode 100644 templates/agents/code-fixing/CodeFixerRCI.json create mode 100644 templates/agents/code-fixing/CodeFixerShort.json create mode 100644 templates/agents/code-fixing/CodeFixerTiny.json create mode 100644 test/Experimental/AgentTools/code_feedback.jl create mode 100644 test/Experimental/AgentTools/lazy_types.jl create mode 100644 test/Experimental/AgentTools/runtests.jl create mode 100644 test/Experimental/AgentTools/utils.jl diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ac5bb55e2..7babf50a3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -22,6 +22,7 @@ jobs: matrix: version: - '1.9' + - '1.10' # - 'nightly' os: - ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index c0927f804..32fc2425a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added new Experimental sub-module AgentTools introducing `AICall` (incl. `AIGenerate`), and `AICodeFixer` structs. The AICall struct provides a "lazy" wrapper for ai* functions, enabling efficient and flexible AI interactions and building Agentic workflows. +- Added the first AI Agent: `AICodeFixer` which iteratively analyzes and improves any code provided by a LLM by evaluating it in a sandbox. It allows a lot of customization (templated responses, feedback function, etc.) See `?AICodeFixer` for more information on usage and `?aicodefixer_feedback` for the example implementation of the feedback function. +- Added `@timeout` macro to allow for limiting the execution time of a block of code in `AICode` via `execution_timeout` kwarg (prevents infinite loops, etc.). See `?AICode` for more information. +- Added `preview(conversation)` utility that allows you to quickly preview the conversation in a Markdown format in your REPL. Requires `Markdown` package for the extension to be loaded. ### Fixed diff --git a/Project.toml b/Project.toml index b03a17e20..73a217949 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.6.0" +version = "0.6.0-DEV" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -14,9 +14,11 @@ Preferences = "21216c6a-2e73-6563-6e65-726566657250" [weakdeps] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] +MarkdownPromptingToolsExt = ["Markdown"] RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] [compat] @@ -26,6 +28,7 @@ HTTP = "1" JSON3 = "1" LinearAlgebra = "<0.0.1, 1" Logging = "<0.0.1, 1" +Markdown = "<0.0.1, 1" OpenAI = "0.8.7" PrecompileTools = "1" Preferences = "1" @@ -40,4 +43,4 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Test", "SparseArrays", "LinearAlgebra"] +test = ["Aqua", "Test", "SparseArrays", "LinearAlgebra", "Markdown"] diff --git a/docs/Project.toml b/docs/Project.toml index 7011d3c5f..8dba66196 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,5 +6,6 @@ JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" PromptingTools = "670122d1-24a8-4d70-bfce-740807c42192" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/docs/make.jl b/docs/make.jl index e2ca988fb..9e2e0543b 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,7 +1,8 @@ using PromptingTools using Documenter -using SparseArrays, LinearAlgebra +using SparseArrays, LinearAlgebra, Markdown using PromptingTools.Experimental.RAGTools +using PromptingTools.Experimental.AgentTools using JSON3, Serialization, DataFramesMeta using Statistics: mean @@ -11,7 +12,11 @@ DocMeta.setdocmeta!(PromptingTools, recursive = true) makedocs(; - modules = [PromptingTools, PromptingTools.Experimental.RAGTools], + modules = [ + PromptingTools, + PromptingTools.Experimental.RAGTools, + PromptingTools.Experimental.AgentTools, + ], authors = "J S <49557684+svilupp@users.noreply.github.com> and contributors", repo = "https://github.com/svilupp/PromptingTools.jl/blob/{commit}{path}#{line}", sitename = "PromptingTools.jl", @@ -36,6 +41,7 @@ makedocs(; "PromptingTools.jl" => "reference.md", "Experimental Modules" => "reference_experimental.md", "RAGTools" => "reference_ragtools.md", + "AgentTools" => "reference_agenttools.md", ], ]) diff --git a/docs/src/reference_agenttools.md b/docs/src/reference_agenttools.md new file mode 100644 index 000000000..6bea34fb2 --- /dev/null +++ b/docs/src/reference_agenttools.md @@ -0,0 +1,9 @@ +# Reference for AgentTools + +```@index +Modules = [PromptingTools.Experimental.AgentTools] +``` + +```@autodocs +Modules = [PromptingTools.Experimental.AgentTools] +``` diff --git a/ext/MarkdownPromptingToolsExt.jl b/ext/MarkdownPromptingToolsExt.jl new file mode 100644 index 000000000..39a50a021 --- /dev/null +++ b/ext/MarkdownPromptingToolsExt.jl @@ -0,0 +1,95 @@ +module MarkdownPromptingToolsExt + +using PromptingTools +using Markdown +const PT = PromptingTools + +""" + preview(msg::PT.AbstractMessage) + +Render a single `AbstractMessage` as a markdown-formatted string, highlighting the role of the message sender and the content of the message. + +This function identifies the type of the message (User, Data, System, AI, or Unknown) and formats it with a header indicating the sender's role, followed by the content of the message. The output is suitable for nicer rendering, especially in REPL or markdown environments. + +# Arguments +- `msg::PT.AbstractMessage`: The message to be rendered. + +# Returns +- `String`: A markdown-formatted string representing the message. + +# Example +```julia +msg = PT.UserMessage("Hello, world!") +println(PT.preview(msg)) +``` + +This will output: +``` +# User Message +Hello, world! +``` +""" +function PT.preview(msg::PT.AbstractMessage) + role = if msg isa Union{PT.UserMessage, PT.UserMessageWithImages} + "User Message" + elseif msg isa PT.DataMessage + "Data Message" + elseif msg isa PT.SystemMessage + "System Message" + elseif msg isa PT.AIMessage + "AI Message" + else + "Unknown Message" + end + content = if msg isa PT.DataMessage + length_ = msg.content isa AbstractArray ? " (Size: $(size(msg.content)))" : "" + "Data: $(typeof(msg.content))$(length_)" + else + msg.content + end + return """# $role\n$(content)\n\n""" +end + +""" + preview(conversation::AbstractVector{<:PT.AbstractMessage}) + +Render a conversation, which is a vector of `AbstractMessage` objects, as a single markdown-formatted string. Each message is rendered individually and concatenated with separators for clear readability. + +This function is particularly useful for displaying the flow of a conversation in a structured and readable format. It leverages the `PT.preview` method for individual messages to create a cohesive view of the entire conversation. + +# Arguments +- `conversation::AbstractVector{<:PT.AbstractMessage}`: A vector of messages representing the conversation. + +# Returns +- `String`: A markdown-formatted string representing the entire conversation. + +# Example +```julia +conversation = [ + PT.SystemMessage("Welcome"), + PT.UserMessage("Hello"), + PT.AIMessage("Hi, how can I help you?") +] +println(PT.preview(conversation)) +``` + +This will output: +``` +# System Message +Welcome +--- +# User Message +Hello +--- +# AI Message +Hi, how can I help you? +--- +``` +""" +function PT.preview(conversation::AbstractVector{<:PT.AbstractMessage}) + io = IOBuffer() + print(io, join(PT.preview.(conversation), "---\n")) + String(take!(io)) |> Markdown.parse +end + +end # end of module \ No newline at end of file diff --git a/src/Experimental/AgentTools/AgentTools.jl b/src/Experimental/AgentTools/AgentTools.jl new file mode 100644 index 000000000..1ad44c6c2 --- /dev/null +++ b/src/Experimental/AgentTools/AgentTools.jl @@ -0,0 +1,22 @@ +""" + AgentTools + +Provides Agentic functionality providing lazy calls for building pipelines (eg, `AIGenerate`) and `AICodeFixer`. + +This module is experimental and may change at any time. It is intended to be moved to a separate package in the future. +""" +module AgentTools + +using PromptingTools +const PT = PromptingTools + +include("utils.jl") + +export aicodefixer_feedback +include("code_feedback.jl") + +export AICall, AIGenerate, AIExtract, AIEmbed, AIClassify, AIScan +export AICodeFixer, run! +include("lazy_types.jl") + +end \ No newline at end of file diff --git a/src/Experimental/AgentTools/code_feedback.jl b/src/Experimental/AgentTools/code_feedback.jl new file mode 100644 index 000000000..8fec67677 --- /dev/null +++ b/src/Experimental/AgentTools/code_feedback.jl @@ -0,0 +1,125 @@ +## Coding Feedback +abstract type AbstractCodeOutcome end +struct CodeEmpty <: AbstractCodeOutcome end +struct CodeFailedParse <: AbstractCodeOutcome end +struct CodeFailedEval <: AbstractCodeOutcome end +struct CodeFailedTimeout <: AbstractCodeOutcome end +struct CodeSuccess <: AbstractCodeOutcome end + +# Feedback function skeleton +""" + aicodefixer_feedback(conversation::AbstractVector{<:PT.AbstractMessage}; max_length::Int = 512) -> NamedTuple(; feedback::String) + +Generate feedback for an AI code fixing session based on the conversation history. +Function is designed to be extensible for different types of feedback and code evaluation outcomes. + +The highlevel wrapper accepts a conversation and returns new kwargs for the AICall. + +Individual feedback functions are dispatched on different subtypes of `AbstractCodeOutcome` and can be extended/overwritten to provide more detailed feedback. + +See also: `AIGenerate`, `AICodeFixer` + +# Arguments +- `conversation::AbstractVector{<:PT.AbstractMessage}`: A vector of messages representing the conversation history, where the last message is expected to contain the code to be analyzed. +- `max_length::Int=512`: An optional argument that specifies the maximum length of the feedback message. + +# Returns +- `NamedTuple`: A feedback message as a kwarg in NamedTuple based on the analysis of the code provided in the conversation. + +# Example +```julia +new_kwargs = aicodefixer_feedback(conversation) +``` + +# Notes +This function is part of the AI code fixing system, intended to interact with code in AIMessage and provide feedback on improving it. + +The highlevel wrapper accepts a conversation and returns new kwargs for the AICall. + +It dispatches for the code feedback based on the subtypes of `AbstractCodeOutcome` below: +- `CodeEmpty`: No code found in the message. +- `CodeFailedParse`: Code parsing error. +- `CodeFailedEval`: Runtime evaluation error. +- `CodeFailedTimeout`: Code execution timed out. +- `CodeSuccess`: Successful code execution. + +You can override the individual methods to customize the feedback. +""" +function aicodefixer_feedback(conversation::AbstractVector{<:PT.AbstractMessage}; + max_length::Int = 512) + @assert max_length>0 "max_length must be positive (provided: $max_length)" + # Extract the last message, evaluate code, determine outcome + cb = AICode(last(conversation); skip_unsafe = true, capture_stdout = true) + outcome = if isempty(cb.code) + CodeEmpty() # No code provided + elseif !PT.isparsed(cb) + CodeFailedParse() # Failed to parse + elseif !isnothing(cb.error) && isa(cb.error, InterruptException) + CodeFailedTimeout() # Failed to evaluate in time provided + elseif !isnothing(cb.error) + CodeFailedEval() # Failed to evaluate + else + CodeSuccess() # Success + end + # Return new kwargs or adjustments based on the outcome + new_kwargs = (; feedback = aicodefixer_feedback(outcome, cb; max_length)) + return new_kwargs +end + +function aicodefixer_feedback(::CodeEmpty, args...; kwargs...) + "**Error Detected**: No Julia code found. Always enclose Julia code in triple backticks code fence (\`\`\`julia\\n ... \\n\`\`\`)." +end +function aicodefixer_feedback(::CodeFailedTimeout, args...; kwargs...) + "**Error Detected**: Evaluation timed out. Please check your code for infinite loops or other issues." +end +function aicodefixer_feedback(::CodeSuccess, cb::AICode; + max_length::Int = 512, + kwargs...) + stdout_str = if isnothing(cb.stdout) || isempty(cb.stdout) + "" + else + temp = string(cb.stdout) + end_idx = min(length(temp), nextind(temp, 0, max_length)) + "\n\n**Output Captured:** $(temp[begin:end_idx])" + end + "Execution has been successful (no errors detected). Consider adding 1-2 challenging unit tests to improve the main function - use `@test` macro, organize them in `@testset begin .. end` block.$(stdout_str)" +end +function aicodefixer_feedback(::CodeFailedParse, + cb::AICode; + max_length::Int = 512, + kwargs...) + ## TODO: grab the parse error from expression? + ## Simple method + error_ = split(string(cb.error), "JuliaSyntax.SourceFile")[begin] + chunk_length = isnothing(cb.stdout) || isempty(cb.stdout) ? max_length : + max_length ÷ 2 + end_idx = min(length(error_), nextind(error_, 0, chunk_length)) + "**Parsing Error Detected:** $(error_[begin:end_idx])" +end + +function aicodefixer_feedback(::CodeFailedEval, + cb::AICode; + max_length::Int = 512, + kwargs...) + feedback = AbstractString[] + ## Grab the error message + error_str = if cb.error isa TaskFailedException + string(cb.error.task.result) + else + split(string(cb.error), "JuliaSyntax.SourceFile")[begin] + end + ## Decide how much space can be dedicated for this error (ie, do we have stdout as well?) + chunk_length = isnothing(cb.stdout) || isempty(cb.stdout) ? max_length : + max_length ÷ 2 + end_idx = min(length(error_str), nextind(error_str, 0, chunk_length)) + push!(feedback, "**Error Detected:** $(error_str[begin:end_idx])") + + if !isnothing(cb.stdout) && !isempty(string(cb.stdout)) + ## Add the optional STDOUT (for test failures) + chunk_length = max_length - sum(length, feedback) + end_idx = min(length(cb.stdout), nextind(cb.stdout, 0, chunk_length)) + push!(feedback, "**Output Captured:** $(cb.stdout[begin:end_idx])") + end + + return isempty(feedback) ? "No feedback provided." : join(feedback, "\n\n") +end \ No newline at end of file diff --git a/src/Experimental/AgentTools/lazy_types.jl b/src/Experimental/AgentTools/lazy_types.jl new file mode 100644 index 000000000..32f39074f --- /dev/null +++ b/src/Experimental/AgentTools/lazy_types.jl @@ -0,0 +1,407 @@ +abstract type AICallBlock end + +""" + AICall(func::F, args...; kwargs...) where {F<:Function} + + AIGenerate(args...; kwargs...) + AIEmbed(args...; kwargs...) + AIExtract(args...; kwargs...) + +A lazy call wrapper for AI functions in the `PromptingTools` module, such as `aigenerate`. + +The `AICall` struct is designed to facilitate a deferred execution model (lazy evaluation) for AI functions that interact with a Language Learning Model (LLM). It stores the necessary information for an AI call and executes the underlying AI function only when supplied with a `UserMessage` or when the `run!` method is applied. This approach allows for more flexible and efficient handling of AI function calls, especially in interactive environments. + +Seel also: `run!`, `AICodeFixer` + +# Fields +- `func::F`: The AI function to be called lazily. This should be a function like `aigenerate` or other `ai*` functions. +- `schema::Union{Nothing, PT.AbstractPromptSchema}`: Optional schema to structure the prompt for the AI function. +- `conversation::Vector{PT.AbstractMessage}`: A vector of messages that forms the conversation context for the AI call. +- `kwargs::NamedTuple`: Keyword arguments to be passed to the AI function. +- `success::Union{Nothing, Bool}`: Indicates whether the last call was successful (true) or not (false). `Nothing` if the call hasn't been made yet. +- `error::Union{Nothing, Exception}`: Stores any exception that occurred during the last call. `Nothing` if no error occurred or if the call hasn't been made yet. + +# Example + +Initiate an `AICall` like any ai* function, eg, `AIGenerate`: + +```julia +aicall = AICall(aigenerate) + +# With arguments and kwargs like ai* functions +# from `aigenerate(schema, conversation; model="abc", api_kwargs=(; temperature=0.1))` +# to +aicall = AICall(aigenerate, schema, conversation; model="abc", api_kwargs=(; temperature=0.1) + +# Or with a template +aicall = AIGenerate(:JuliaExpertAsk; ask="xyz", model="abc", api_kwargs=(; temperature=0.1)) +``` + +Trigger the AICall with `run!` (it returns the update `AICall` struct back): +```julia +aicall |> run! +```` + +You can also use `AICall` as a functor to trigger the AI call with a `UserMessage` or simply the text to send: +```julia +aicall(UserMessage("Hello, world!")) # Triggers the lazy call +result = run!(aicall) # Explicitly runs the AI call +``` +This can be used to "reply" to previous message / continue the stored conversation + +# Notes +- The `AICall` struct is a key component in building flexible and efficient Agentic pipelines +- The lazy evaluation model allows for setting up the call parameters in advance and deferring the actual execution until it is explicitly triggered. +- This struct is particularly useful in scenarios where the timing of AI function execution needs to be deferred or where multiple potential calls need to be prepared and selectively executed. +""" +@kwdef mutable struct AICall{F <: Function} <: AICallBlock + func::F + schema::Union{Nothing, PT.AbstractPromptSchema} = nothing + conversation::Vector{<:PT.AbstractMessage} = Vector{PT.AbstractMessage}() + kwargs::NamedTuple = NamedTuple() + success::Union{Nothing, Bool} = nothing + error::Union{Nothing, Exception} = nothing +end + +function AICall(func::F, args...; kwargs...) where {F <: Function} + @assert length(args)<=2 "AICall takes at most 2 positional arguments (provided: $(length(args)))" + schema = nothing + conversation = Vector{PT.AbstractMessage}() + for arg in args + if isa(arg, PT.AbstractPromptSchema) + schema = arg + elseif isa(arg, Vector{<:PT.AbstractMessage}) + conversation = arg + elseif isa(arg, AbstractString) && isempty(conversation) + ## User Prompt -- create a UserMessage + push!(conversation, PT.UserMessage(arg)) + elseif isa(arg, Symbol) && isempty(conversation) + conversation = PT.render(schema, AITemplate(arg)) + elseif isa(arg, AITemplate) && isempty(conversation) + conversation = PT.render(schema, arg) + else + error("Invalid argument type: $(typeof(arg))") + end + end + + return AICall{F}(; func, schema, conversation, kwargs = NamedTuple(kwargs)) +end + +""" + AIGenerate(args...; kwargs...) + +Creates a lazy instance of `aigenerate`. +It is an instance of `AICall` with `aigenerate` as the function. + +Use exactly the same arguments and keyword arguments as `aigenerate` (see `?aigenerate` for details). + +""" +function AIGenerate(args...; kwargs...) + return AICall(aigenerate, args...; kwargs...) +end + +""" + AIExtract(args...; kwargs...) + +Creates a lazy instance of `aiextract`. +It is an instance of `AICall` with `aiextract` as the function. + +Use exactly the same arguments and keyword arguments as `aiextract` (see `?aiextract` for details). + +""" +function AIExtract(args...; kwargs...) + return AICall(aiextract, args...; kwargs...) +end + +""" + AIEmbed(args...; kwargs...) + +Creates a lazy instance of `aiembed`. +It is an instance of `AICall` with `aiembed` as the function. + +Use exactly the same arguments and keyword arguments as `aiembed` (see `?aiembed` for details). + +""" +function AIEmbed(args...; kwargs...) + return AICall(aiembed, args...; kwargs...) +end + +""" + AIClassify(args...; kwargs...) + +Creates a lazy instance of `aiclassify`. +It is an instance of `AICall` with `aiclassify` as the function. + +Use exactly the same arguments and keyword arguments as `aiclassify` (see `?aiclassify` for details). + +""" +function AIClassify(args...; kwargs...) + return AICall(aiclassify, args...; kwargs...) +end + +""" + AIScan(args...; kwargs...) + +Creates a lazy instance of `aiscan`. +It is an instance of `AICall` with `aiscan` as the function. + +Use exactly the same arguments and keyword arguments as `aiscan` (see `?aiscan` for details). + +""" +function AIScan(args...; kwargs...) + return AICall(aiscan, args...; kwargs...) +end + +""" + run!(aicall::AICallBlock; verbose::Int = 1, catch_errors::Bool = false, return_all::Bool = true, kwargs...) + +Executes the AI call wrapped by an `AICallBlock` instance. This method triggers the actual communication with the AI model and processes the response based on the provided conversation context and parameters. + +# Arguments +- `aicall::AICallBlock`: An instance of `AICallBlock` which encapsulates the AI function call along with its context and parameters (eg, `AICall`, `AIGenerate`) +- `verbose::Int=1`: A verbosity level for logging. A higher value indicates more detailed logging. +- `catch_errors::Bool=false`: If set to `true`, the method will catch and handle errors internally. Otherwise, errors are propagated. +- `return_all::Bool=true`: A flag to indicate whether the whole conversation from the AI call should be returned. It should always be true. +- `kwargs...`: Additional keyword arguments that are passed to the AI function. + +# Returns +- `AICallBlock`: The same `AICallBlock` instance, updated with the results of the AI call. This includes updated conversation, success status, and potential error information. + +# Example +```julia +aicall = AICall(aigenerate) +run!(aicall) +``` + +Alternatively, you can trigger the `run!` call by using the AICall as a functor and calling it with a string or a UserMessage: +```julia +aicall = AICall(aigenerate) +aicall("Say hi!") +``` + +# Notes +- The `run!` method is a key component of the lazy evaluation model in `AICall`. It allows for the deferred execution of AI function calls, providing flexibility in how and when AI interactions are conducted. +- The method updates the `AICallBlock` instance with the outcome of the AI call, including any generated responses, success or failure status, and error information if an error occurred. +- This method is essential for scenarios where AI interactions are based on dynamic or evolving contexts, as it allows for real-time updates and responses based on the latest information. +""" +function run!(aicall::AICallBlock; + verbose::Int = 1, + catch_errors::Bool = false, + return_all::Bool = true, + kwargs...) + @assert return_all "`return_all` must be true (provided: $return_all)" + (; schema, conversation) = aicall + try + result = if isnothing(schema) + aicall.func(conversation; aicall.kwargs..., kwargs..., return_all) + else + aicall.func(schema, conversation; aicall.kwargs..., kwargs..., return_all) + end + # Remove used kwargs (for placeholders) + aicall.kwargs = remove_used_kwargs(aicall.kwargs, conversation) + aicall.conversation = result + aicall.success = true + catch e + verbose > 0 && @info "Error detected and caught in AICall" + aicall.success = false + aicall.error = e + !catch_errors && rethrow(aicall.error) + end + return aicall +end + +function Base.show(io::IO, aicall::AICallBlock; max_length::Int = 100) + print(io, + "$(typeof(aicall))(Messages: $(length(aicall.conversation)), Success: $(aicall.success))") + ## If AIMessage, show the first 100 characters of the content + if !isempty(aicall.conversation) && last(aicall.conversation) isa PT.AIMessage + str = last(aicall.conversation).content + print(io, + "\n- Preview of the Latest AIMessage (see property `:conversation`):\n $(first(str,max_length))") + end +end + +function (aicall::AICall)(str::AbstractString; kwargs...) + return aicall(PT.UserMessage(str); kwargs...) +end +function (aicall::AICall)(msg::PT.UserMessage; kwargs...) + push!(aicall.conversation, msg) + return run!(aicall; kwargs...) +end + +""" + AICodeFixer(aicall::AICall, templates::Vector{<:PT.UserMessage}; num_rounds::Int = 3, feedback_func::Function = aicodefixer_feedback; kwargs...) + AICodeFixer(aicall::AICall, template::Union{AITemplate, Symbol} = :CodeFixerRCI; kwargs...) + +An AIAgent that iteratively evaluates any received Julia code and provides feedback back to the AI model if `num_rounds>0`. +`AICodeFixer` manages the lifecycle of a code fixing session, including tracking conversation history, rounds of interaction, and applying user feedback through a specialized feedback function. + +It integrates with lazy AI call structures like `AIGenerate`. + +The operation is "lazy", ie, the agent is only executed when needed, eg, when `run!` is called. + +# Fields +- `call::AICall`: The AI call that is being used for code generation or processing, eg, AIGenerate (same as `aigenerate` but "lazy", ie, called only when needed +- `templates::Union{Symbol, AITemplate, Vector{PT.UserMessage}}`: A set of user messages or templates that guide the AI's code fixing process. + The first UserMessage is used in the first round of code fixing, the second UserMessage is used for every subsequent iteration. +- `num_rounds::Int`: The number of rounds for the code fixing session. Defaults to 3. +- `round_counter::Int`: Counter to track the current round of interaction. +- `feedback_func::Function`: Function to generate feedback based on the AI's proposed code, defaults to `aicodefixer_feedback` + (modular thanks to type dispatch on `AbstractOutcomes`) +- `kwargs::NamedTuple`: Additional keyword arguments for customizing the AI call. + +Note: Any kwargs provided to `run!()` will be passed to the underlying AICall. + +# Example + +Let's create an AIGenerate call and then pipe it to AICodeFixer to run a few rounds of the coding fixing: + +```julia +# Create an AIGenerate call +lazy_call = AIGenerate("Write a function to do XYZ...") + +# the action starts only when `run!` is called +result = lazy_call |> AICodeFixer |> run! + +# Access the result of the code fixing session +# result.call refers to the AIGenerate lazy call above +conversation = result.call.conversation +fixed_code = last(conversation) # usually in the last message + +# Preview the conversation history +preview(conversation) +``` + +You can change the template used to provide user feedback and number of counds via arguments: + +```julia +# Setup an AIGenerate call +lazy_call = AIGenerate(aigenerate, "Write code to do XYZ...") + +# Custom template and 2 fixing rounds +result = AICodeFixer(lazy_call, [PT.UserMessage("Please fix the code.\n\nFeedback: {{feedback}}")]; num_rounds = 2) |> run! + +# The result now contains the AI's attempts to fix the code +preview(result.call.conversation) +``` + +# Notes +- `AICodeFixer` is particularly useful when code is hard to get right in one shot (eg, smaller models, complex syntax) +- The structure leverages the lazy evaluation model of `AICall` (/AIGenerate) to efficiently manage AI interactions and be able to repeatedly call it. +- The `run!` function executes the AI call and applies the feedback loop for the specified number of rounds, enabling an interactive code fixing process. +""" +@kwdef mutable struct AICodeFixer + call::AICall + templates::Vector{PT.UserMessage} = Vector{PT.UserMessage}() + num_rounds::Int + round_counter::Int = 0 + feedback_func::Function + kwargs::NamedTuple = NamedTuple() + + function AICodeFixer(aicall::AICall, templates::Vector{<:PT.UserMessage}, + num_rounds::Int, + round_counter::Int, + feedback_func::Function, + kwargs::NamedTuple) + @assert num_rounds>=0 "`num_rounds` must be non-negative (provided: $num_rounds))" + @assert !isempty(templates) "Must provide a template / user message (provided: $(length(templates)))" + new(aicall, templates, num_rounds, round_counter, feedback_func, kwargs) + end +end +function AICodeFixer(aicall::AICall, templates::Vector{<:PT.UserMessage}; + round_counter::Int = 0, + num_rounds::Int = 3, + feedback_func::Function = aicodefixer_feedback, + kwargs...) + AICodeFixer(aicall, + templates, + num_rounds, + round_counter, + feedback_func, + NamedTuple(kwargs)) +end +# Dispatch on template/symbol +function AICodeFixer(aicall::AICall, + template::Union{AITemplate, Symbol} = :CodeFixerRCI; + kwargs...) + # Prepare template -- we expect two messages: the first is the intro, the second is the iteration steps + template_rendered = if template isa AITemplate + PT.render(aicall.schema, template) + else + @assert haskey(PT.TEMPLATE_STORE, template) "Template $(template) not found in TEMPLATE_STORE" + PT.render(aicall.schema, AITemplate(template)) + end + user_messages = filter(msg -> isa(msg, PT.UserMessage), template_rendered) + @assert length(user_messages)>0 "Template $(template) must have at least 1 user message (provided: $(length(user_messages)))" + AICodeFixer(aicall, convert(Vector{PT.UserMessage}, user_messages); kwargs...) +end + +function Base.show(io::IO, fixer::AICodeFixer) + print(io, + "$(typeof(fixer))(Rounds: $(fixer.round_counter)/$(fixer.num_rounds))") +end + +""" + run!(codefixer::AICodeFixer; verbose::Int = 1, max_conversation_length::Int = 32000, run_kwargs...) + +Executes the code fixing process encapsulated by the `AICodeFixer` instance. +This method iteratively refines and fixes code by running the AI call in a loop for a specified number of rounds, using feedback from the code evaluation (`aicodefixer_feedback`) to improve the outcome in each iteration. + +# Arguments +- `codefixer::AICodeFixer`: An instance of `AICodeFixer` containing the AI call, templates, and settings for the code fixing session. +- `verbose::Int=1`: Verbosity level for logging. A higher value indicates more detailed logging. +- `max_conversation_length::Int=32000`: Maximum length in characters for the conversation history to keep it within manageable limits, especially for large code fixing sessions. +- `num_rounds::Union{Nothing, Int}=nothing`: Number of additional rounds for the code fixing session. If `nothing`, the value from the `AICodeFixer` instance is used. +- `run_kwargs...`: Additional keyword arguments that are passed to the AI function. + +# Returns +- `AICodeFixer`: The updated `AICodeFixer` instance with the results of the code fixing session. + +# Usage +```julia +aicall = AICall(aigenerate, schema=mySchema, conversation=myConversation) +codefixer = AICodeFixer(aicall, myTemplates; num_rounds=5) +result = run!(codefixer, verbose=2) +``` + +# Notes +- The `run!` method drives the core logic of the `AICodeFixer`, iterating through rounds of AI interactions to refine and fix code. +- In each round, it applies feedback based on the current state of the conversation, allowing the AI to respond more effectively. +- The conversation history is managed to ensure it stays within the specified `max_conversation_length`, keeping the AI's focus on relevant parts of the conversation. +- This iterative process is essential for complex code fixing tasks where multiple interactions and refinements are required to achieve the desired outcome. +""" +function run!(codefixer::AICodeFixer; + verbose::Int = 1, + max_conversation_length::Int = 32000, + num_rounds::Union{Nothing, Int} = nothing, + run_kwargs...) + (; call, templates, round_counter, feedback_func) = codefixer + ## Select main num_rounds + num_rounds_ = !isnothing(num_rounds) ? (codefixer.round_counter + num_rounds) : + codefixer.num_rounds + + # Call the aicall for the first time + isnothing(call.success) && (run!(call; verbose, run_kwargs...)) + + # Early exit + num_rounds_ == 0 && return codefixer + + # Run the fixing loop `num_rounds` times + while round_counter < num_rounds_ + round_counter += 1 + verbose > 0 && @info "CodeFixing Round: $(round_counter)/$(num_rounds_)" + kwargs_new = feedback_func(call.conversation) # will update the feedback kwarg + call.kwargs = (; call.kwargs..., kwargs_new...) + # In the first round, add the intro message (first template) + msg = round_counter == 1 ? first(templates) : last(templates) + push!(call.conversation, msg) + call.conversation = truncate_conversation(call.conversation; + max_conversation_length) + ## Call LLM again for the fix + call = run!(call; verbose, run_kwargs...) + end + codefixer.round_counter = round_counter + codefixer.call = call + + return codefixer +end \ No newline at end of file diff --git a/src/Experimental/AgentTools/utils.jl b/src/Experimental/AgentTools/utils.jl new file mode 100644 index 000000000..babdcbafd --- /dev/null +++ b/src/Experimental/AgentTools/utils.jl @@ -0,0 +1,50 @@ +"Removes the kwargs that have already been used in the conversation. Returns NamedTuple." +function remove_used_kwargs(kwargs::NamedTuple, + conversation::AbstractVector{<:PT.AbstractMessage}) + used_kwargs = Set{Symbol}() + for message in conversation + if hasproperty(message, :variables) + union!(used_kwargs, message.variables) + end + end + return filter(pair -> !(pair.first in used_kwargs), pairs(kwargs)) |> NamedTuple +end + +""" + truncate_conversation(conversation::AbstractVector{<:PT.AbstractMessage}; + max_conversation_length::Int = 32000) + +Truncates a given conversation to a `max_conversation_length` characters by removing messages "in the middle". +It tries to retain the original system+user message and also the most recent messages. + +Practically, if a conversation is too long, it will start by removing the most recent message EXCEPT for the last two (assumed to be the last AIMessage with the code and UserMessage with the feedback + +# Arguments +`max_conversation_length` is in characters; assume c. 2-3 characters per LLM token, so 32000 should correspond to 16K context window. +""" +function truncate_conversation(conversation::AbstractVector{<:PT.AbstractMessage}; + max_conversation_length::Int = 32000) + @assert max_conversation_length>0 "max_conversation_length must be positive (provided: $max_conversation_length)" + total_length = sum(length.(getproperty.(conversation, :content)); init = 0) + # Truncate the conversation to the max length + new_conversation = if total_length > max_conversation_length && + length(conversation) > 2 + # start with the last two messages' length (always included) + new_conversation = similar(conversation) |> empty! + current_length = sum(length.(getproperty.(conversation[(end - 1):end], + :content)); init = 0) + for i in eachindex(conversation[begin:(end - 2)]) + length_ = length(conversation[i].content) + if current_length + length_ <= max_conversation_length + push!(new_conversation, conversation[i]) + current_length += length_ + end + end + # add the last two messages + append!(new_conversation, conversation[(end - 1):end]) + new_conversation + else + conversation + end + return new_conversation +end diff --git a/src/Experimental/Experimental.jl b/src/Experimental/Experimental.jl index 6a1f2cb20..eaa0d8938 100644 --- a/src/Experimental/Experimental.jl +++ b/src/Experimental/Experimental.jl @@ -6,10 +6,14 @@ It is not included in the main module, so it must be explicitly imported. Contains: - `RAGTools`: Retrieval-Augmented Generation (RAG) functionality. +- `AgentTools`: Agentic functionality - lazy calls for building pipelines (eg, `AIGenerate`) and `AICodeFixer`. """ module Experimental export RAGTools include("RAGTools/RAGTools.jl") +export AgentTools +include("AgentTools/AgentTools.jl") + end # module Experimental \ No newline at end of file diff --git a/src/code_generation.jl b/src/code_generation.jl index e8d8b1dc7..258f805a3 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -18,11 +18,11 @@ abstract type AbstractCodeBlock end """ AICode(code::AbstractString; auto_eval::Bool=true, safe_eval::Bool=false, skip_unsafe::Bool=false, capture_stdout::Bool=true, verbose::Bool=false, - prefix::AbstractString="", suffix::AbstractString="") + prefix::AbstractString="", suffix::AbstractString="", remove_tests::Bool=false, execution_timeout::Int = 60) AICode(msg::AIMessage; auto_eval::Bool=true, safe_eval::Bool=false, skip_unsafe::Bool=false, skip_invalid::Bool=false, capture_stdout::Bool=true, - verbose::Bool=false, prefix::AbstractString="", suffix::AbstractString="") + verbose::Bool=false, prefix::AbstractString="", suffix::AbstractString="", remove_tests::Bool=false, execution_timeout::Int = 60) A mutable structure representing a code block (received from the AI model) with automatic parsing, execution, and output/error capturing capabilities. @@ -60,6 +60,8 @@ See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` Useful to add some additional code definition or necessary imports. Defaults to an empty string. - `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. Useful to check that tests pass or that an example executes. Defaults to an empty string. +- `remove_tests::Bool`: If set to `true`, we remove any `@test` or `@testset` macros from the code block before parsing and evaluation. Defaults to `false`. +- `execution_timeout::Int`: The maximum time (in seconds) allowed for the code block to execute. Defaults to 60 seconds. # Methods - `Base.isvalid(cb::AICode)`: Check if the code block has executed successfully. Returns `true` if `cb.success == true`. @@ -117,10 +119,29 @@ function (CB::Type{T})(md::AbstractString; capture_stdout::Bool = true, verbose::Bool = false, prefix::AbstractString = "", - suffix::AbstractString = "") where {T <: AbstractCodeBlock} + suffix::AbstractString = "", + remove_tests::Bool = false, + execution_timeout::Int = 60) where {T <: AbstractCodeBlock} + ## + @assert execution_timeout>0 "execution_timeout must be positive" skip_unsafe && (md = remove_unsafe_lines(md; verbose)) cb = CB(; code = md) - auto_eval && eval!(cb; safe_eval, capture_stdout, prefix, suffix) + if auto_eval + # set to timeout in `execution_timeout` seconds + result = @timeout execution_timeout begin + eval!(cb; + safe_eval, + capture_stdout, + prefix, + suffix, + remove_tests) + end nothing # set to nothing if it fails + # Check if we timed out + if isnothing(result) + cb.success = false + cb.error = InterruptException() + end + end return cb end Base.isvalid(cb::AbstractCodeBlock) = cb.success == true @@ -208,6 +229,40 @@ function is_julia_expr(ex::Expr) return false end +# Remove any given macro expression from the expression tree, used to remove tests +function remove_macro_expr!(expr, sym::Symbol = Symbol("@testset")) + if expr isa Expr + expr.args = filter(x -> !(x isa Expr && x.head == :macrocall && !isempty(x.args) && + x.args[1] == sym), + expr.args) + foreach(x -> remove_macro_expr!(x, sym), expr.args) + end + expr +end + +# Remove testsets and sets from the expression tree +function remove_tests_from_expr!(expr) + # Focus only on the three most common test macros + remove_macro_expr!(expr, Symbol("@testset")) + remove_macro_expr!(expr, Symbol("@test")) + remove_macro_expr!(expr, Symbol("@test_throws")) +end + +# Utility to identify the module name in a given expression (to evaluate subsequent calls in it) +function extract_module_name(expr) + if isa(expr, Expr) && expr.head == :module + return expr.args[2] # The second argument is typically the module name + elseif isa(expr, Expr) && !isempty(expr.args) + output = extract_module_name.(expr.args) + for item in output + if !isnothing(item) + return item + end + end + end + nothing +end + ## Check if a given String seems to be a valid Julia expression (simple heuristics) function is_julia_code(code::AbstractString) # Try to parse the expression, return false if parsing fails @@ -633,14 +688,28 @@ function eval!(cb::AbstractCodeBlock; safe_eval::Bool = true, capture_stdout::Bool = true, prefix::AbstractString = "", - suffix::AbstractString = "") + suffix::AbstractString = "", + remove_tests::Bool = false) (; code) = cb # reset cb.success = nothing cb.error = nothing cb.expression = nothing cb.output = nothing - code_extra = string(prefix, "\n", code, "\n", suffix) + + code_extra = if safe_eval + safe_module = string(gensym("SafeCustomModule")) |> + x -> replace(x, "#" => "") + string("module $safe_module \nusing Test\n", + prefix, + "\n", + code, + "\n", + suffix, + "\nend") + else + string(prefix, "\n", code, "\n", suffix) + end ## Safety checks on `code` only -- treat it as a parsing failure if safe_eval detected, missing_packages = detect_missing_packages(extract_julia_imports(code)) @@ -669,26 +738,26 @@ function eval!(cb::AbstractCodeBlock; return cb end + ## Remove any tests + if remove_tests + ex = remove_tests_from_expr!(ex) + end + ## Eval - safe_module = gensym("SafeCustomModule") + eval!(cb, cb.expression; capture_stdout, eval_module = Main) + return cb +end + +# Evaluation of any arbitrary expression with result recorded in `cb` +function eval!(cb::AbstractCodeBlock, expr::Expr; + capture_stdout::Bool = true, + eval_module::Module = Main) # Prepare to catch any stdout if capture_stdout pipe = Pipe() redirect_stdout(pipe) do try - # eval in Main module to have access to std libs, but inside a custom module for safety - if safe_eval - cb.output = @eval(Main, module $safe_module - using Test # just in case unit tests are provided - $(cb.expression) - end) - else - # Evaluate the code directly into Main - cb.output = @eval(Main, begin - using Test # just in case unit tests are provided - $(cb.expression) - end) - end + cb.output = @eval(eval_module, $(expr)) cb.success = true catch e cb.error = e @@ -700,19 +769,7 @@ function eval!(cb::AbstractCodeBlock; else # Ignore stdout, just eval try - # eval in Main module to have access to std libs, but inside a custom module for safety - if safe_eval - cb.output = @eval(Main, module $safe_module - using Test # just in case unit tests are provided - $(cb.expression) - end) - else - # Evaluate the code directly into Main - cb.output = @eval(Main, begin - using Test # just in case unit tests are provided - $(cb.expression) - end) - end + cb.output = @eval(eval_module, $(expr)) cb.success = true catch e cb.error = e diff --git a/src/templates.jl b/src/templates.jl index 86ee9c3fc..bfcdd5abe 100644 --- a/src/templates.jl +++ b/src/templates.jl @@ -112,6 +112,11 @@ function render(template::AITemplate; kwargs...) global PROMPT_SCHEMA render(PROMPT_SCHEMA, template; kwargs...) end +# Since we don't distinguish between schema, support schema=nothing as well +function render(schema::Nothing, template::AITemplate; kwargs...) + global PROMPT_SCHEMA + render(PROMPT_SCHEMA, template; kwargs...) +end ## Loading/saving -- see src/serialization.jl diff --git a/src/utils.jl b/src/utils.jl index 37a89a28a..967a59c3c 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -325,4 +325,37 @@ function resize_conversation!(conv_history, popfirst!(conv_history) end return conv_history -end \ No newline at end of file +end + +""" + @timeout(seconds, expr_to_run, expr_when_fails) + +Simple macro to run an expression with a timeout of `seconds`. If the `expr_to_run` fails to finish in `seconds` seconds, `expr_when_fails` is returned. + +# Example +```julia +x = @timeout 1 begin + sleep(1.1) + println("done") + 1 +end "failed" + +``` +""" +macro timeout(seconds, expr_to_run, expr_when_fails) + quote + tsk = @task $(esc(expr_to_run)) + schedule(tsk) + Timer($(esc(seconds))) do timer + istaskdone(tsk) || Base.throwto(tsk, InterruptException()) + end + try + fetch(tsk) + catch _ + $(esc(expr_when_fails)) + end + end +end + +"Utility for rendering the conversation (vector of messages) as markdown. REQUIRES the Markdown package to load the extension!" +function preview end \ No newline at end of file diff --git a/templates/agents/code-fixing/CodeFixerRCI.json b/templates/agents/code-fixing/CodeFixerRCI.json new file mode 100644 index 000000000..7c158b68a --- /dev/null +++ b/templates/agents/code-fixing/CodeFixerRCI.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"This template is meant to be used with `AICodeFixer`. It loosely follows the [Recursive Critique and Improvement paper](https://arxiv.org/pdf/2303.17491.pdf) with two steps Critique and Improve based on `feedback`. Placeholders: `feedback`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"Ignore all previous instructions. \nYour goal is to satisfy the user's request by using several rounds of self-reflection (Critique step) and improvement of the previously provided solution (Improve step).\nAlways enclose Julia code in triple backticks code fence (```julia\\n ... \\n```).\n\n1. **Recall Past Critique:**\n- Summarize past critique to refresh your memory (use inline quotes to highlight the few characters of the code that caused the mistakes). It must not repeat.\n\n2. **Critique Step Instructions:** \n- Read the user request word-by-word. Does the code implementation follow the request to the the letter? Think it though step-by-step.\n- Review the provided feedback in detail.\n- Provide 2-3 bullet points of criticism for the code. Each bullet point must refer to a different type of error or issue.\n - If there are any errors, explain why and what needs to be changed to FIX THEM! Be specific. \n - If an error repeats or critique repeats, previous issue was not addressed. YOU MUST SUGGEST A DIFFERENT IMPROVEMENT THAN BEFORE.\n - If there are no errors, identify and list specific issues or areas for improvement to write more idiomatic Julia code.\n\n\n3. **Improve Step Instructions:** \n- Specify what you'll change to address the above critique.\n- Provide the revised code reflecting your suggested improvements. Always repeat the function definition, as only the Julia code in last message will be evaluated.\n- Ensure the new version of the code resolves the problems while fulfilling the original task. Ensure it has the same function name.\n- Write 2-3 correct and helpful unit tests for the function requested by the user (organize in `@testset \"name\" begin ... end` block, use `@test` macro).\n\n\n3. **Response Format:**\n---\n### Past Critique\n\n\n### Critique\n\n\n### Improve\n\n\n```julia\n\n```\n---\n\nBe concise and focused in all steps.\n\n### Feedback from the User\n\n{{feedback}}\n\nI believe in you. You can actually do it, so do it ffs. Avoid shortcuts or placing comments instead of code. I also need code, actual working Julia code.\nWhat are your Critique and Improve steps?\n ","variables":["feedback"],"_type":"usermessage"},{"content":"### Feedback from the User\n\n{{feedback}}\n\nBased on your past critique and the latest feedback, what are your Critique and Improve steps?\n","variables":["feedback"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/agents/code-fixing/CodeFixerShort.json b/templates/agents/code-fixing/CodeFixerShort.json new file mode 100644 index 000000000..4cbea55e2 --- /dev/null +++ b/templates/agents/code-fixing/CodeFixerShort.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"This template is meant to be used with `AICodeFixer` to ask for code improvements based on `feedback`. It uses the same message for both the introduction of the new task and for the iterations. Placeholders: `feedback`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"\nThe above Julia code has been executed with the following results:\n\n```plaintext\n{{feedback}}\n```\n\n0. Read the user request word-by-word. Does the code implementation follow the request to the the letter? Think it though step-by-step.\n1. Review the execution results in detail and, if there is an error, explain why it happened.\n2. Suggest improvements to the code. Be EXTREMELY SPECIFIC. Think step-by-step and break it down.\n3. Write an improved implemented based on your reflection.\n\nAll code must be enclosed in triple backticks code fence (```julia\\n ... \\n```) and included in one message to be re-evaluated.\n\nI believe in you. Take a deep breath. You can actually do it, so do it ffs. Avoid shortcuts or placing comments instead of code. I also need code, actual working Julia code.\n","variables":["feedback"],"_type":"usermessage"}] \ No newline at end of file diff --git a/templates/agents/code-fixing/CodeFixerTiny.json b/templates/agents/code-fixing/CodeFixerTiny.json new file mode 100644 index 000000000..73ed159b0 --- /dev/null +++ b/templates/agents/code-fixing/CodeFixerTiny.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"This tiniest template to use with `AICodeFixer`. Iteratively asks to improve the code based on provided `feedback`. Placeholders: `feedback`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"### Execution Results\n\n```plaintext\n{{feedback}}\n```\n\nTake a deep break. Think step-by-step and fix the above errors. I believe in you. You can do it! I also need code, actual working Julia code, no shortcuts.\n","variables":["feedback"],"_type":"usermessage"}] \ No newline at end of file diff --git a/test/Experimental/AgentTools/code_feedback.jl b/test/Experimental/AgentTools/code_feedback.jl new file mode 100644 index 000000000..c2d77000b --- /dev/null +++ b/test/Experimental/AgentTools/code_feedback.jl @@ -0,0 +1,69 @@ +using PromptingTools.Experimental.AgentTools: aicodefixer_feedback +using PromptingTools.Experimental.AgentTools: CodeEmpty, + CodeFailedParse, CodeFailedEval, CodeFailedTimeout, CodeSuccess + +@testset "aicodefixer_feedback" begin + # Empty code + conv = [PT.AIMessage("test")] + feedback = aicodefixer_feedback(conv).feedback + code_missing_err = "**Error Detected**: No Julia code found. Always enclose Julia code in triple backticks code fence (```julia\\n ... \\n```)." + @test feedback == code_missing_err + @test aicodefixer_feedback(CodeEmpty()) == code_missing_err + + # CodeFailedParse + cb = AICode("println(\"a\"") + feedback = aicodefixer_feedback(CodeFailedParse(), cb) + @test occursin("**Parsing Error Detected:**", feedback) + conv = [PT.AIMessage(""" + ```julia + println(\"a\" + ``` + """)] + feedback = aicodefixer_feedback(conv).feedback + @test occursin("**Parsing Error Detected:**", feedback) + + # CodeFailedEval -- for failed tasks and normal errors + cb = AICode(""" + tsk=@task error("xx") + schedule(tsk) + fetch(tsk) + """) + cb.stdout = "STDOUT" + feedback = aicodefixer_feedback(CodeFailedEval(), cb) + @test feedback == + "**Error Detected:** ErrorException(\"xx\")\n\n**Output Captured:** STDOUT" + cb = AICode("error(\"xx\")") + cb.stdout = "STDOUT" + feedback = aicodefixer_feedback(CodeFailedEval(), cb) + @test feedback == + "**Error Detected:** ErrorException(\"xx\")\n\n**Output Captured:** STDOUT" + conv = [PT.AIMessage(""" + ```julia + error(\"xx\") + ``` + """)] + feedback = aicodefixer_feedback(conv).feedback + @test feedback == + "**Error Detected:** ErrorException(\"xx\")" + + # CodeFailedTimeout + cb = AICode("InterruptException()") + feedback = aicodefixer_feedback(CodeFailedTimeout(), cb) + @test feedback == + "**Error Detected**: Evaluation timed out. Please check your code for infinite loops or other issues." + conv = [PT.AIMessage(""" + ```julia + throw(InterruptException()) + ``` + """)] + feedback = aicodefixer_feedback(conv).feedback + @test feedback == + "**Error Detected**: Evaluation timed out. Please check your code for infinite loops or other issues." + + # CodeSuccess + cb = AICode("1") + cb.stdout = "STDOUT" + feedback = aicodefixer_feedback(CodeSuccess(), cb) + @test occursin("Execution has been successful", feedback) + @test occursin("**Output Captured:**", feedback) +end \ No newline at end of file diff --git a/test/Experimental/AgentTools/lazy_types.jl b/test/Experimental/AgentTools/lazy_types.jl new file mode 100644 index 000000000..76b3c4ef1 --- /dev/null +++ b/test/Experimental/AgentTools/lazy_types.jl @@ -0,0 +1,165 @@ +@testset "AICall" begin + # Create AICall with default parameters + default_call = AICall(identity) + @test default_call.func === identity + @test isnothing(default_call.schema) + @test isempty(default_call.conversation) + @test isempty(default_call.kwargs) + @test isnothing(default_call.success) + @test isnothing(default_call.error) + + # Custom function + custom_func = x -> x * 2 + custom_call = AICall(custom_func) + @test custom_call.func === custom_func + + # Different conversation types + aicall = AICall(identity, [PT.UserMessage("Hi")]) + @test aicall.conversation == [PT.UserMessage("Hi")] + aicall = AICall(identity, "Hi") + @test aicall.conversation == [PT.UserMessage("Hi")] + aicall = AICall(identity, :BlankSystemUser) + @test aicall.conversation == [PT.SystemMessage("{{system}}") + PT.UserMessage("{{user}}")] + aicall = AICall(identity, AITemplate(:BlankSystemUser)) + @test aicall.conversation == [PT.SystemMessage("{{system}}") + PT.UserMessage("{{user}}")] + + # derived methods + aicall = AIGenerate() + @test aicall.func == aigenerate + aicall = AIExtract() + @test aicall.func == aiextract + aicall = AIEmbed() + @test aicall.func == aiembed + aicall = AIScan() + @test aicall.func == aiscan + aicall = AIClassify() + @test aicall.func == aiclassify + + # Wrong arguments + @test_throws AssertionError AICall(identity, "arg1", "arg2", "arg3") + + # run! method + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema = PT.TestEchoOpenAISchema(; response, status = 200) + aicall = AICall(aigenerate, schema) + run!(aicall) + @test isa(aicall, AICall) + @test aicall.conversation[end].content == "Hello!" + + # catch_error + pass_func(args...; kwargs...) = nothing + @test_throws Exception run!(AICall(pass_func, PT.PROMPT_SCHEMA)) + + # Functor with String + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema = PT.TestEchoOpenAISchema(; response, status = 200) + aicall = AICall(aigenerate, schema) + result = aicall("test string") + @test isa(result, AICall) + @test result.conversation[2] == PT.UserMessage("test string") + @test result.conversation[3].content == "Hello!" + + # Functor with UserMessage + user_msg = PT.UserMessage("test message") + result = aicall(user_msg) + @test isa(result, AICall) + @test result.conversation[end].content == "Hello!" + @test result.conversation[end - 1] == PT.UserMessage("test message") + # Invalid Argument Type + @test_throws ErrorException AICall(identity, 123) + + # Show method + aicall = AICall(identity, schema) + aicall.conversation = [PT.UserMessage("Hi!"), PT.AIMessage("Test message")] + aicall.success = true + io = IOBuffer() + show(io, aicall) + output = String(take!(io)) + @test output == + "AICall{typeof(identity)}(Messages: 2, Success: true)\n- Preview of the Latest AIMessage (see property `:conversation`):\n Test message" +end + +@testset "AICodeFixer" begin + # default constructor + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema = PT.TestEchoOpenAISchema(; response, status = 200) + aicall = AICall(aigenerate, schema) + codefixer = AICodeFixer(aicall, [PT.UserMessage("Test")]) + @test codefixer.call === aicall + @test length(codefixer.templates) == 1 + @test codefixer.num_rounds == 3 # Default value + @test codefixer.round_counter == 0 + @test codefixer.feedback_func === aicodefixer_feedback + @test isempty(codefixer.kwargs) + + # Custom Constructor + custom_func = x -> x * 2 + aicall = AICall(custom_func) + custom_template = [PT.UserMessage("Custom Test")] + custom_rounds = 5 + codefixer = AICodeFixer(aicall, custom_template; num_rounds = custom_rounds) + @test codefixer.call.func == custom_func + @test codefixer.templates == custom_template + @test codefixer.num_rounds == custom_rounds + + # run! Method + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema = PT.TestEchoOpenAISchema(; response, status = 200) + aicall = AICall(aigenerate, schema) + codefixer = AICodeFixer(aicall, [PT.UserMessage("Test")]) + run!(codefixer) + @test codefixer.round_counter == codefixer.num_rounds + @test codefixer.call.success == true + @test codefixer.call.conversation[end - 1] == PT.UserMessage("Test") + @test codefixer.call.conversation[end].content == PT.AIMessage("Hello!").content + + ## Run for a few more iterations + run!(codefixer; num_rounds = 2) + @test codefixer.round_counter == codefixer.num_rounds + 2 + + # symbol template + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema = PT.TestEchoOpenAISchema(; response, status = 200) + aicall = AICall(aigenerate, schema) + codefixer = AICodeFixer(aicall, :CodeFixerShort) + run!(codefixer) + @test codefixer.round_counter == codefixer.num_rounds + @test codefixer.call.success == true + @test codefixer.call.conversation[end].content == PT.AIMessage("Hello!").content + # AITemplate template + codefixer = AICodeFixer(aicall, AITemplate(:CodeFixerShort)) + @test length(codefixer.templates) == 1 + + # Zero Rounds + response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema = PT.TestEchoOpenAISchema(; response, status = 200) + aicall = AICall(aigenerate, schema) + codefixer = AICodeFixer(aicall, [PT.UserMessage("Test")]; num_rounds = 0) + run!(codefixer) + @test codefixer.round_counter == 0 + @test codefixer.num_rounds == 0 + ## No Test + @test codefixer.call.conversation[end].content == PT.AIMessage("Hello!").content + + # Invalid Template + aicall = AICall(identity) + @test_throws AssertionError AICodeFixer(aicall, :InvalidSymbol; num_rounds = 0) + + # Show method + aicall = AICall(identity) + codefixer = AICodeFixer(aicall, [PT.UserMessage("Fix this")], num_rounds = 5) + + # Capture the output of show + io = IOBuffer() + show(io, codefixer) + output = String(take!(io)) + @test output == "AICodeFixer(Rounds: 0/5)" +end \ No newline at end of file diff --git a/test/Experimental/AgentTools/runtests.jl b/test/Experimental/AgentTools/runtests.jl new file mode 100644 index 000000000..55eef4bdc --- /dev/null +++ b/test/Experimental/AgentTools/runtests.jl @@ -0,0 +1,10 @@ +using Test +using PromptingTools +using PromptingTools.Experimental.AgentTools +const PT = PromptingTools + +@testset "AgentTools" begin + include("utils.jl") + include("code_feedback.jl") + include("lazy_types.jl") +end diff --git a/test/Experimental/AgentTools/utils.jl b/test/Experimental/AgentTools/utils.jl new file mode 100644 index 000000000..b488643cb --- /dev/null +++ b/test/Experimental/AgentTools/utils.jl @@ -0,0 +1,58 @@ +using PromptingTools.Experimental.AgentTools: remove_used_kwargs, truncate_conversation + +@testset "remove_used_kwargs" begin + # Test 1: No overlapping keys + @test remove_used_kwargs((a = 1, b = 2), [PT.UserMessage("{{c}} {{d}}")]) == + (a = 1, b = 2) + + # Test 2: All keys used + @test remove_used_kwargs((a = 1, b = 2), [PT.UserMessage("{{a}} {{b}}")]) == + NamedTuple() + + # Test 3: Some overlapping keys + @test remove_used_kwargs((a = 1, b = 2, c = 3), + [PT.UserMessage("{{a}} {{d}}"), PT.UserMessage("{{b}}")]) == (; c = 3) + + # Test 4: Empty conversation + @test remove_used_kwargs((a = 1, b = 2), PT.AbstractMessage[]) == (a = 1, b = 2) + + # Test 5: Empty kwargs + @test remove_used_kwargs(NamedTuple(), [PT.UserMessage("{{c}} {{d}}")]) == NamedTuple() +end + +@testset "truncate_conversation" begin + conversation = [ + PT.SystemMessage("Hello"), + PT.UserMessage("World"), + PT.AIMessage("Hello"), + PT.UserMessage("World"), + PT.AIMessage("Hello"), + PT.UserMessage("World"), + PT.AIMessage("Hello"), + PT.UserMessage("World"), + ] + #### Test 1: Short Conversation + truncated = truncate_conversation(conversation, max_conversation_length = 32000) + @test length(truncated) == length(conversation) + @test truncated === conversation + + #### Test 2: Exactly Max Length Conversation + truncated = truncate_conversation(conversation, + max_conversation_length = 15) + @test sum(x -> length(x.content), truncated) <= 15 + + #### Test 3: Exactly Two Messages + truncated = truncate_conversation(conversation, max_conversation_length = 1) + @test length(truncated) == 2 + @test truncated == conversation[(end - 1):end] + + ### Test 4: Keep System Image and User Image + truncated = truncate_conversation(conversation, max_conversation_length = 20) + @test length(truncated) == 4 + @test truncated == vcat(conversation[begin:(begin + 1)], conversation[(end - 1):end]) + + #### Test 5: No Messages + conversation = PT.AbstractMessage[] + truncated = truncate_conversation(conversation, max_conversation_length = 32000) + @test isempty(truncated) +end \ No newline at end of file diff --git a/test/code_generation.jl b/test/code_generation.jl index 7c7c8839f..2819fad25 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -5,6 +5,7 @@ using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, extract_code_blocks_fallback, eval! using PromptingTools: escape_interpolation, find_subsequence_positions using PromptingTools: AICode, isparsed, isparseerror, is_julia_code, is_julia_expr +using PromptingTools: remove_tests_from_expr!, remove_macro_expr!, extract_module_name @testset "is_julia_expr" begin # Valid Julia Expressions @@ -35,6 +36,80 @@ using PromptingTools: AICode, isparsed, isparseerror, is_julia_code, is_julia_ex @test is_julia_expr([1, 2, 3]) == false end +@testset "remove_macro_expr!" begin + # Test with @testset macro + expr = Meta.parseall(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + end + y = 3 + 3 + """) + expected = Meta.parseall("y = 3 + 3") + result = remove_macro_expr!(expr) + @test result.args[end] == expected.args[end] + + # Test with nested @testset + expr = Meta.parseall(""" + @testset "Outer Test" begin + @testset "Inner Test" begin + x = 1 + 1 + end + y = 2 + 2 + end + """) + expected = Meta.parseall("") # All expressions are removed + result = remove_macro_expr!(expr) + # 1.9 parser eats the empty row, 1.10 retains it + @test length(result.args) == 1 || result == expected + + # Test without @testset + expr = Meta.parseall("z = 4 + 4") + expected = Meta.parseall("z = 4 + 4") + result = remove_macro_expr!(expr) + @test result == expected + + # Test with different macro + expr = Meta.parseall("@chain x begin; end") + expected = Meta.parseall("@chain x begin; end") + result = remove_macro_expr!(expr, Symbol("@test")) + @test result == expected +end + +@testset "remove_tests_from_expr!" begin + # Test with both @testset and @test macros + expr = Meta.parseall(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + end + @test x == 2 + @test_throws AssertionError func(1) + y = 3 + 3 + """) + expected = Meta.parseall("y = 3 + 3") + result = remove_tests_from_expr!(expr) + @test result.args[end] == expected.args[end] +end + +@testset "extract_module_name" begin + # Test with a valid module expression + module_expr = Meta.parse("module MyTestModule\nend") + @test extract_module_name(module_expr) == :MyTestModule + + # Test with an expression that is not a module + non_module_expr = Meta.parse("x = 1 + 1") + @test extract_module_name(non_module_expr) === nothing + + # In a nested expression tree + module_expr = Meta.parseall("module MyTestModule\nfoo()=\"hello\"\nend") + @test extract_module_name(module_expr) == :MyTestModule + + # Test with an empty expression + empty_expr = Meta.parse("") + @test extract_module_name(empty_expr) === nothing +end + @testset "is_julia_code" begin # Valid Julia Code @@ -452,6 +527,28 @@ end @test cb.stdout == "Hello\n" @test cb.code == "println(\"Hello\")" @test isvalid(cb) + + # Test execution_timeout + cb = AICode("sleep(1.1)", execution_timeout = 1) + @test cb.success == false + @test isnothing(cb.output) + @test cb.error isa InterruptException + cb = AICode("sleep(1.1)", execution_timeout = 2) + @test cb.success == true + @test isnothing(cb.error) + + # expression-only method + cb = AICode(""" + module MyModule + function foo() + println("Hello") + end + end + """; safe_eval = false) + eval_module = getfield(Main, extract_module_name(cb.expression)) + eval!(cb, Meta.parseall("foo()"); eval_module) + @test isnothing(cb.error) + @test cb.stdout == "Hello\n" end @testset "AICode constructors" begin diff --git a/test/runtests.jl b/test/runtests.jl index f063201a5..d598defd3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,6 @@ using PromptingTools using OpenAI, HTTP, JSON3 -using SparseArrays, LinearAlgebra +using SparseArrays, LinearAlgebra, Markdown using Test using Aqua const PT = PromptingTools @@ -41,4 +41,5 @@ end ## Run experimental @testset "Experimental" begin include("Experimental/RAGTools/runtests.jl") + include("Experimental/AgentTools/runtests.jl") end diff --git a/test/templates.jl b/test/templates.jl index ac71a9807..4239215f3 100644 --- a/test/templates.jl +++ b/test/templates.jl @@ -9,6 +9,7 @@ using PromptingTools: TestEchoOpenAISchema UserMessage("# Statement\n\n{{it}}")] @test expected_output == render(PT.PROMPT_SCHEMA, template) @test expected_output == render(template) + @test expected_output == render(nothing, template) end @testset "Templates - search" begin diff --git a/test/utils.jl b/test/utils.jl index 8046016bc..4c1865c8d 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -2,7 +2,7 @@ using PromptingTools: split_by_length, replace_words using PromptingTools: _extract_handlebar_variables, call_cost, _report_stats using PromptingTools: _string_to_vector, _encode_local_image using PromptingTools: DataMessage, AIMessage -using PromptingTools: push_conversation!, resize_conversation! +using PromptingTools: push_conversation!, resize_conversation!, @timeout, preview @testset "replace_words" begin words = ["Disney", "Snow White", "Mickey Mouse"] @@ -182,4 +182,37 @@ end conv_history = [[AIMessage("Test message")] for i in 1:7] resize_conversation!(conv_history, nothing) @test length(conv_history) == 7 -end \ No newline at end of file +end + +@testset "@timeout" begin + #### Test 1: Successful Execution Within Timeout + result = @timeout 2 begin + sleep(1) + "success" + end "timeout" + @test result == "success" + + #### Test 2: Execution Exceeds Timeout + result = @timeout 1 begin + sleep(2) + "success" + end "timeout" + @test result == "timeout" + + #### Test 4: Negative Timeout + @test_throws ArgumentError @timeout -1 begin + "success" + end "timeout" +end + +@testset "preview" begin + conversation = [ + PT.SystemMessage("Welcome"), + PT.UserMessage("Hello"), + PT.AIMessage("World"), + PT.DataMessage(; content = ones(10)), + ] + preview_output = preview(conversation) + expected_output = Markdown.parse("# System Message\n\nWelcome\n\n---\n\n# User Message\n\nHello\n\n---\n\n# AI Message\n\nWorld\n\n---\n\n# Data Message\n\nData: Vector{Float64} (Size: (10,))\n") + @test preview_output == expected_output +end From 9c737d2a215479fa602f715b809a106e6a3f75f2 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 4 Jan 2024 07:58:01 +0000 Subject: [PATCH 093/251] Add ItemsExtract (#45) --- CHANGELOG.md | 1 + src/extraction.jl | 7 +++++++ src/llm_openai.jl | 12 +++++++++++- test/extraction.jl | 28 +++++++++++++++++++++++++--- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fc2425a..a8a3a43e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the first AI Agent: `AICodeFixer` which iteratively analyzes and improves any code provided by a LLM by evaluating it in a sandbox. It allows a lot of customization (templated responses, feedback function, etc.) See `?AICodeFixer` for more information on usage and `?aicodefixer_feedback` for the example implementation of the feedback function. - Added `@timeout` macro to allow for limiting the execution time of a block of code in `AICode` via `execution_timeout` kwarg (prevents infinite loops, etc.). See `?AICode` for more information. - Added `preview(conversation)` utility that allows you to quickly preview the conversation in a Markdown format in your REPL. Requires `Markdown` package for the extension to be loaded. +- Added `ItemsExtract` convenience wrapper for `aiextract` when you want to extract one or more of a specific `return_type` (eg, `return_type = ItemsExtract{MyMeasurement}`) ### Fixed diff --git a/src/extraction.jl b/src/extraction.jl index 206ff711d..1c45acf9a 100644 --- a/src/extraction.jl +++ b/src/extraction.jl @@ -180,4 +180,11 @@ struct MaybeExtract{T <: Any} result::Union{Nothing, T} error::Bool message::Union{Nothing, String} +end + +""" +Extract zero, one or more specified items from the provided data. +""" +struct ItemsExtract{T <: Any} + items::Vector{T} end \ No newline at end of file diff --git a/src/llm_openai.jl b/src/llm_openai.jl index f9aaa9538..c35b533b7 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -503,7 +503,7 @@ If `return_all=true`: - `conversation`: A vector of `AbstractMessage` objects representing the full conversation history, including the response from the AI model (`DataMessage`). -See also: `function_call_signature`, `MaybeExtract`, `aigenerate` +See also: `function_call_signature`, `MaybeExtract`, `ItemsExtract`, `aigenerate` # Example @@ -543,6 +543,16 @@ msg.content.measurements # MyMeasurement(19, 190, nothing) ``` +Or you can use the convenience wrapper `ItemsExtract` to extract multiple measurements (zero, one or more): +```julia +using PromptingTools: ItemsExtract + +return_type = ItemsExtract{MyMeasurement} +msg = aiextract("James is 30, weighs 80kg. He's 180cm tall. Then Jack is 19 but really tall - over 190!"; return_type) + +msg.content.items # see the extracted items +``` + Or if you want your extraction to fail gracefully when data isn't found, use `MaybeExtract{T}` wrapper (this trick is inspired by the Instructor package!): ``` diff --git a/test/extraction.jl b/test/extraction.jl index dc65bd693..a75c4fefa 100644 --- a/test/extraction.jl +++ b/test/extraction.jl @@ -1,4 +1,4 @@ -using PromptingTools: MaybeExtract, extract_docstring +using PromptingTools: MaybeExtract, extract_docstring, ItemsExtract using PromptingTools: has_null_type, is_required_field, remove_null_types, to_json_schema using PromptingTools: function_call_signature @@ -169,7 +169,7 @@ end @test_broken haskey(schema, "description") end -@testset "to_json_schema" begin +@testset "to_json_schema-MaybeExtract" begin "Represents person's age, height, and weight" struct MyMeasurement1 age::Int @@ -193,7 +193,29 @@ end @test schema_measurement["description"] == "Represents person's age, height, and weight\n" end - +@testset "to_json_schema-ItemsExtract" begin + "Represents person's age, height, and weight" + struct MyMeasurement1 + age::Int + height::Union{Int, Nothing} + weight::Union{Nothing, Float64} + end + schema = to_json_schema(ItemsExtract{MyMeasurement1}) + @test schema["type"] == "object" + @test schema["properties"]["items"]["type"] == "array" + @test schema["required"] == ["items"] + @test haskey(schema, "description") + ## Check that the nested struct is extracted correctly + schema_measurement = schema["properties"]["items"]["items"] + @test schema_measurement["type"] == "object" + @test schema_measurement["properties"]["age"]["type"] == "integer" + @test schema_measurement["properties"]["height"]["type"] == "integer" + @test schema_measurement["properties"]["weight"]["type"] == "number" + @test schema_measurement["required"] == ["age"] + ## Check that the nested docstring is extracted correctly + @test schema_measurement["description"] == + "Represents person's age, height, and weight\n" +end @testset "function_call_signature" begin "Some docstring" struct MyMeasurement2 From 2ba5b25b651da12b62414989e82899b8119d5a6c Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 7 Jan 2024 12:06:28 +0000 Subject: [PATCH 094/251] Fix aiembed args (#46) --- CHANGELOG.md | 1 + src/llm_ollama_managed.jl | 6 +++--- src/llm_openai.jl | 6 +++--- test/llm_openai.jl | 10 ++++++++++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8a3a43e4..65314b67a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `ItemsExtract` convenience wrapper for `aiextract` when you want to extract one or more of a specific `return_type` (eg, `return_type = ItemsExtract{MyMeasurement}`) ### Fixed +- Fixed `aiembed` to accept any AbstractVector of documents (eg, a view of a vector of documents) ## [0.6.0] diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index e6d12df80..23fa175f5 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -238,7 +238,7 @@ end """ aiembed(prompt_schema::AbstractOllamaManagedSchema, - doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, + doc_or_docs::Union{AbstractString, AbstractVector{<:AbstractString}}, postprocess::F = identity; verbose::Bool = true, api_key::String = "", @@ -253,7 +253,7 @@ The `aiembed` function generates embeddings for the given input using a specifie ## Arguments - `prompt_schema::AbstractOllamaManagedSchema`: The schema for the prompt. -- `doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}`: The document or list of documents to generate embeddings for. The list of documents is processed sequentially, +- `doc_or_docs::Union{AbstractString, AbstractVector{<:AbstractString}}`: The document or list of documents to generate embeddings for. The list of documents is processed sequentially, so users should consider implementing an async version with with `Threads.@spawn` - `postprocess::F`: The post-processing function to apply to each embedding. Defaults to the identity function, but could be `LinearAlgebra.normalize`. - `verbose::Bool`: A flag indicating whether to print verbose information. Defaults to `true`. @@ -334,7 +334,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, return msg end function aiembed(prompt_schema::AbstractOllamaManagedSchema, - docs::Vector{<:AbstractString}, + docs::AbstractVector{<:AbstractString}, postprocess::F = identity; verbose::Bool = true, api_key::String = "", model::String = MODEL_EMBEDDING, diff --git a/src/llm_openai.jl b/src/llm_openai.jl index c35b533b7..20696d5c3 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -318,7 +318,7 @@ end """ aiembed(prompt_schema::AbstractOpenAISchema, - doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, + doc_or_docs::Union{AbstractString, AbstractVector{<:AbstractString}}, postprocess::F = identity; verbose::Bool = true, api_key::String = OPENAI_API_KEY, @@ -333,7 +333,7 @@ The `aiembed` function generates embeddings for the given input using a specifie ## Arguments - `prompt_schema::AbstractOpenAISchema`: The schema for the prompt. -- `doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}`: The document or list of documents to generate embeddings for. +- `doc_or_docs::Union{AbstractString, AbstractVector{<:AbstractString}}`: The document or list of documents to generate embeddings for. - `postprocess::F`: The post-processing function to apply to each embedding. Defaults to the identity function. - `verbose::Bool`: A flag indicating whether to print verbose information. Defaults to `true`. - `api_key::String`: The API key to use for the OpenAI API. Defaults to `OPENAI_API_KEY`. @@ -370,7 +370,7 @@ msg.content' * msg.content[:, 1] # [1.0, 0.787] """ function aiembed(prompt_schema::AbstractOpenAISchema, - doc_or_docs::Union{AbstractString, Vector{<:AbstractString}}, + doc_or_docs::Union{AbstractString, AbstractVector{<:AbstractString}}, postprocess::F = identity; verbose::Bool = true, api_key::String = OPENAI_API_KEY, model::String = MODEL_EMBEDDING, diff --git a/test/llm_openai.jl b/test/llm_openai.jl index cc45494d0..a6d1ec870 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -300,4 +300,14 @@ end @test msg == expected_output @test schema2.inputs == ["Hello World", "Hello back"] @test schema2.model_id == "gpt-4" # not possible - just an example + msg = aiembed(schema2, view(["Hello World", "Hello back"], :), + model = "gpt4", http_kwargs = (; verbose = 3), api_kwargs = (; temperature = 0)) + expected_output = DataMessage(; + content = ones(128, 2), + status = 200, + tokens = (4, 0), + elapsed = msg.elapsed) + @test msg == expected_output + @test schema2.inputs == ["Hello World", "Hello back"] + @test schema2.model_id == "gpt-4" # not possible - just an example end From 8ba40958859dbcbf4ae8c45e90863c1a67f566d7 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 7 Jan 2024 20:43:54 +0000 Subject: [PATCH 095/251] Tag update --- CHANGELOG.md | 6 ++++++ Project.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65314b67a..080946339 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.7.0] + ### Added - Added new Experimental sub-module AgentTools introducing `AICall` (incl. `AIGenerate`), and `AICodeFixer` structs. The AICall struct provides a "lazy" wrapper for ai* functions, enabling efficient and flexible AI interactions and building Agentic workflows. - Added the first AI Agent: `AICodeFixer` which iteratively analyzes and improves any code provided by a LLM by evaluating it in a sandbox. It allows a lot of customization (templated responses, feedback function, etc.) See `?AICodeFixer` for more information on usage and `?aicodefixer_feedback` for the example implementation of the feedback function. diff --git a/Project.toml b/Project.toml index 73a217949..e412d2d4e 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.6.0-DEV" +version = "0.7.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From 1af9279c4fc27cdd0e76fbd29bab6c6365ea3e60 Mon Sep 17 00:00:00 2001 From: Pietro Monticone <38562595+pitmonticone@users.noreply.github.com> Date: Wed, 10 Jan 2024 23:15:36 +0100 Subject: [PATCH 096/251] Fix typos (#49) --- README.md | 2 +- docs/src/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9b4860d21..41ed9bbf8 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Some features: - **`aigenerate` Function**: Simplify prompt templates with handlebars (eg, `{{variable}}`) and keyword arguments - **`@ai_str` String Macro**: Save keystrokes with a string macro for simple prompts - **Easy to Remember**: All exported functions start with `ai...` for better discoverability -- **Light Wraper Types**: Benefit from Julia's multiple dispatch by having AI outputs wrapped in specific types +- **Light Wrapper Types**: Benefit from Julia's multiple dispatch by having AI outputs wrapped in specific types - **Minimal Dependencies**: Enjoy an easy addition to your global environment with very light dependencies - **No Context Switching**: Access cutting-edge LLMs with no context switching and minimum extra keystrokes directly in your REPL diff --git a/docs/src/index.md b/docs/src/index.md index d6f988025..136adb3eb 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -18,7 +18,7 @@ Some features: - **`aigenerate` Function**: Simplify prompt templates with handlebars (eg, `{{variable}}`) and keyword arguments - **`@ai_str` String Macro**: Save keystrokes with a string macro for simple prompts - **Easy to Remember**: All exported functions start with `ai...` for better discoverability -- **Light Wraper Types**: Benefit from Julia's multiple dispatch by having AI outputs wrapped in specific types +- **Light Wrapper Types**: Benefit from Julia's multiple dispatch by having AI outputs wrapped in specific types - **Minimal Dependencies**: Enjoy an easy addition to your global environment with very light dependencies - **No Context Switching**: Access cutting-edge LLMs with no context switching and minimum extra keystrokes directly in your REPL From 1c6a5ccf7e2d9578c7abe0308ca78c69c0431b25 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 12 Jan 2024 10:44:33 +0000 Subject: [PATCH 097/251] Add LocalServerOpenAISchema to support Llama.jl --- CHANGELOG.md | 1 + src/llm_interface.jl | 53 +++++++++++++++++++++++++++++++++++++++-- src/llm_openai.jl | 36 ++++++++++++++++++++++++++++ src/user_preferences.jl | 20 ++++++++++++++-- 4 files changed, 106 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 080946339..8cb2c3b1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Initial support for [Llama.jl](https://github.com/marcom/Llama.jl) and other local servers. Once your server is started, simply use `model="local"` to route your queries to the local server, eg, `ai"Say hi!"local`. Option to permanently set the `LOCAL_SERVER` (URL) added to preference management. See `?LocalServerOpenAISchema` for more information. ### Fixed diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 17d7c51c8..43657b037 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -53,17 +53,66 @@ All user needs to do is to pass this schema as the first argument and provide th # Example -Assumes that we have a local server running at `http://localhost:8081`: +Assumes that we have a local server running at `http://127.0.0.1:8081`: ```julia api_key = "..." prompt = "Say hi!" -msg = aigenerate(CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://localhost:8081")) +msg = aigenerate(CustomOpenAISchema(), prompt; model="my_model", api_key, api_kwargs=(; url="http://127.0.0.1:8081")) ``` """ struct CustomOpenAISchema <: AbstractOpenAISchema end +""" + LocalServerOpenAISchema + +Designed to be used with local servers. It's automatically called with model alias "local" (see `MODEL_REGISTRY`). + +This schema is a flavor of CustomOpenAISchema with a `url` key` preset by global Preference key `LOCAL_SERVER`. See `?PREFERENCES` for more details on how to change it. +It assumes that the server follows OpenAI API conventions (eg, `POST /v1/chat/completions`). + +Note: Llama.cpp (and hence Llama.jl built on top of it) do NOT support embeddings endpoint! You'll get an address error. + +# Example + +Assumes that we have a local server running at `http://127.0.0.1:10897/v1` (port and address used by Llama.jl, "v1" at the end is needed for OpenAI endpoint compatibility): + +Three ways to call it: +```julia + +# Use @ai_str with "local" alias +ai"Say hi!"local + +# model="local" +aigenerate("Say hi!"; model="local") + +# Or set schema explicitly +const PT = PromptingTools +msg = aigenerate(PT.LocalServerOpenAISchema(), "Say hi!") +``` + +How to start a LLM local server? You can use `run_server` function from [Llama.jl](https://github.com/marcom/Llama.jl). Use a separate Julia session. +```julia +using Llama +model = "...path..." # see Llama.jl README how to download one +run_server(; model) +``` + +To change the default port and address: +```julia +# For a permanent change, set the preference: +using Preferences +set_preferences!("LOCAL_SERVER"=>"http://127.0.0.1:10897/v1") + +# Or if it's a temporary fix, just change the variable `LOCAL_SERVER`: +const PT = PromptingTools +PT.LOCAL_SERVER = "http://127.0.0.1:10897/v1" +``` + +""" +struct LocalServerOpenAISchema <: AbstractOpenAISchema end + """ MistralOpenAISchema diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 20696d5c3..59d6a60e6 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -120,6 +120,26 @@ function OpenAI.create_chat(schema::CustomOpenAISchema, OpenAI.create_chat(provider, model, conversation; kwargs...) end +""" + OpenAI.create_chat(schema::LocalServerOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String = "http://localhost:8080", + kwargs...) + +Dispatch to the OpenAI.create_chat function, but with the LocalServer API parameters, ie, defaults to `url` specified by the `LOCAL_SERVER` preference. See `?PREFERENCES` + +""" +function OpenAI.create_chat(schema::LocalServerOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String = LOCAL_SERVER, + kwargs...) + OpenAI.create_chat(CustomOpenAISchema(), api_key, model, conversation; url, kwargs...) +end + """ OpenAI.create_chat(schema::MistralOpenAISchema, api_key::AbstractString, @@ -172,6 +192,22 @@ function OpenAI.create_embeddings(schema::CustomOpenAISchema, provider = CustomProvider(; api_key, base_url = url) OpenAI.create_embeddings(provider, docs, model; kwargs...) end +# Set url and just forward to CustomOpenAISchema otherwise +# Note: Llama.cpp and hence Llama.jl DO NOT support the embeddings endpoint !! (they use `/embedding`) +function OpenAI.create_embeddings(schema::LocalServerOpenAISchema, + api_key::AbstractString, + docs, + model::AbstractString; + ## Strip the "v1" from the end of the url + url::String = LOCAL_SERVER, + kwargs...) + OpenAI.create_embeddings(CustomOpenAISchema(), + api_key, + docs, + model; + url, + kwargs...) +end function OpenAI.create_embeddings(schema::MistralOpenAISchema, api_key::AbstractString, docs, diff --git a/src/user_preferences.jl b/src/user_preferences.jl index a0fb21872..22f597eab 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -21,6 +21,8 @@ Check your preferences by calling `get_preferences(key::String)`. See `MODEL_ALIASES` for more information. - `MAX_HISTORY_LENGTH`: The maximum length of the conversation history. Defaults to 5. Set to `nothing` to disable history. See `CONV_HISTORY` for more information. +- `LOCAL_SERVER`: The URL of the local server to use for `ai*` calls. Defaults to `http://localhost:10897/v1`. This server is called when you call `model="local"` + See `?LocalServerOpenAISchema` for more information and examples. At the moment it is not possible to persist changes to `MODEL_REGISTRY` across sessions. Define your `register_model!()` calls in your `startup.jl` file to make them available across sessions or put them at the top of your script. @@ -28,6 +30,7 @@ Define your `register_model!()` calls in your `startup.jl` file to make them ava # Available ENV Variables - `OPENAI_API_KEY`: The API key for the OpenAI API. - `MISTRALAI_API_KEY`: The API key for the Mistral AI API. +- `LOCAL_SERVER`: The URL of the local server to use for `ai*` calls. Defaults to `http://localhost:10897/v1`. This server is called when you call `model="local"` Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. @@ -58,6 +61,7 @@ function set_preferences!(pairs::Pair{String, <:Any}...) "MODEL_ALIASES", "PROMPT_SCHEMA", "MAX_HISTORY_LENGTH", + "LOCAL_SERVER", ] for (key, value) in pairs @assert key in allowed_preferences "Unknown preference '$key'! (Allowed preferences: $(join(allowed_preferences,", "))" @@ -91,6 +95,8 @@ function get_preferences(key::String) "MODEL_EMBEDDING", "MODEL_ALIASES", "PROMPT_SCHEMA", + "MAX_HISTORY_LENGTH", + "LOCAL_SERVER", ] @assert key in allowed_preferences "Unknown preference '$key'! (Allowed preferences: $(join(allowed_preferences,", "))" getproperty(@__MODULE__, Symbol(key)) @@ -113,6 +119,10 @@ isempty(OPENAI_API_KEY) && const MISTRALAI_API_KEY::String = @load_preference("MISTRALAI_API_KEY", default=get(ENV, "MISTRALAI_API_KEY", "")); +## Address of the local server +const LOCAL_SERVER::String = @load_preference("LOCAL_SERVER", + default=get(ENV, "LOCAL_SERVER", "http://127.0.0.1:10897/v1")); + ## CONVERSATION HISTORY """ CONV_HISTORY @@ -234,7 +244,8 @@ aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", "ada" => "text-embedding-ada-002", "yi34c" => "yi:34b-chat", "oh25" => "openhermes2.5-mistral", - "starling" => "starling-lm"), + "starling" => "starling-lm", + "local" => "local-server"), ## Load aliases from preferences as well @load_preference("MODEL_ALIASES", default=Dict{String, String}())) @@ -325,7 +336,12 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", :completion_tokens => 1)), status = 200), 0.0, 0.0, - "Echo is only for testing. It always responds with 'Hello!'")) + "Echo is only for testing. It always responds with 'Hello!'"), + "local-server" => ModelSpec("local-server", + LocalServerOpenAISchema(), + 0.0, + 0.0, + "Local server, eg, powered by [Llama.jl](https://github.com/marcom/Llama.jl). Model is specified when instantiating the server itself.")) ### Model Registry Structure @kwdef mutable struct ModelRegistry From ae13c9f51365c522c0145c3bd8d57583a3416a4c Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:29:08 +0000 Subject: [PATCH 098/251] Fix ollama repeated calls (#52) Fixes https://github.com/svilupp/PromptingTools.jl/issues/51 --- CHANGELOG.md | 1 + Project.toml | 2 +- src/llm_ollama.jl | 8 ++++---- src/llm_ollama_managed.jl | 4 ++-- test/llm_ollama.jl | 8 ++++++++ test/llm_ollama_managed.jl | 9 +++++++++ 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb2c3b1e..c18124608 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial support for [Llama.jl](https://github.com/marcom/Llama.jl) and other local servers. Once your server is started, simply use `model="local"` to route your queries to the local server, eg, `ai"Say hi!"local`. Option to permanently set the `LOCAL_SERVER` (URL) added to preference management. See `?LocalServerOpenAISchema` for more information. ### Fixed +- Repeated calls to Ollama models were failing due to missing `prompt_eval_count` key in subsequent calls. ## [0.7.0] diff --git a/Project.toml b/Project.toml index e412d2d4e..81f0a16d9 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.7.0" +version = "0.8.1" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/llm_ollama.jl b/src/llm_ollama.jl index 241af35d2..f03c0c2ff 100644 --- a/src/llm_ollama.jl +++ b/src/llm_ollama.jl @@ -159,8 +159,8 @@ function aigenerate(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_ msg = AIMessage(; content = resp.response[:message][:content] |> strip, status = Int(resp.status), - tokens = (resp.response[:prompt_eval_count], - resp.response[:eval_count]), + tokens = (get(resp.response, :prompt_eval_count, 0), + get(resp.response, :eval_count, 0)), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) @@ -316,8 +316,8 @@ function aiscan(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_TYPE api_kwargs...) msg = AIMessage(; content = resp.response[:message][:content] |> strip, status = Int(resp.status), - tokens = (resp.response[:prompt_eval_count], - resp.response[:eval_count]), + tokens = (get(resp.response, :prompt_eval_count, 0), + get(resp.response, :eval_count, 0)), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 23fa175f5..a8918feea 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -216,8 +216,8 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ api_kwargs...) msg = AIMessage(; content = resp.response[:response] |> strip, status = Int(resp.status), - tokens = (resp.response[:prompt_eval_count], - resp.response[:eval_count]), + tokens = (get(resp.response, :prompt_eval_count, 0), + get(resp.response, :eval_count, 0)), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) diff --git a/test/llm_ollama.jl b/test/llm_ollama.jl index 232fc65e9..13052e7cb 100644 --- a/test/llm_ollama.jl +++ b/test/llm_ollama.jl @@ -109,6 +109,14 @@ end conversation; weather = "sunny", return_all = true)[1] == expected_convo_output[1] + + # Test if subsequent eval misses the prompt_eval_count key + response = Dict(:message => Dict(:content => "Prompt message")) + # :prompt_eval_count => 2, + # :eval_count => 1) + schema = TestEchoOllamaSchema(; response, status = 200) + msg = [aigenerate(schema, "hi") for i in 1:3] |> last + @test msg.tokens == (0, 0) end # @testset "aiembed-ollama" begin diff --git a/test/llm_ollama_managed.jl b/test/llm_ollama_managed.jl index 35a3461c0..6456b1dac 100644 --- a/test/llm_ollama_managed.jl +++ b/test/llm_ollama_managed.jl @@ -154,7 +154,16 @@ end @test_throws ErrorException aigenerate(schema, UserMessageWithImages("abc"; image_url = "https://example.com")) end + + # Test if subsequent eval misses the prompt_eval_count key + response = Dict(:response => "Hello John") + # :prompt_eval_count => 2, + # :eval_count => 1) + schema = TestEchoOllamaManagedSchema(; response, status = 200) + msg = [aigenerate(schema, "hi") for i in 1:3] |> last + @test msg.tokens == (0, 0) end + @testset "aiembed-ollama" begin @testset "single doc" begin response = Dict(:embedding => ones(16)) From d81a2d3a73c29eeeac1cf952380c4309b790622b Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 17 Jan 2024 08:57:54 +0000 Subject: [PATCH 099/251] Add template and project version update --- CHANGELOG.md | 7 +++++++ Project.toml | 2 +- templates/persona-task/StorytellerExplainSHAP.json | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 templates/persona-task/StorytellerExplainSHAP.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c18124608..adfb96f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.8.0] + ### Added - Initial support for [Llama.jl](https://github.com/marcom/Llama.jl) and other local servers. Once your server is started, simply use `model="local"` to route your queries to the local server, eg, `ai"Say hi!"local`. Option to permanently set the `LOCAL_SERVER` (URL) added to preference management. See `?LocalServerOpenAISchema` for more information. +- Added a new template `StorytellerExplainSHAP` (see the metadata) ### Fixed - Repeated calls to Ollama models were failing due to missing `prompt_eval_count` key in subsequent calls. diff --git a/Project.toml b/Project.toml index 81f0a16d9..d878ac13f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.8.1" +version = "0.8.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/templates/persona-task/StorytellerExplainSHAP.json b/templates/persona-task/StorytellerExplainSHAP.json new file mode 100644 index 000000000..82b295dcb --- /dev/null +++ b/templates/persona-task/StorytellerExplainSHAP.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"Explain ML model predictions with storytelling, use `instructions` to adjust the audience and style as needed. All placeholders should be used. Inspired by [Tell me a story!](https://arxiv.org/abs/2309.17057). If you don't need any instructions, set `instructions=\"None.\"`. Placeholders: `task_definition`,`feature_description`,`label_definition`, `probability_pct`, `prediction`, `outcome`, `classified_correctly`, `shap_table`,`instructions`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You're a data science storyteller. Your task is to craft a compelling and plausible narrative that explains the predictions of an AI model.\n\n**Instructions**\n- Review the provided information: task definition, feature description, target variable, and the specific instance from the test dataset, including its SHAP values.\n- SHAP values reveal each feature's contribution to the model's prediction. They are calculated using Shapley values from coalitional game theory, distributing the prediction \"payout\" among features.\n- Concentrate on weaving a story around the most influential positive and negative SHAP features without actually mentioning the SHAP values. Consider potential feature interactions that fit the story. Skip all features outside of the story.\n- SHAP and its values are TOP SECRET. They must not be mentioned.\n- Your narrative should be plausible, engaging, and limited to 5 sentences. \n- Do not address or speak to the audience, focus only on the story.\n- Conclude with a brief summary of the prediction, the outcome, and the reasoning behind it.\n\n**Context**\nAn AI model predicts {{task_definition}}. \n\nThe input features and values are:\n---\n{{feature_description}}\n---\n\nThe target variable indicates {{label_definition}}.\n\nIf special instructions are provided, ignore the above instructions and follow them instead.\n ","variables":["task_definition","feature_description","label_definition"],"_type":"systemmessage"},{"content":"Explain this particular instance. \n\nIt was {{classified_correctly}}, with the AI model assigning a {{probability_pct}}% probability of {{prediction}}. The actual outcome was {{outcome}}. \n\nThe SHAP table for this instance details each feature with its value and corresponding SHAP value.\n---\n{{shap_table}}\n---\n\nSpecial Instructions: {{instructions}}\n\nOur story begins\n","variables":["classified_correctly","probability_pct","prediction","outcome","shap_table","instructions"],"_type":"usermessage"}] \ No newline at end of file From 362aa867d21a3aa786f81808e6769408a3ca43a6 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 21 Jan 2024 10:14:24 +0000 Subject: [PATCH 100/251] Fix separators (#55) --- CHANGELOG.md | 5 +++++ Project.toml | 2 +- src/utils.jl | 7 ++++--- test/utils.jl | 10 ++++++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adfb96f71..fae61485a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [0.8.1] + +### Fixed +- Fixed `split_by_length` to not mutate `separators` argument (appeared in RAG use cases where we repeatedly apply splits to different documents) + ## [0.8.0] ### Added diff --git a/Project.toml b/Project.toml index d878ac13f..81f0a16d9 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.8.0" +version = "0.8.1" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/utils.jl b/src/utils.jl index 967a59c3c..3b8cacc9f 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -151,12 +151,13 @@ chunks = split_by_length(text, [","], max_length=10000) """ function split_by_length(text, separators::Vector{String}; max_length) @assert !isempty(separators) "`separators` can't be empty" - separator = popfirst!(separators) + separators_ = copy(separators) + separator = popfirst!(separators_) chunks = split_by_length(text; separator, max_length) - isempty(separators) && return chunks + isempty(separators_) && return chunks ## Iteratively split by separators - for separator in separators + for separator in separators_ chunks = mapreduce(text_ -> split_by_length(text_; max_length, separator), vcat, chunks) diff --git a/test/utils.jl b/test/utils.jl index 4c1865c8d..04a452d71 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -67,12 +67,22 @@ end # empty separators text = "Some text without separators." @test_throws AssertionError split_by_length(text, String[], max_length = 10) + # edge cases text = "Short text" separators = ["\n\n", ". ", "\n"] chunks = split_by_length(text, separators, max_length = 50) @test length(chunks) == 1 @test chunks[1] == text + + # do not mutate separators input + text = "Paragraph 1\n\nParagraph 2. Sentence 1. Sentence 2.\nParagraph 3" + separators = ["\n\n", ". ", "\n"] + sep_length = length(separators) + chunks = split_by_length(text, separators, max_length = 20) + chunks = split_by_length(text, separators, max_length = 20) + chunks = split_by_length(text, separators, max_length = 20) + @test length(separators) == sep_length end @testset "extract_handlebar_variables" begin From 38924ce18f1e3c85d9d4c083f0dcc7400afab254 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 22 Jan 2024 21:53:18 +0000 Subject: [PATCH 101/251] Refactor RAGTools (#56) - Split `Experimental.RAGTools.build_index` into smaller functions to easier sharing with other packages (`get_chunks`, `get_embeddings`, `get_metadata`) - Added support for Cohere-based RAG re-ranking strategy (and introduced associated `COHERE_API_KEY` global variable and ENV variable) --- CHANGELOG.md | 8 + Project.toml | 2 +- src/Experimental/RAGTools/RAGTools.jl | 5 +- src/Experimental/RAGTools/api_services.jl | 36 +++ src/Experimental/RAGTools/generation.jl | 16 +- src/Experimental/RAGTools/preparation.jl | 264 ++++++++++++++++------ src/Experimental/RAGTools/retrieval.jl | 105 ++++++++- src/Experimental/RAGTools/types.jl | 83 ++++++- src/PromptingTools.jl | 1 + src/user_preferences.jl | 7 + src/utils.jl | 16 +- test/Experimental/RAGTools/evaluation.jl | 2 +- test/Experimental/RAGTools/preparation.jl | 16 +- test/Experimental/RAGTools/retrieval.jl | 61 ++++- test/Experimental/RAGTools/types.jl | 55 ++++- test/utils.jl | 13 +- 16 files changed, 598 insertions(+), 92 deletions(-) create mode 100644 src/Experimental/RAGTools/api_services.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index fae61485a..b8d7dd0db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [0.9.0] + +### Added +- Split `Experimental.RAGTools.build_index` into smaller functions to easier sharing with other packages (`get_chunks`, `get_embeddings`, `get_metadata`) +- Added support for Cohere-based RAG re-ranking strategy (and introduced associated `COHERE_API_KEY` global variable and ENV variable) + +### Fixed + ## [0.8.1] ### Fixed diff --git a/Project.toml b/Project.toml index 81f0a16d9..7e7f82e32 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.8.1" +version = "0.9.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/Experimental/RAGTools/RAGTools.jl b/src/Experimental/RAGTools/RAGTools.jl index a315a2a7f..7f491fd36 100644 --- a/src/Experimental/RAGTools/RAGTools.jl +++ b/src/Experimental/RAGTools/RAGTools.jl @@ -10,11 +10,14 @@ This module is experimental and may change at any time. It is intended to be mov module RAGTools using PromptingTools -using JSON3 +using HTTP, JSON3 const PT = PromptingTools include("utils.jl") +# eg, cohere_api +include("api_services.jl") + export ChunkIndex, CandidateChunks # MultiIndex include("types.jl") diff --git a/src/Experimental/RAGTools/api_services.jl b/src/Experimental/RAGTools/api_services.jl new file mode 100644 index 000000000..adc7ac824 --- /dev/null +++ b/src/Experimental/RAGTools/api_services.jl @@ -0,0 +1,36 @@ +""" + cohere_api(; + api_key::AbstractString, + endpoint::String, + url::AbstractString="https://api.cohere.ai/v1", + http_kwargs::NamedTuple=NamedTuple(), + kwargs...) + +Lightweight wrapper around the Cohere API. See https://cohere.com/docs for more details. + +# Arguments +- `api_key`: Your Cohere API key. You can get one from https://dashboard.cohere.com/welcome/register (trial access is for free). +- `endpoint`: The Cohere endpoint to call. +- `url`: The base URL for the Cohere API. Default is `https://api.cohere.ai/v1`. +- `http_kwargs`: Any additional keyword arguments to pass to `HTTP.post`. +- `kwargs`: Any additional keyword arguments to pass to the Cohere API. +""" +function cohere_api(; + api_key::AbstractString, + endpoint::String, + url::AbstractString = "https://api.cohere.ai/v1", + http_kwargs::NamedTuple = NamedTuple(), + kwargs...) + @assert endpoint in ["chat", "generate", "embed", "rerank", "classify"] "Only 'chat', 'generate',`embed`,`rerank`,`classify` Cohere endpoints are supported." + @assert !isempty(api_key) "Cohere `api_key` cannot be empty. Check `PT.COHERE_API_KEY` or pass it as a keyword argument." + ## + input_body = Dict(kwargs...) + + # https://api.cohere.ai/v1/rerank + api_url = string(url, "/", endpoint) + resp = HTTP.post(api_url, + PT.auth_header(api_key), + JSON3.write(input_body); http_kwargs...) + body = JSON3.read(resp.body) + return (; response = body) +end \ No newline at end of file diff --git a/src/Experimental/RAGTools/generation.jl b/src/Experimental/RAGTools/generation.jl index 59b130b3c..d9725f6bb 100644 --- a/src/Experimental/RAGTools/generation.jl +++ b/src/Experimental/RAGTools/generation.jl @@ -39,7 +39,7 @@ end """ airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromContext; question::AbstractString, - top_k::Int = 3, `minimum_similarity::AbstractFloat`= -1.0, + top_k::Int = 100, top_n::Int = 5, minimum_similarity::AbstractFloat = -1.0, tag_filter::Union{Symbol, Vector{String}, Regex, Nothing} = :auto, rerank_strategy::RerankingStrategy = Passthrough(), model_embedding::String = PT.MODEL_EMBEDDING, model_chat::String = PT.MODEL_CHAT, @@ -47,6 +47,7 @@ end metadata_template::Symbol = :RAGExtractMetadataShort, chunks_window_margin::Tuple{Int, Int} = (1, 1), return_context::Bool = false, verbose::Bool = true, + rerank_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) @@ -59,9 +60,10 @@ The function selects relevant chunks from an `ChunkIndex`, optionally filters th - `rag_template::Symbol`: Template for the RAG model, defaults to `:RAGAnswerFromContext`. - `question::AbstractString`: The question to be answered. - `top_k::Int`: Number of top candidates to retrieve based on embedding similarity. +- `top_n::Int`: Number of candidates to return after reranking. - `minimum_similarity::AbstractFloat`: Minimum similarity threshold (between -1 and 1) for filtering chunks based on embedding similarity. Defaults to -1.0. - `tag_filter::Union{Symbol, Vector{String}, Regex}`: Mechanism for filtering chunks based on tags (either automatically detected, specific tags, or a regex pattern). Disabled by setting to `nothing`. -- `rerank_strategy::RerankingStrategy`: Strategy for reranking the retrieved chunks. +- `rerank_strategy::RerankingStrategy`: Strategy for reranking the retrieved chunks. Defaults to `Passthrough()`. Use `CohereRerank` for better results (requires `COHERE_API_KEY` to be set) - `model_embedding::String`: Model used for embedding the question, default is `PT.MODEL_EMBEDDING`. - `model_chat::String`: Model used for generating the final response, default is `PT.MODEL_CHAT`. - `model_metadata::String`: Model used for extracting metadata, default is `PT.MODEL_CHAT`. @@ -97,7 +99,7 @@ See also `build_index`, `build_context`, `CandidateChunks`, `find_closest`, `fin """ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromContext; question::AbstractString, - top_k::Int = 3, minimum_similarity::AbstractFloat = -1.0, + top_k::Int = 100, top_n::Int = 5, minimum_similarity::AbstractFloat = -1.0, tag_filter::Union{Symbol, Vector{String}, Regex, Nothing} = :auto, rerank_strategy::RerankingStrategy = Passthrough(), model_embedding::String = PT.MODEL_EMBEDDING, model_chat::String = PT.MODEL_CHAT, @@ -105,6 +107,7 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC metadata_template::Symbol = :RAGExtractMetadataShort, chunks_window_margin::Tuple{Int, Int} = (1, 1), return_context::Bool = false, verbose::Bool = true, + rerank_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), kwargs...) ## Note: Supports only single ChunkIndex for now @@ -148,7 +151,12 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC filtered_candidates = isnothing(tag_candidates) ? emb_candidates : (emb_candidates & tag_candidates) - reranked_candidates = rerank(rerank_strategy, index, question, filtered_candidates) + reranked_candidates = rerank(rerank_strategy, + index, + question, + filtered_candidates; + top_n, + verbose = false, rerank_kwargs...) ## Build the context context = build_context(index, reranked_candidates; chunks_window_margin) diff --git a/src/Experimental/RAGTools/preparation.jl b/src/Experimental/RAGTools/preparation.jl index 50e937805..2b80a9e0f 100644 --- a/src/Experimental/RAGTools/preparation.jl +++ b/src/Experimental/RAGTools/preparation.jl @@ -34,26 +34,181 @@ function build_tags end "Build an index for RAG (Retriever-Augmented Generation) applications. REQUIRES SparseArrays and LinearAlgebra packages to be loaded!!" function build_index end +"Shortcut to LinearAlgebra.normalize. Provided in the package extension `RAGToolsExperimentalExt` (Requires SparseArrays and LinearAlgebra)" +function _normalize end + """ - build_index(files::Vector{<:AbstractString}; - separators = ["\n\n", ". ", "\n"], max_length::Int = 256, - extract_metadata::Bool = false, verbose::Bool = true, + get_chunks(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, + sources::Vector{<:AbstractString} = files_or_docs, + verbose::Bool = true, + separators = ["\n\n", ". ", "\n"], max_length::Int = 256) + +Chunks the provided `files_or_docs` into chunks of maximum length `max_length` (if possible with provided `separators`). + +Supports two modes of operation: +- `reader=:files`: The function opens each file in `files_or_docs` and reads its content. +- `reader=:docs`: The function assumes that `files_or_docs` is a vector of strings to be chunked. + +# Arguments +- `files_or_docs`: A vector of valid file paths OR string documents to be chunked. +- `reader`: A symbol indicating the type of input, can be either `:files` or `:docs`. Default is `:files`. +- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\n\n", ". ", "\n"]`. +- `max_length`: The maximum length of each chunk (if possible with provided separators). Default is 256. +- `sources`: A vector of strings indicating the source of each chunk. Default is equal to `files_or_docs` (for `reader=:files`) + +""" +function get_chunks(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, + sources::Vector{<:AbstractString} = files_or_docs, + verbose::Bool = true, + separators = ["\n\n", ". ", "\n"], max_length::Int = 256) + + ## Check that all items must be existing files or strings + @assert reader in [:files, :docs] "Invalid `read` argument. Must be one of [:files, :docs]" + if reader == :files + @assert all(isfile, files_or_docs) "Some paths in `files_or_docs` don't exist (Check: $(join(filter(!isfile,files_or_docs),", "))" + else + @assert sources!=files_or_docs "When `reader=:docs`, vector of `sources` must be provided" + end + @assert isnothing(sources)||(length(sources) == length(files_or_docs)) "Length of `sources` must match length of `files_or_docs`" + @assert maximum(length.(sources))<=512 "Each source must be less than 512 characters long (Detected: $(maximum(length.(sources))))" + + output_chunks = Vector{SubString{String}}() + output_sources = Vector{eltype(sources)}() + + # Do chunking first + for i in eachindex(files_or_docs, sources) + # if reader == :files, we open the files and read them + doc_raw = if reader == :files + fn = files_or_docs[i] + (verbose > 0) && @info "Processing file: $fn" + read(fn, String) + else + files_or_docs[i] + end + isempty(doc_raw) && continue + # split into chunks, if you want to start simple - just do `split(text,"\n\n")` + doc_chunks = PT.split_by_length(doc_raw, separators; max_length) .|> strip |> + x -> filter(!isempty, x) + # skip if no chunks found + isempty(doc_chunks) && continue + append!(output_chunks, doc_chunks) + append!(output_sources, fill(sources[i], length(doc_chunks))) + end + + return output_chunks, output_sources +end + +""" + get_embeddings(docs::Vector{<:AbstractString}; + verbose::Bool = true, + cost_tracker = Threads.Atomic{Float64}(0.0), + kwargs...) + +Embeds a vector of `docs` using the provided model (kwarg `model`). + +Tries to batch embedding calls for roughly 80K characters per call (to avoid exceeding the API limit) but reduce network latency. + +Note: `docs` are assumed to be already chunked to the reasonable sizes that fit within the embedding context limit. + +# Arguments +- `docs`: A vector of strings to be embedded. +- `verbose`: A boolean flag for verbose output. Default is `true`. +- `model`: The model to use for embedding. Default is `PT.MODEL_EMBEDDING`. +- `cost_tracker`: A `Threads.Atomic{Float64}` object to track the total cost of the API calls. Useful to pass the total cost to the parent call. + +""" +function get_embeddings(docs::Vector{<:AbstractString}; + verbose::Bool = true, + cost_tracker = Threads.Atomic{Float64}(0.0), + kwargs...) + verbose && @info "Embedding $(length(docs)) documents..." + model = hasproperty(kwargs, :model) ? kwargs.model : PT.MODEL_EMBEDDING + # Notice that we embed multiple docs at once, not one by one + # OpenAI supports embedding multiple documents to reduce the number of API calls/network latency time + # We do batch them just in case the documents are too large (targeting at most 80K characters per call) + avg_length = sum(length.(docs)) / length(docs) + embedding_batch_size = floor(Int, 80_000 / avg_length) + embeddings = asyncmap(Iterators.partition(docs, embedding_batch_size)) do docs_chunk + msg = aiembed(docs_chunk, + _normalize; + verbose = false, + kwargs...) + Threads.atomic_add!(cost_tracker, PT.call_cost(msg, model)) # track costs + msg.content + end + embeddings = hcat(embeddings...) .|> Float32 # flatten, columns are documents + verbose && @info "Done embedding. Total cost: \$$(round(cost_tracker[],digits=3))" + return embeddings +end + +""" + get_metadata(docs::Vector{<:AbstractString}; + verbose::Bool = true, + cost_tracker = Threads.Atomic{Float64}(0.0), + kwargs...) + +Extracts metadata from a vector of `docs` using the provided model (kwarg `model`). + +# Arguments +- `docs`: A vector of strings to be embedded. +- `verbose`: A boolean flag for verbose output. Default is `true`. +- `model`: The model to use for metadata extraction. Default is `PT.MODEL_CHAT`. +- `metadata_template`: A template to be used for metadata extraction. Default is `:RAGExtractMetadataShort`. +- `cost_tracker`: A `Threads.Atomic{Float64}` object to track the total cost of the API calls. Useful to pass the total cost to the parent call. + +""" +function get_metadata(docs::Vector{<:AbstractString}; + verbose::Bool = true, + metadata_template::Symbol = :RAGExtractMetadataShort, + cost_tracker = Threads.Atomic{Float64}(0.0), + kwargs...) + model = hasproperty(kwargs, :model) ? kwargs.model : PT.MODEL_CHAT + _check_aiextract_capability(model) + verbose && @info "Extracting metadata from $(length(docs)) documents..." + metadata = asyncmap(docs) do docs_chunk + try + msg = aiextract(metadata_template; + return_type = MaybeMetadataItems, + text = docs_chunk, + instructions = "None.", + verbose = false, + model, kwargs...) + Threads.atomic_add!(cost_tracker, PT.call_cost(msg, model)) # track costs + items = metadata_extract(msg.content.items) + catch + String[] + end + end + verbose && + @info "Done extracting the metadata. Total cost: \$$(round(cost_tracker[],digits=3))" + return metadata +end + +""" + build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, + separators = ["\\n\\n", ". ", "\\n"], max_length::Int = 256, + sources::Vector{<:AbstractString} = files_or_docs, + extract_metadata::Bool = false, verbose::Int = 1, + index_id = gensym("ChunkIndex"), metadata_template::Symbol = :RAGExtractMetadataShort, model_embedding::String = PT.MODEL_EMBEDDING, model_metadata::String = PT.MODEL_CHAT, - api_kwargs::NamedTuple = NamedTuple()) + api_kwargs::NamedTuple = NamedTuple(), + cost_tracker = Threads.Atomic{Float64}(0.0)) Build an index for RAG (Retriever-Augmented Generation) applications from the provided file paths. The function processes each file, splits its content into chunks, embeds these chunks, optionally extracts metadata, and then compiles this information into a retrievable index. # Arguments -- `files`: A vector of valid file paths to be indexed. -- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `["\n\n", ". ", "\n"]`. +- `files_or_docs`: A vector of valid file paths OR string documents to be indexed (chunked and embedded). +- `reader`: A symbol indicating the type of input, can be either `:files` or `:docs`. Default is `:files`. +- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\\n\\n", ". ", "\\n"]`. - `max_length`: The maximum length of each chunk (if possible with provided separators). Default is 256. +- `sources`: A vector of strings indicating the source of each chunk. Default is equal to `files_or_docs` (for `reader=:files`) - `extract_metadata`: A boolean flag indicating whether to extract metadata from each chunk (to build filter `tags` in the index). Default is `false`. Metadata extraction incurs additional cost and requires `model_metadata` and `metadata_template` to be provided. -- `verbose`: A boolean flag for verbose output. Default is `true`. +- `verbose`: An Integer specifying the verbosity of the logs. Default is `1` (high-level logging). `0` is disabled. - `metadata_template`: A symbol indicating the template to be used for metadata extraction. Default is `:RAGExtractMetadataShort`. - `model_embedding`: The model to use for embedding. - `model_metadata`: The model to use for metadata extraction. @@ -69,79 +224,56 @@ See also: `MultiIndex`, `CandidateChunks`, `find_closest`, `find_tags`, `rerank` # Assuming `test_files` is a vector of file paths index = build_index(test_files; max_length=10, extract_metadata=true) -# Another example with metadata extraction and verbose output +# Another example with metadata extraction and verbose output (`reader=:files` is implicit) index = build_index(["file1.txt", "file2.txt"]; separators=[". "], extract_metadata=true, verbose=true) ``` """ -function build_index(files::Vector{<:AbstractString}; +function build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, separators = ["\n\n", ". ", "\n"], max_length::Int = 256, - extract_metadata::Bool = false, verbose::Bool = true, + sources::Vector{<:AbstractString} = files_or_docs, + extract_metadata::Bool = false, verbose::Integer = 1, + index_id = gensym("ChunkIndex"), metadata_template::Symbol = :RAGExtractMetadataShort, model_embedding::String = PT.MODEL_EMBEDDING, model_metadata::String = PT.MODEL_CHAT, - api_kwargs::NamedTuple = NamedTuple()) - ## - @assert all(isfile, files) "Some `files` don't exist (Check: $(join(filter(!isfile,files),", "))" - - output_chunks = Vector{Vector{SubString{String}}}() - output_embeddings = Vector{Matrix{Float32}}() - output_metadata = Vector{Vector{Vector{String}}}() - output_sources = Vector{Vector{eltype(files)}}() - cost_tracker = Threads.Atomic{Float64}(0.0) - - for fn in files - verbose && @info "Processing file: $fn" - doc_raw = read(fn, String) - isempty(doc_raw) && continue - # split into chunks, if you want to start simple - just do `split(text,"\n\n")` - doc_chunks = PT.split_by_length(doc_raw, separators; max_length) .|> strip |> - x -> filter(!isempty, x) - # skip if no chunks found - isempty(doc_chunks) && continue - push!(output_chunks, doc_chunks) - push!(output_sources, fill(fn, length(doc_chunks))) - - # Notice that we embed all doc_chunks at once, not one by one - # OpenAI supports embedding multiple documents to reduce the number of API calls/network latency time - emb = aiembed(doc_chunks, _normalize; model = model_embedding, verbose, api_kwargs) - Threads.atomic_add!(cost_tracker, PT.call_cost(emb, model_embedding)) # track costs - push!(output_embeddings, Float32.(emb.content)) - - if extract_metadata && !isempty(model_metadata) - _check_aiextract_capability(model_metadata) - metadata_ = asyncmap(doc_chunks) do chunk - try - msg = aiextract(metadata_template; - return_type = MaybeMetadataItems, - text = chunk, - instructions = "None.", - verbose, - model = model_metadata, api_kwargs) - Threads.atomic_add!(cost_tracker, PT.call_cost(msg, model_metadata)) # track costs - items = metadata_extract(msg.content.items) - catch - String[] - end - end - push!(output_metadata, metadata_) - end - end - ## Create metadata tags and associated vocabulary - tags, tags_vocab = if !isempty(output_metadata) - # Requires SparseArrays.jl! - build_tags(vcat(output_metadata...)) # need to vcat to be on the "chunk-level" + api_kwargs::NamedTuple = NamedTuple(), + cost_tracker = Threads.Atomic{Float64}(0.0)) + + ## Split into chunks + output_chunks, output_sources = get_chunks(files_or_docs; + reader, sources, separators, max_length) + + ## Embed chunks + embeddings = get_embeddings(output_chunks; + verbose = (verbose > 1), + cost_tracker, + model = model_embedding, + api_kwargs) + + ## Extract metadata + tags, tags_vocab = if extract_metadata + output_metadata = get_metadata(output_chunks; + verbose = (verbose > 1), + cost_tracker, + model = model_metadata, + metadata_template, + api_kwargs) + # Requires SparseArrays.jl to be loaded + build_tags(output_metadata) else - tags, tags_vocab = nothing, nothing + nothing, nothing end - verbose && @info "Index built! (cost: \$$(round(cost_tracker[], digits=3)))" + ## Create metadata tag array and associated vocabulary + (verbose > 0) && @info "Index built! (cost: \$$(round(cost_tracker[], digits=3)))" index = ChunkIndex(; - embeddings = hcat(output_embeddings...), + id = index_id, + embeddings, tags, tags_vocab, - chunks = vcat(output_chunks...), - sources = vcat(output_sources...)) + chunks = output_chunks, + sources = output_sources) return index end diff --git a/src/Experimental/RAGTools/retrieval.jl b/src/Experimental/RAGTools/retrieval.jl index a3d136aa4..d7c998731 100644 --- a/src/Experimental/RAGTools/retrieval.jl +++ b/src/Experimental/RAGTools/retrieval.jl @@ -32,6 +32,25 @@ function find_closest(index::AbstractChunkIndex, minimum_similarity) return CandidateChunks(index.id, positions, Float32.(distances)) end +## function find_closest(index::AbstractMultiIndex, +## query_emb::AbstractVector{<:Real}; +## top_k::Int = 100, minimum_similarity::AbstractFloat = -1.0) +## all_candidates = CandidateChunks[] +## for idxs in indexes(index) +## candidates = find_closest(idxs, query_emb; +## top_k, +## minimum_similarity) +## if !isempty(candidates.positions) +## push!(all_candidates, candidates) +## end +## end +## ## build vector of all distances and pick top_k +## all_distances = mapreduce(x -> x.distances, vcat, all_candidates) +## top_k_order = all_distances |> sortperm |> x -> last(x, top_k) +## return CandidateChunks(index.id, +## all_candidates[top_k_order], +## all_distances[top_k_order]) +## end function find_tags(index::AbstractChunkIndex, tag::Union{AbstractString, Regex}) @@ -57,8 +76,90 @@ end abstract type RerankingStrategy end struct Passthrough <: RerankingStrategy end +struct CohereRerank <: RerankingStrategy end -function rerank(strategy::Passthrough, index, question, candidate_chunks; kwargs...) +function rerank(strategy::Passthrough, + index, + question, + candidate_chunks; + top_n::Integer = length(candidate_chunks), + kwargs...) # Since this is a Passthrough strategy, it returns the candidate_chunks unchanged - return candidate_chunks + return first(candidate_chunks, top_n) +end + +function rerank(strategy::CohereRerank, + index::AbstractDocumentIndex, args...; kwargs...) + throw(ArgumentError("Not implemented yet")) +end + +""" + rerank(strategy::CohereRerank, index::AbstractChunkIndex, question, + candidate_chunks; + verbose::Bool = false, + api_key::AbstractString = PT.COHERE_API_KEY, + top_n::Integer = length(candidate_chunks.distances), + model::AbstractString = "rerank-english-v2.0", + return_documents::Bool = false, + kwargs...) + +Re-ranks a list of candidate chunks using the Cohere Rerank API. See https://cohere.com/rerank for more details. + +# Arguments +- `query`: The query to be used for the search. +- `documents`: A vector of documents to be reranked. + The total max chunks (`length of documents * max_chunks_per_doc`) must be less than 10000. We recommend less than 1000 documents for optimal performance. +- `top_n`: The number of most relevant documents to return. Default is `length(documents)`. +- `model`: The model to use for reranking. Default is `rerank-english-v2.0`. +- `return_documents`: A boolean flag indicating whether to return the reranked documents in the response. Default is `false`. +- `max_chunks_per_doc`: The maximum number of chunks to use per document. Default is `10`. +- `verbose`: A boolean flag indicating whether to print verbose logging. Default is `false`. + +""" +function rerank(strategy::CohereRerank, index::AbstractChunkIndex, question, + candidate_chunks; + verbose::Bool = false, + api_key::AbstractString = PT.COHERE_API_KEY, + top_n::Integer = length(candidate_chunks.distances), + model::AbstractString = "rerank-english-v2.0", + return_documents::Bool = false, + kwargs...) + @assert top_n>0 "top_n must be a positive integer." + @assert index.id==candidate_chunks.index_id "The index id of the index and candidate_chunks must match." + + ## Call the API + documents = index[candidate_chunks, :chunks] + verbose && + @info "Calling Cohere Rerank API with $(length(documents)) candidate chunks..." + r = cohere_api(; + api_key, + endpoint = "rerank", + query = question, + documents, + top_n, + model, + return_documents, + kwargs...) + + ## Unwrap re-ranked positions + positions = Vector{Int}(undef, length(r.response[:results])) + distances = Vector{Float32}(undef, length(r.response[:results])) + for i in eachindex(r.response[:results]) + doc = r.response[:results][i] + positions[i] = candidate_chunks.positions[doc[:index] + 1] + distances[i] = doc[:relevance_score] + end + + ## Check the cost + search_units_str = if haskey(r.response, :meta) && + haskey(r.response[:meta], :billed_units) && + haskey(r.response[:meta][:billed_units], :search_units) + units = r.response[:meta][:billed_units][:search_units] + "Charged $(units) search units." + else + "" + end + verbose && @info "Reranking done. $search_units_str" + + return CandidateChunks(index.id, positions, distances) end \ No newline at end of file diff --git a/src/Experimental/RAGTools/types.jl b/src/Experimental/RAGTools/types.jl index 2aeb7a4d0..dab4e78ba 100644 --- a/src/Experimental/RAGTools/types.jl +++ b/src/Experimental/RAGTools/types.jl @@ -3,6 +3,7 @@ # In addition, RAGContext is defined for debugging purposes abstract type AbstractDocumentIndex end +abstract type AbstractMultiIndex <: AbstractDocumentIndex end abstract type AbstractChunkIndex <: AbstractDocumentIndex end # More advanced index would be: HybridChunkIndex @@ -35,6 +36,10 @@ function Base.var"=="(i1::ChunkIndex, i2::ChunkIndex) (i1.embeddings == i2.embeddings) && (i1.chunks == i2.chunks) && (i1.tags == i2.tags)) end +function Base.vcat(i1::AbstractDocumentIndex, i2::AbstractDocumentIndex) + throw(ArgumentError("Not implemented")) +end + function Base.vcat(i1::ChunkIndex, i2::ChunkIndex) tags_, tags_vocab_ = if (isnothing(tags(i1)) || isnothing(tags(i2))) nothing, nothing @@ -54,9 +59,9 @@ function Base.vcat(i1::ChunkIndex, i2::ChunkIndex) end "Composite index that stores multiple ChunkIndex objects and their embeddings" -@kwdef struct MultiIndex <: AbstractDocumentIndex +@kwdef struct MultiIndex <: AbstractMultiIndex id::Symbol = gensym("MultiIndex") - indexes::Vector{<:ChunkIndex} + indexes::Vector{<:AbstractChunkIndex} end indexes(index::MultiIndex) = index.indexes # check that each index has a counterpart in the other MultiIndex @@ -76,13 +81,27 @@ function Base.var"=="(i1::MultiIndex, i2::MultiIndex) end abstract type AbstractCandidateChunks end -@kwdef struct CandidateChunks{T <: Real} <: AbstractCandidateChunks +@kwdef struct CandidateChunks{TP <: Union{Integer, AbstractCandidateChunks}, TD <: Real} <: + AbstractCandidateChunks index_id::Symbol - positions::Vector{Int} = Int[] - distances::Vector{T} = Float32[] + ## if TP is Int, then positions are indices into the index + ## if TP is CandidateChunks, then positions are indices into the positions of the child index in MultiIndex + positions::Vector{TP} = Int[] + distances::Vector{TD} = Float32[] +end +Base.length(cc::CandidateChunks) = length(cc.positions) +function Base.first(cc::CandidateChunks, k::Integer) + CandidateChunks(cc.index_id, first(cc.positions, k), first(cc.distances, k)) end # combine/intersect two candidate chunks. average the score if available -function Base.var"&"(cc1::CandidateChunks, cc2::CandidateChunks) +function Base.var"&"(cc1::AbstractCandidateChunks, + cc2::AbstractCandidateChunks) + throw(ArgumentError("Not implemented")) +end +function Base.var"&"(cc1::CandidateChunks{TP1, TD1}, + cc2::CandidateChunks{TP2, TD2}) where + {TP1 <: Integer, TP2 <: Integer, TD1 <: Real, TD2 <: Real} + ## cc1.index_id != cc2.index_id && return CandidateChunks(; index_id = cc1.index_id) positions = intersect(cc1.positions, cc2.positions) @@ -93,17 +112,39 @@ function Base.var"&"(cc1::CandidateChunks, cc2::CandidateChunks) end CandidateChunks(cc1.index_id, positions, distances) end -function Base.getindex(ci::ChunkIndex, candidate::CandidateChunks, field::Symbol = :chunks) - @assert field==:chunks "Only `chunks` field is supported for now" + +function Base.getindex(ci::AbstractDocumentIndex, + candidate::AbstractCandidateChunks, + field::Symbol) + throw(ArgumentError("Not implemented")) +end +function Base.getindex(ci::ChunkIndex, + candidate::CandidateChunks{TP, TD}, + field::Symbol = :chunks) where {TP <: Integer, TD <: Real} + @assert field in [:chunks, :embeddings, :sources] "Only `chunks`, `embeddings`, `sources` fields are supported for now" len_ = length(chunks(ci)) @assert all(1 .<= candidate.positions .<= len_) "Some positions are out of bounds" if ci.id == candidate.index_id - chunks(ci)[candidate.positions] + if field == :chunks + @views chunks(ci)[candidate.positions] + elseif field == :embeddings + @views embeddings(ci)[:, candidate.positions] + elseif field == :sources + @views sources(ci)[candidate.positions] + end else - eltype(chunks(ci))[] + if field == :chunks + eltype(chunks(ci))[] + elseif field == :embeddings + eltype(embeddings(ci))[] + elseif field == :sources + eltype(sources(ci))[] + end end end -function Base.getindex(mi::MultiIndex, candidate::CandidateChunks, field::Symbol = :chunks) +function Base.getindex(mi::MultiIndex, + candidate::CandidateChunks{TP, TD}, + field::Symbol = :chunks) where {TP <: Integer, TD <: Real} @assert field==:chunks "Only `chunks` field is supported for now" valid_index = findfirst(x -> x.id == candidate.index_id, indexes(mi)) if isnothing(valid_index) @@ -112,6 +153,26 @@ function Base.getindex(mi::MultiIndex, candidate::CandidateChunks, field::Symbol getindex(indexes(mi)[valid_index], candidate) end end +# Dispatch for multi-candidate chunks +function Base.getindex(ci::ChunkIndex, + candidate::CandidateChunks{TP, TD}, + field::Symbol = :chunks) where {TP <: AbstractCandidateChunks, TD <: Real} + @assert field==:chunks "Only `chunks` field is supported for now" + + index_pos = findfirst(x -> x.index_id == ci.id, candidate.positions) + @info index_pos + if isnothing(index_pos) + eltype(chunks(ci))[] + else + getindex(chunks(ci), candidate.positions[index_pos].positions) + end +end +function Base.getindex(mi::MultiIndex, + candidate::CandidateChunks{TP, TD}, + field::Symbol = :chunks) where {TP <: AbstractCandidateChunks, TD <: Real} + @assert field==:chunks "Only `chunks` field is supported for now" + mapreduce(idxs -> Base.getindex(idxs, candidate, field), vcat, indexes(mi)) +end """ RAGContext diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index c48c28257..51c47ddd3 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -25,6 +25,7 @@ const RESERVED_KWARGS = [ :model, ] +# export replace_words, split_by_length, call_cost, auth_header # for debugging only include("utils.jl") export aigenerate, aiembed, aiclassify, aiextract, aiscan diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 22f597eab..fe1be50c4 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -13,6 +13,7 @@ Check your preferences by calling `get_preferences(key::String)`. # Available Preferences (for `set_preferences!`) - `OPENAI_API_KEY`: The API key for the OpenAI API. See [OpenAI's documentation](https://platform.openai.com/docs/quickstart?context=python) for more information. - `MISTRALAI_API_KEY`: The API key for the Mistral AI API. See [Mistral AI's documentation](https://docs.mistral.ai/) for more information. +- `COHERE_API_KEY`: The API key for the Cohere API. See [Cohere's documentation](https://docs.cohere.com/docs/the-cohere-platform) for more information. - `MODEL_CHAT`: The default model to use for aigenerate and most ai* calls. See `MODEL_REGISTRY` for a list of available models or define your own. - `MODEL_EMBEDDING`: The default model to use for aiembed (embedding documents). See `MODEL_REGISTRY` for a list of available models or define your own. - `PROMPT_SCHEMA`: The default prompt schema to use for aigenerate and most ai* calls (if not specified in `MODEL_REGISTRY`). Set as a string, eg, `"OpenAISchema"`. @@ -30,6 +31,7 @@ Define your `register_model!()` calls in your `startup.jl` file to make them ava # Available ENV Variables - `OPENAI_API_KEY`: The API key for the OpenAI API. - `MISTRALAI_API_KEY`: The API key for the Mistral AI API. +- `COHERE_API_KEY`: The API key for the Cohere API. - `LOCAL_SERVER`: The URL of the local server to use for `ai*` calls. Defaults to `http://localhost:10897/v1`. This server is called when you call `model="local"` Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. @@ -56,6 +58,7 @@ function set_preferences!(pairs::Pair{String, <:Any}...) allowed_preferences = [ "MISTRALAI_API_KEY", "OPENAI_API_KEY", + "COHERE_API_KEY", "MODEL_CHAT", "MODEL_EMBEDDING", "MODEL_ALIASES", @@ -91,6 +94,7 @@ function get_preferences(key::String) allowed_preferences = [ "MISTRALAI_API_KEY", "OPENAI_API_KEY", + "COHERE_API_KEY", "MODEL_CHAT", "MODEL_EMBEDDING", "MODEL_ALIASES", @@ -119,6 +123,9 @@ isempty(OPENAI_API_KEY) && const MISTRALAI_API_KEY::String = @load_preference("MISTRALAI_API_KEY", default=get(ENV, "MISTRALAI_API_KEY", "")); +const COHERE_API_KEY::String = @load_preference("COHERE_API_KEY", + default=get(ENV, "COHERE_API_KEY", "")); + ## Address of the local server const LOCAL_SERVER::String = @load_preference("LOCAL_SERVER", default=get(ENV, "LOCAL_SERVER", "http://127.0.0.1:10897/v1")); diff --git a/src/utils.jl b/src/utils.jl index 3b8cacc9f..d7a145a73 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -359,4 +359,18 @@ macro timeout(seconds, expr_to_run, expr_when_fails) end "Utility for rendering the conversation (vector of messages) as markdown. REQUIRES the Markdown package to load the extension!" -function preview end \ No newline at end of file +function preview end + +""" + auth_header(api_key::String) + +Builds an authorization header for API calls with the given API key. +""" +function auth_header(api_key::String) + isempty(api_key) && throw(ArgumentError("api_key cannot be empty")) + [ + "Authorization" => "Bearer $api_key", + "Content-Type" => "application/json", + "Accept" => "application/json", + ] +end \ No newline at end of file diff --git a/test/Experimental/RAGTools/evaluation.jl b/test/Experimental/RAGTools/evaluation.jl index f9ea295e8..db8ed3be3 100644 --- a/test/Experimental/RAGTools/evaluation.jl +++ b/test/Experimental/RAGTools/evaluation.jl @@ -75,7 +75,7 @@ end @testset "build_qa_evals" begin # test with a mock server - PORT = rand(1000:2000) + PORT = rand(9000:11000) PT.register_model!(; name = "mock-emb", schema = PT.CustomOpenAISchema()) PT.register_model!(; name = "mock-meta", schema = PT.CustomOpenAISchema()) PT.register_model!(; name = "mock-gen", schema = PT.CustomOpenAISchema()) diff --git a/test/Experimental/RAGTools/preparation.jl b/test/Experimental/RAGTools/preparation.jl index 3e8396fbc..1a20749dd 100644 --- a/test/Experimental/RAGTools/preparation.jl +++ b/test/Experimental/RAGTools/preparation.jl @@ -72,7 +72,7 @@ end @testset "build_index" begin # test with a mock server - PORT = rand(1000:2000) + PORT = rand(9000:11000) PT.register_model!(; name = "mock-emb", schema = PT.CustomOpenAISchema()) PT.register_model!(; name = "mock-meta", schema = PT.CustomOpenAISchema()) PT.register_model!(; name = "mock-get", schema = PT.CustomOpenAISchema()) @@ -123,6 +123,20 @@ end @test index.tags == ones(8, 1) @test index.tags_vocab == ["category:::yes"] + ## Test docs reader + index = build_index([text, text]; reader = :docs, sources = ["x", "x"], max_length = 10, + extract_metadata = true, + model_embedding = "mock-emb", + model_metadata = "mock-meta", api_kwargs = (; url = "http://localhost:$(PORT)")) + @test index.embeddings == hcat(fill(normalize(ones(Float32, 128)), 8)...) + @test index.chunks[1:4] == index.chunks[5:8] + @test index.sources == fill("x", 8) + @test index.tags == ones(8, 1) + @test index.tags_vocab == ["category:::yes"] + + # Assertion if sources is missing + @test_throws AssertionError build_index([text, text]; reader = :docs) + # clean up close(echo_server) end \ No newline at end of file diff --git a/test/Experimental/RAGTools/retrieval.jl b/test/Experimental/RAGTools/retrieval.jl index fcb9bc819..296904f88 100644 --- a/test/Experimental/RAGTools/retrieval.jl +++ b/test/Experimental/RAGTools/retrieval.jl @@ -1,5 +1,5 @@ using PromptingTools.Experimental.RAGTools: find_closest, find_tags -using PromptingTools.Experimental.RAGTools: Passthrough, rerank +using PromptingTools.Experimental.RAGTools: Passthrough, rerank, CohereRerank @testset "find_closest" begin test_embeddings = [1.0 2.0 -1.0; 3.0 4.0 -3.0; 5.0 6.0 -6.0] |> @@ -22,6 +22,34 @@ using PromptingTools.Experimental.RAGTools: Passthrough, rerank # Test behavior with edge values (top_k == 0) @test find_closest(test_embeddings, query_embedding, top_k = 0) == ([], []) + + ## Test with ChunkIndex + embeddings1 = ones(Float32, 2, 2) + embeddings1[2, 2] = 5.0 + embeddings1 = mapreduce(normalize, hcat, eachcol(embeddings1)) + ci1 = ChunkIndex(id = :TestChunkIndex1, + chunks = ["chunk1", "chunk2"], + sources = ["source1", "source2"], + embeddings = embeddings1) + ci2 = ChunkIndex(id = :TestChunkIndex2, + chunks = ["chunk1", "chunk2"], + sources = ["source1", "source2"], + embeddings = ones(Float32, 2, 2)) + mi = MultiIndex(id = :multi, indexes = [ci1, ci2]) + + ## find_closest with ChunkIndex + query_emb = [0.5, 0.5] # Example query embedding vector + result = find_closest(ci1, query_emb) + @test result isa CandidateChunks + @test result.positions == [1, 2] + @test all(1.0 .>= result.distances .>= -1.0) # Assuming default minimum_similarity + + ## find_closest with MultiIndex + ## query_emb = [0.5, 0.5] # Example query embedding vector + ## result = find_closest(mi, query_emb) + ## @test result isa CandidateChunks + ## @test result.positions == [1, 2] + ## @test all(1.0 .>= result.distances .>= -1.0) # Assuming default minimum_similarity end @testset "find_tags" begin @@ -60,5 +88,34 @@ end # Passthrough Strategy strategy = Passthrough() - @test rerank(strategy, index, question, candidate_chunks) === candidate_chunks + @test rerank(strategy, index, question, candidate_chunks) == + candidate_chunks + + # Cohere assertion + ci1 = ChunkIndex(id = :TestChunkIndex1, + chunks = ["chunk1", "chunk2"], + sources = ["source1", "source2"]) + ci2 = ChunkIndex(id = :TestChunkIndex2, + chunks = ["chunk1", "chunk2"], + sources = ["source1", "source2"]) + mi = MultiIndex(; id = :multi, indexes = [ci1, ci2]) + @test_throws ArgumentError rerank(CohereRerank(), + mi, + question, + candidate_chunks) + + # Bad top_n + @test_throws AssertionError rerank(CohereRerank(), + ci1, + question, + candidate_chunks; top_n = 0) + + # Bad index_id + cc2 = CandidateChunks(index_id = :TestChunkIndex2, + positions = [1, 2], + distances = [0.3, 0.4]) + @test_throws AssertionError rerank(CohereRerank(), + ci1, + question, + cc2; top_n = 1) end \ No newline at end of file diff --git a/test/Experimental/RAGTools/types.jl b/test/Experimental/RAGTools/types.jl index bfb915919..538a0669e 100644 --- a/test/Experimental/RAGTools/types.jl +++ b/test/Experimental/RAGTools/types.jl @@ -92,6 +92,29 @@ end mi1 = MultiIndex(indexes = [cin1]) mi2 = MultiIndex(indexes = [cin2]) @test mi1 != mi2 + + ## not implemented + @test_throws ArgumentError vcat(mi1, mi2) +end + +@testset "CandidateChunks" begin + chunk_sym = Symbol("TestChunkIndex") + cc1 = CandidateChunks(index_id = chunk_sym, + positions = [1, 3], + distances = [0.1, 0.2]) + @test Base.length(cc1) == 2 + + # Test intersection & + cc2 = CandidateChunks(index_id = chunk_sym, + positions = [2, 4], + distances = [0.3, 0.4]) + @test isempty((cc1 & cc2).positions) + cc3 = CandidateChunks(index_id = chunk_sym, + positions = [1, 4], + distances = [0.3, 0.4]) + joint = (cc1 & cc3) + @test joint.positions == [1] + @test joint.distances == [0.2] end @testset "getindex with CandidateChunks" begin @@ -113,12 +136,21 @@ end positions = [1, 3], distances = [0.1, 0.2]) @test collect(test_chunk_index[candidate_chunks]) == ["First chunk", "Third chunk"] + @test collect(test_chunk_index[candidate_chunks, :chunks]) == + ["First chunk", "Third chunk"] + @test collect(test_chunk_index[candidate_chunks, :sources]) == + ["test_source", "test_source"] + @test collect(test_chunk_index[candidate_chunks, :embeddings]) == + embeddings_data[:, [1, 3]] # Test with empty positions, which should result in an empty array candidate_chunks_empty = CandidateChunks(index_id = chunk_sym, positions = Int[], distances = Float32[]) @test isempty(test_chunk_index[candidate_chunks_empty]) + @test isempty(test_chunk_index[candidate_chunks_empty, :chunks]) + @test isempty(test_chunk_index[candidate_chunks_empty, :embeddings]) + @test isempty(test_chunk_index[candidate_chunks_empty, :sources]) # Test with positions out of bounds, should handle gracefully without errors candidate_chunks_oob = CandidateChunks(index_id = chunk_sym, @@ -151,4 +183,25 @@ end # Test error case when trying to use a non-chunks field, should assert error as only :chunks field is supported @test_throws AssertionError test_chunk_index[candidate_chunks, :nonexistent_field] -end \ No newline at end of file + + # Multi-Candidate CandidateChunks + cc1 = CandidateChunks(index_id = :TestChunkIndex1, + positions = [1, 2], + distances = [0.3, 0.4]) + cc2 = CandidateChunks(index_id = :TestChunkIndex2, + positions = [2], + distances = [0.1]) + cc = CandidateChunks(; index_id = :multi, positions = [cc1, cc2], distances = zeros(2)) + ci1 = ChunkIndex(id = :TestChunkIndex1, + chunks = ["chunk1", "chunk2"], + sources = ["source1", "source2"]) + ci2 = ChunkIndex(id = :TestChunkIndex2, + chunks = ["chunk1", "chunk2"], + sources = ["source1", "source2"]) + @test ci1[cc] == ["chunk1", "chunk2"] + @test ci2[cc] == ["chunk2"] + + # with MultiIndex + mi = MultiIndex(; id = :multi, indexes = [ci1, ci2]) + @test mi[cc] == ["chunk1", "chunk2", "chunk2"] +end diff --git a/test/utils.jl b/test/utils.jl index 04a452d71..789fff4a3 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -2,7 +2,8 @@ using PromptingTools: split_by_length, replace_words using PromptingTools: _extract_handlebar_variables, call_cost, _report_stats using PromptingTools: _string_to_vector, _encode_local_image using PromptingTools: DataMessage, AIMessage -using PromptingTools: push_conversation!, resize_conversation!, @timeout, preview +using PromptingTools: push_conversation!, + resize_conversation!, @timeout, preview, auth_header @testset "replace_words" begin words = ["Disney", "Snow White", "Mickey Mouse"] @@ -226,3 +227,13 @@ end expected_output = Markdown.parse("# System Message\n\nWelcome\n\n---\n\n# User Message\n\nHello\n\n---\n\n# AI Message\n\nWorld\n\n---\n\n# Data Message\n\nData: Vector{Float64} (Size: (10,))\n") @test preview_output == expected_output end + +@testset "auth_header" begin + headers = auth_header("") + @test headers == [ + "Authorization" => "Bearer ", + "Content-Type" => "application/json", + "Accept" => "application/json", + ] + @test_throws ArgumentError auth_header("") +end \ No newline at end of file From 6f98096f6b2bbabd21d81b9061d61b09b25eb159 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 25 Jan 2024 20:06:43 +0000 Subject: [PATCH 102/251] OpenAI model changes --- CHANGELOG.md | 14 ++++++++ Project.toml | 2 +- src/Experimental/RAGTools/preparation.jl | 4 +-- src/user_preferences.jl | 43 +++++++++++++++++++----- src/utils.jl | 2 +- 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8d7dd0db..f463129fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [0.10.0] + +### Added +- [BREAKING CHANGE] The default embedding model (`MODEL_EMBEDDING`) changes to "text-embedding-3-small" effectively immediately (lower cost, higher performance). The default chat model (`MODEL_CHAT`) will be changed by OpenAI to 0125 (from 0613) by mid-February. If you have older embeddings or rely on the exact chat model version, please set the model explicitly in your code or in your preferences. +- New OpenAI models added to the model registry (see the [release notes](https://openai.com/blog/new-embedding-models-and-api-updates)). + - "gpt4t" refers to whichever is the latest GPT-4 Turbo model ("gpt-4-0125-preview" at the time of writing) + - "gpt3t" refers to the latest GPT-3.5 Turbo model version 0125, which is 25-50% cheaper and has an updated knowledge (available from February 2024) + - "gpt3" still refers to the general endpoint "gpt-3.5-turbo", which OpenAI will move to version 0125 by mid-February (ie, "gpt3t" will be the same as "gpt3" then. We have reflected the approximate cost in the model registry but note that it will be incorrect in the transition period) + - "emb3small" refers to the small version of the new embedding model (dim=1536), which is 5x cheaper than Ada and promises higher quality + - "emb3large" refers to the large version of the new embedding model (dim=3072), which is only 30% more expensive than Ada + +### Fixed +- Fixed typos in the documentation + ## [0.9.0] ### Added diff --git a/Project.toml b/Project.toml index 7e7f82e32..3a5093961 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.9.0" +version = "0.10.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/Experimental/RAGTools/preparation.jl b/src/Experimental/RAGTools/preparation.jl index 2b80a9e0f..924b58b36 100644 --- a/src/Experimental/RAGTools/preparation.jl +++ b/src/Experimental/RAGTools/preparation.jl @@ -41,7 +41,7 @@ function _normalize end get_chunks(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, sources::Vector{<:AbstractString} = files_or_docs, verbose::Bool = true, - separators = ["\n\n", ". ", "\n"], max_length::Int = 256) + separators = ["\\n\\n", ". ", "\\n"], max_length::Int = 256) Chunks the provided `files_or_docs` into chunks of maximum length `max_length` (if possible with provided `separators`). @@ -52,7 +52,7 @@ Supports two modes of operation: # Arguments - `files_or_docs`: A vector of valid file paths OR string documents to be chunked. - `reader`: A symbol indicating the type of input, can be either `:files` or `:docs`. Default is `:files`. -- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\n\n", ". ", "\n"]`. +- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\\n\\n", ". ", "\\n"]`. - `max_length`: The maximum length of each chunk (if possible with provided separators). Default is 256. - `sources`: A vector of strings indicating the source of each chunk. Default is equal to `files_or_docs` (for `reader=:files`) diff --git a/src/user_preferences.jl b/src/user_preferences.jl index fe1be50c4..4142f0454 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -109,7 +109,7 @@ end ## Load up GLOBALS const MODEL_CHAT::String = @load_preference("MODEL_CHAT", default="gpt-3.5-turbo") const MODEL_EMBEDDING::String = @load_preference("MODEL_EMBEDDING", - default="text-embedding-ada-002") + default="text-embedding-3-small") # the prompt schema default is defined in llm_interace.jl ! # const PROMPT_SCHEMA = OpenAISchema() @@ -246,9 +246,11 @@ end aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", "gpt4" => "gpt-4", "gpt4v" => "gpt-4-vision-preview", # 4v is for "4 vision" - "gpt4t" => "gpt-4-1106-preview", # 4t is for "4 turbo" - "gpt3t" => "gpt-3.5-turbo-1106", # 3t is for "3 turbo" + "gpt4t" => "gpt-4-turbo-preview", # 4t is for "4 turbo" + "gpt3t" => "gpt-3.5-turbo-0125", # 3t is for "3 turbo" "ada" => "text-embedding-ada-002", + "emb3small" => "text-embedding-3-small", + "emb3large" => "text-embedding-3-large", "yi34c" => "yi:34b-chat", "oh25" => "openhermes2.5-mistral", "starling" => "starling-lm", @@ -258,14 +260,19 @@ aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", OpenAISchema(), + 0.5e-6, 1.5e-6, - 2e-6, - "GPT-3.5 Turbo is a 175B parameter model and a common default on the OpenAI API."), + "GPT-3.5 Turbo is a 175B parameter model and a common default on the OpenAI API. From mid-Feb 2024, it will be using the new GPT-3.5 Turbo 0125 version (pricing is set assuming the 0125 version)."), "gpt-3.5-turbo-1106" => ModelSpec("gpt-3.5-turbo-1106", OpenAISchema(), 1e-6, 2e-6, - "GPT-3.5 Turbo is the latest version of GPT3.5 and the cheapest to use."), + "GPT-3.5 Turbo is an updated version of GPT3.5 that is much faster and cheaper to use. 1106 refers to the release date of November 6, 2023."), + "gpt-3.5-turbo-0125" => ModelSpec("gpt-3.5-turbo-0125", + OpenAISchema(), + 0.5e-6, + 1.5e-6, + "GPT-3.5 Turbo is an updated version of GPT3.5 that is much faster and cheaper to use. This is the cheapest GPT-3.5 Turbo model. 0125 refers to the release date of January 25, 2024."), "gpt-4" => ModelSpec("gpt-4", OpenAISchema(), 3e-5, @@ -275,7 +282,17 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", OpenAISchema(), 1e-5, 3e-5, - "GPT-4 Turbo is the latest version of GPT4 that is much faster and the cheapest to use."), + "GPT-4 Turbo 1106 is an updated version of GPT4 that is much faster and the cheaper to use. 1106 refers to the release date of November 6, 2023."), + "gpt-4-0125-preview" => ModelSpec("gpt-4-0125-preview", + OpenAISchema(), + 1e-5, + 3e-5, + "GPT-4 Turbo is an updated version of GPT4 that is much faster and the cheaper to use. 0125 refers to the release date of January 25, 2024."), + "gpt-4-turbo-preview" => ModelSpec("gpt-4-turbo-preview", + OpenAISchema(), + 1e-5, + 3e-5, + "GPT-4 Turbo is an updated version of GPT4 that is much faster and the cheaper to use. This is the general name for whatever is the latest GPT4 Turbo preview release. Right now it is 0125."), "gpt-4-vision-preview" => ModelSpec("gpt-4-vision-preview", OpenAISchema(), 1e-5, @@ -285,7 +302,17 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", OpenAISchema(), 1e-7, 0.0, - "Text Embedding Ada is a 1.75T parameter model and the largest model available on the OpenAI API."), + "Classic text embedding endpoint Ada from 2022 with 1536 dimensions."), + "text-embedding-3-small" => ModelSpec("text-embedding-3-small", + OpenAISchema(), + 0.2e-7, + 0.0, + "New text embedding endpoint with 1536 dimensions, but 5x cheaper than Ada and more performant."), + "text-embedding-3-large" => ModelSpec("text-embedding-3-large", + OpenAISchema(), + 1.3e-7, + 0.0, + "New text embedding endpoint with 3072 dimensions, c. 30% more expensive than Ada but more performant."), "llama2" => ModelSpec("llama2", OllamaSchema(), 0.0, diff --git a/src/utils.jl b/src/utils.jl index d7a145a73..c7dcd83f6 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -55,7 +55,7 @@ This is particularly useful for splitting larger documents or texts into smaller Splitting text with the default separator (" "): ```julia text = "Hello world. How are you?" -chunks = splitbysize(text; max_length=13) +chunks = split_by_length(text; max_length=13) length(chunks) # Output: 2 ``` From 01476622c2d5698dfeb29d1d6e4951e28958b49c Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 25 Jan 2024 20:07:41 +0000 Subject: [PATCH 103/251] Model change gpt3t --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f463129fb..f86d4fcb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [BREAKING CHANGE] The default embedding model (`MODEL_EMBEDDING`) changes to "text-embedding-3-small" effectively immediately (lower cost, higher performance). The default chat model (`MODEL_CHAT`) will be changed by OpenAI to 0125 (from 0613) by mid-February. If you have older embeddings or rely on the exact chat model version, please set the model explicitly in your code or in your preferences. - New OpenAI models added to the model registry (see the [release notes](https://openai.com/blog/new-embedding-models-and-api-updates)). - "gpt4t" refers to whichever is the latest GPT-4 Turbo model ("gpt-4-0125-preview" at the time of writing) - - "gpt3t" refers to the latest GPT-3.5 Turbo model version 0125, which is 25-50% cheaper and has an updated knowledge (available from February 2024) + - "gpt3t" refers to the latest GPT-3.5 Turbo model version 0125, which is 25-50% cheaper and has an updated knowledge (available from February 2024, you will get an error in the interim) - "gpt3" still refers to the general endpoint "gpt-3.5-turbo", which OpenAI will move to version 0125 by mid-February (ie, "gpt3t" will be the same as "gpt3" then. We have reflected the approximate cost in the model registry but note that it will be incorrect in the transition period) - "emb3small" refers to the small version of the new embedding model (dim=1536), which is 5x cheaper than Ada and promises higher quality - "emb3large" refers to the large version of the new embedding model (dim=3072), which is only 30% more expensive than Ada From 252dc8e00ae5dcff25a2cc1a98bc9c6b9a076a54 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 25 Jan 2024 20:32:02 +0000 Subject: [PATCH 104/251] Fix tests on updated OpenAI model costs --- CHANGELOG.md | 2 +- test/llm_openai.jl | 5 +++-- test/utils.jl | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f86d4fcb8..68f0bab23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [BREAKING CHANGE] The default embedding model (`MODEL_EMBEDDING`) changes to "text-embedding-3-small" effectively immediately (lower cost, higher performance). The default chat model (`MODEL_CHAT`) will be changed by OpenAI to 0125 (from 0613) by mid-February. If you have older embeddings or rely on the exact chat model version, please set the model explicitly in your code or in your preferences. - New OpenAI models added to the model registry (see the [release notes](https://openai.com/blog/new-embedding-models-and-api-updates)). - "gpt4t" refers to whichever is the latest GPT-4 Turbo model ("gpt-4-0125-preview" at the time of writing) - - "gpt3t" refers to the latest GPT-3.5 Turbo model version 0125, which is 25-50% cheaper and has an updated knowledge (available from February 2024, you will get an error in the interim) + - "gpt3t" refers to the latest GPT-3.5 Turbo model version 0125, which is 25-50% cheaper and has updated knowledge (available from February 2024, you will get an error in the interim) - "gpt3" still refers to the general endpoint "gpt-3.5-turbo", which OpenAI will move to version 0125 by mid-February (ie, "gpt3t" will be the same as "gpt3" then. We have reflected the approximate cost in the model registry but note that it will be incorrect in the transition period) - "emb3small" refers to the small version of the new embedding model (dim=1536), which is 5x cheaper than Ada and promises higher quality - "emb3large" refers to the large version of the new embedding model (dim=3072), which is only 30% more expensive than Ada diff --git a/test/llm_openai.jl b/test/llm_openai.jl index a6d1ec870..a7ee5e0b4 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -1,7 +1,8 @@ using PromptingTools: TestEchoOpenAISchema, render, OpenAISchema using PromptingTools: AIMessage, SystemMessage, AbstractMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage -using PromptingTools: CustomProvider, CustomOpenAISchema, MistralOpenAISchema +using PromptingTools: CustomProvider, + CustomOpenAISchema, MistralOpenAISchema, MODEL_EMBEDDING @testset "render-OpenAI" begin schema = OpenAISchema() @@ -284,7 +285,7 @@ end elapsed = msg.elapsed) @test msg == expected_output @test schema1.inputs == "Hello World" - @test schema1.model_id == "text-embedding-ada-002" + @test schema1.model_id == MODEL_EMBEDDING # Test different input combinations and multiple strings response2 = Dict(:data => [Dict(:embedding => ones(128, 2))], diff --git a/test/utils.jl b/test/utils.jl index 789fff4a3..31d43fd42 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -113,12 +113,12 @@ end msg = AIMessage(; content = "", tokens = (1000, 2000)) cost = call_cost(msg, "unknown_model") @test cost == 0.0 - @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 1.5e-6 + 2e-6 * 2000 + @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 0.5e-6 + 1.5e-6 * 2000 msg = DataMessage(; content = nothing, tokens = (1000, 1000)) cost = call_cost(msg, "unknown_model") @test cost == 0.0 - @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 1.5e-6 + 2e-6 * 1000 + @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 0.5e-6 + 1.5e-6 * 1000 @test call_cost(msg, "gpt-3.5-turbo"; @@ -135,7 +135,7 @@ end # Returns a string with a cost msg = AIMessage(; content = "", tokens = (1000, 5000), elapsed = 5.0) - expected_output = "Tokens: 6000 @ Cost: \$0.0115 in 5.0 seconds" + expected_output = "Tokens: 6000 @ Cost: \$0.008 in 5.0 seconds" @test _report_stats(msg, "gpt-3.5-turbo") == expected_output end From 9794d4617a0a9d5dc1a0a72a50e46b3f50d72fbc Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 25 Jan 2024 20:37:56 +0000 Subject: [PATCH 105/251] Re-apply format --- Makefile | 12 ++++++++++++ docs/generate_examples.jl | 2 +- examples/building_RAG.jl | 2 +- examples/working_with_ollama.jl | 2 +- ext/MarkdownPromptingToolsExt.jl | 2 +- ext/RAGToolsExperimentalExt.jl | 4 ++-- src/Experimental/AgentTools/AgentTools.jl | 2 +- src/Experimental/AgentTools/code_feedback.jl | 2 +- src/Experimental/AgentTools/lazy_types.jl | 2 +- src/Experimental/Experimental.jl | 2 +- src/Experimental/RAGTools/RAGTools.jl | 2 +- src/Experimental/RAGTools/api_services.jl | 2 +- src/Experimental/RAGTools/evaluation.jl | 2 +- src/Experimental/RAGTools/generation.jl | 2 +- src/Experimental/RAGTools/retrieval.jl | 2 +- src/Experimental/RAGTools/utils.jl | 2 +- src/code_generation.jl | 2 +- src/extraction.jl | 2 +- src/llm_interface.jl | 2 +- src/llm_ollama_managed.jl | 2 +- src/llm_openai.jl | 2 +- src/templates.jl | 2 +- src/utils.jl | 2 +- test/Experimental/AgentTools/code_feedback.jl | 2 +- test/Experimental/AgentTools/lazy_types.jl | 2 +- test/Experimental/AgentTools/utils.jl | 2 +- test/Experimental/RAGTools/evaluation.jl | 2 +- test/Experimental/RAGTools/preparation.jl | 2 +- test/Experimental/RAGTools/retrieval.jl | 2 +- test/Experimental/RAGTools/utils.jl | 2 +- test/code_generation.jl | 2 +- test/extraction.jl | 2 +- test/llm_interface.jl | 2 +- test/llm_ollama.jl | 2 +- test/llm_ollama_managed.jl | 2 +- test/llm_shared.jl | 2 +- test/macros.jl | 2 +- test/messages.jl | 2 +- test/serialization.jl | 2 +- test/templates.jl | 2 +- test/utils.jl | 2 +- 41 files changed, 53 insertions(+), 41 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..90460fa12 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +# This is not a true makefile, just a collection of convenient scripts + +default: help + +format: + # assumes you have JuliaFormatter installed in your global env / somewhere on LOAD_PATH + julia -e 'using JuliaFormatter; format(".")' + + +help: + echo "make help - show this help" + echo "make format - format the code" \ No newline at end of file diff --git a/docs/generate_examples.jl b/docs/generate_examples.jl index f71187d5b..0b373e847 100644 --- a/docs/generate_examples.jl +++ b/docs/generate_examples.jl @@ -10,4 +10,4 @@ for fn in example_files Literate.markdown(fn, output_dir; execute = true) end -# TODO: change meta fields at the top of each file! \ No newline at end of file +# TODO: change meta fields at the top of each file! diff --git a/examples/building_RAG.jl b/examples/building_RAG.jl index fa55c307a..7532f5780 100644 --- a/examples/building_RAG.jl +++ b/examples/building_RAG.jl @@ -144,4 +144,4 @@ first(df, 5) # - Add re-ranking of context (see `rerank` function, you can use Cohere ReRank API) # - Improve the question embedding (eg, rephrase it, generate hypothetical answers and use them to find better context) # -# ... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) \ No newline at end of file +# ... and much more! See some ideas in [Anyscale RAG tutorial](https://www.anyscale.com/blog/a-comprehensive-guide-for-building-rag-based-llm-applications-part-1) diff --git a/examples/working_with_ollama.jl b/examples/working_with_ollama.jl index 78b23f67c..64186779d 100644 --- a/examples/working_with_ollama.jl +++ b/examples/working_with_ollama.jl @@ -91,4 +91,4 @@ msg = aiembed(schema, model = "openhermes2.5-mistral") # Cosine similarity is then a simple multiplication -msg.content' * msg.content[:, 1] \ No newline at end of file +msg.content' * msg.content[:, 1] diff --git a/ext/MarkdownPromptingToolsExt.jl b/ext/MarkdownPromptingToolsExt.jl index 39a50a021..a029b502e 100644 --- a/ext/MarkdownPromptingToolsExt.jl +++ b/ext/MarkdownPromptingToolsExt.jl @@ -92,4 +92,4 @@ function PT.preview(conversation::AbstractVector{<:PT.AbstractMessage}) String(take!(io)) |> Markdown.parse end -end # end of module \ No newline at end of file +end # end of module diff --git a/ext/RAGToolsExperimentalExt.jl b/ext/RAGToolsExperimentalExt.jl index 7047853c0..9b095a446 100644 --- a/ext/RAGToolsExperimentalExt.jl +++ b/ext/RAGToolsExperimentalExt.jl @@ -12,7 +12,7 @@ PromptingTools.Experimental.RAGTools._normalize(arr::AbstractArray) = normalize( # "Builds a sparse matrix of tags and a vocabulary from the given vector of chunk metadata. Requires SparseArrays.jl to be loaded." function PromptingTools.Experimental.RAGTools.build_tags(chunk_metadata::Vector{ Vector{String}, - }) +}) tags_vocab_ = vcat(chunk_metadata...) |> unique |> sort tags_vocab_index = Dict{String, Int}(t => i for (i, t) in enumerate(tags_vocab_)) Is, Js = Int[], Int[] @@ -31,4 +31,4 @@ function PromptingTools.Experimental.RAGTools.build_tags(chunk_metadata::Vector{ return tags_, tags_vocab_ end -end \ No newline at end of file +end diff --git a/src/Experimental/AgentTools/AgentTools.jl b/src/Experimental/AgentTools/AgentTools.jl index 1ad44c6c2..4c22c2546 100644 --- a/src/Experimental/AgentTools/AgentTools.jl +++ b/src/Experimental/AgentTools/AgentTools.jl @@ -19,4 +19,4 @@ export AICall, AIGenerate, AIExtract, AIEmbed, AIClassify, AIScan export AICodeFixer, run! include("lazy_types.jl") -end \ No newline at end of file +end diff --git a/src/Experimental/AgentTools/code_feedback.jl b/src/Experimental/AgentTools/code_feedback.jl index 8fec67677..a4283ed41 100644 --- a/src/Experimental/AgentTools/code_feedback.jl +++ b/src/Experimental/AgentTools/code_feedback.jl @@ -122,4 +122,4 @@ function aicodefixer_feedback(::CodeFailedEval, end return isempty(feedback) ? "No feedback provided." : join(feedback, "\n\n") -end \ No newline at end of file +end diff --git a/src/Experimental/AgentTools/lazy_types.jl b/src/Experimental/AgentTools/lazy_types.jl index 32f39074f..2984ae1ac 100644 --- a/src/Experimental/AgentTools/lazy_types.jl +++ b/src/Experimental/AgentTools/lazy_types.jl @@ -404,4 +404,4 @@ function run!(codefixer::AICodeFixer; codefixer.call = call return codefixer -end \ No newline at end of file +end diff --git a/src/Experimental/Experimental.jl b/src/Experimental/Experimental.jl index eaa0d8938..408e2091d 100644 --- a/src/Experimental/Experimental.jl +++ b/src/Experimental/Experimental.jl @@ -16,4 +16,4 @@ include("RAGTools/RAGTools.jl") export AgentTools include("AgentTools/AgentTools.jl") -end # module Experimental \ No newline at end of file +end # module Experimental diff --git a/src/Experimental/RAGTools/RAGTools.jl b/src/Experimental/RAGTools/RAGTools.jl index 7f491fd36..01788b686 100644 --- a/src/Experimental/RAGTools/RAGTools.jl +++ b/src/Experimental/RAGTools/RAGTools.jl @@ -33,4 +33,4 @@ include("generation.jl") export build_qa_evals, run_qa_evals include("evaluation.jl") -end \ No newline at end of file +end diff --git a/src/Experimental/RAGTools/api_services.jl b/src/Experimental/RAGTools/api_services.jl index adc7ac824..866fb1635 100644 --- a/src/Experimental/RAGTools/api_services.jl +++ b/src/Experimental/RAGTools/api_services.jl @@ -33,4 +33,4 @@ function cohere_api(; JSON3.write(input_body); http_kwargs...) body = JSON3.read(resp.body) return (; response = body) -end \ No newline at end of file +end diff --git a/src/Experimental/RAGTools/evaluation.jl b/src/Experimental/RAGTools/evaluation.jl index 1f0da8329..4c02d372e 100644 --- a/src/Experimental/RAGTools/evaluation.jl +++ b/src/Experimental/RAGTools/evaluation.jl @@ -282,4 +282,4 @@ function run_qa_evals(index::AbstractChunkIndex, qa_items::AbstractVector{<:QAEv verbose && @info "QA Evaluations complete ($((success_count)/length(qa_items)) evals successful)!" return results -end \ No newline at end of file +end diff --git a/src/Experimental/RAGTools/generation.jl b/src/Experimental/RAGTools/generation.jl index d9725f6bb..712acf07f 100644 --- a/src/Experimental/RAGTools/generation.jl +++ b/src/Experimental/RAGTools/generation.jl @@ -181,4 +181,4 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC else return msg end -end \ No newline at end of file +end diff --git a/src/Experimental/RAGTools/retrieval.jl b/src/Experimental/RAGTools/retrieval.jl index d7c998731..06ef38c5d 100644 --- a/src/Experimental/RAGTools/retrieval.jl +++ b/src/Experimental/RAGTools/retrieval.jl @@ -162,4 +162,4 @@ function rerank(strategy::CohereRerank, index::AbstractChunkIndex, question, verbose && @info "Reranking done. $search_units_str" return CandidateChunks(index.id, positions, distances) -end \ No newline at end of file +end diff --git a/src/Experimental/RAGTools/utils.jl b/src/Experimental/RAGTools/utils.jl index f980a61e0..6eba9954e 100644 --- a/src/Experimental/RAGTools/utils.jl +++ b/src/Experimental/RAGTools/utils.jl @@ -20,4 +20,4 @@ function merge_labeled_matrices(mat1::AbstractMatrix{T1}, aligned_mat2 = aligned_mat2 |> Base.Splat(hcat) return vcat(aligned_mat1, aligned_mat2), combined_vocab -end \ No newline at end of file +end diff --git a/src/code_generation.jl b/src/code_generation.jl index 258f805a3..b85a1aeec 100644 --- a/src/code_generation.jl +++ b/src/code_generation.jl @@ -777,4 +777,4 @@ function eval!(cb::AbstractCodeBlock, expr::Expr; end end return cb -end \ No newline at end of file +end diff --git a/src/extraction.jl b/src/extraction.jl index 1c45acf9a..72dba37bd 100644 --- a/src/extraction.jl +++ b/src/extraction.jl @@ -187,4 +187,4 @@ Extract zero, one or more specified items from the provided data. """ struct ItemsExtract{T <: Any} items::Vector{T} -end \ No newline at end of file +end diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 43657b037..f47237d01 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -225,4 +225,4 @@ end function aiscan(prompt; model = MODEL_CHAT, kwargs...) schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema aiscan(schema, prompt; model, kwargs...) -end \ No newline at end of file +end diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index a8918feea..23286c3f6 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -375,4 +375,4 @@ end function aiscan(prompt_schema::AbstractManagedSchema, prompt::ALLOWED_PROMPT_TYPE; kwargs...) error("Managed schema does not support aiscan. Please use OpenAISchema instead.") -end \ No newline at end of file +end diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 59d6a60e6..369330904 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -814,4 +814,4 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE kwargs...) return output -end \ No newline at end of file +end diff --git a/src/templates.jl b/src/templates.jl index bfcdd5abe..c9b82f313 100644 --- a/src/templates.jl +++ b/src/templates.jl @@ -304,4 +304,4 @@ function aiextract(schema::AbstractPromptSchema, template::Symbol; kwargs...) end function aiscan(schema::AbstractPromptSchema, template::Symbol; kwargs...) aiscan(schema, AITemplate(template); kwargs...) -end \ No newline at end of file +end diff --git a/src/utils.jl b/src/utils.jl index c7dcd83f6..5b0f07458 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -373,4 +373,4 @@ function auth_header(api_key::String) "Content-Type" => "application/json", "Accept" => "application/json", ] -end \ No newline at end of file +end diff --git a/test/Experimental/AgentTools/code_feedback.jl b/test/Experimental/AgentTools/code_feedback.jl index c2d77000b..e2a5e1847 100644 --- a/test/Experimental/AgentTools/code_feedback.jl +++ b/test/Experimental/AgentTools/code_feedback.jl @@ -66,4 +66,4 @@ using PromptingTools.Experimental.AgentTools: CodeEmpty, feedback = aicodefixer_feedback(CodeSuccess(), cb) @test occursin("Execution has been successful", feedback) @test occursin("**Output Captured:**", feedback) -end \ No newline at end of file +end diff --git a/test/Experimental/AgentTools/lazy_types.jl b/test/Experimental/AgentTools/lazy_types.jl index 76b3c4ef1..fe1dc19a2 100644 --- a/test/Experimental/AgentTools/lazy_types.jl +++ b/test/Experimental/AgentTools/lazy_types.jl @@ -162,4 +162,4 @@ end show(io, codefixer) output = String(take!(io)) @test output == "AICodeFixer(Rounds: 0/5)" -end \ No newline at end of file +end diff --git a/test/Experimental/AgentTools/utils.jl b/test/Experimental/AgentTools/utils.jl index b488643cb..a52bc0320 100644 --- a/test/Experimental/AgentTools/utils.jl +++ b/test/Experimental/AgentTools/utils.jl @@ -55,4 +55,4 @@ end conversation = PT.AbstractMessage[] truncated = truncate_conversation(conversation, max_conversation_length = 32000) @test isempty(truncated) -end \ No newline at end of file +end diff --git a/test/Experimental/RAGTools/evaluation.jl b/test/Experimental/RAGTools/evaluation.jl index db8ed3be3..1e5e79f03 100644 --- a/test/Experimental/RAGTools/evaluation.jl +++ b/test/Experimental/RAGTools/evaluation.jl @@ -204,4 +204,4 @@ end Ref(Dict(:key1 => "value1", :key2 => 2))) # clean up close(echo_server) -end \ No newline at end of file +end diff --git a/test/Experimental/RAGTools/preparation.jl b/test/Experimental/RAGTools/preparation.jl index 1a20749dd..f781c2e05 100644 --- a/test/Experimental/RAGTools/preparation.jl +++ b/test/Experimental/RAGTools/preparation.jl @@ -139,4 +139,4 @@ end # clean up close(echo_server) -end \ No newline at end of file +end diff --git a/test/Experimental/RAGTools/retrieval.jl b/test/Experimental/RAGTools/retrieval.jl index 296904f88..4328397f9 100644 --- a/test/Experimental/RAGTools/retrieval.jl +++ b/test/Experimental/RAGTools/retrieval.jl @@ -118,4 +118,4 @@ end ci1, question, cc2; top_n = 1) -end \ No newline at end of file +end diff --git a/test/Experimental/RAGTools/utils.jl b/test/Experimental/RAGTools/utils.jl index cc93c31f9..45b8b75b4 100644 --- a/test/Experimental/RAGTools/utils.jl +++ b/test/Experimental/RAGTools/utils.jl @@ -43,4 +43,4 @@ end @test size(merged_mat) == (4, 3) @test combined_vocab == ["word1", "word2", "word3"] @test merged_mat ≈ [1.0 2.0 0.0; 3.0 4.0 0.0; 0.0 5.0 6.0; 0.0 7.0 8.0] -end \ No newline at end of file +end diff --git a/test/code_generation.jl b/test/code_generation.jl index 2819fad25..1c98654f9 100644 --- a/test/code_generation.jl +++ b/test/code_generation.jl @@ -733,4 +733,4 @@ end @test isparsed(cb) == false cb = AICode("a+1") @test isparsed(cb) == true -end \ No newline at end of file +end diff --git a/test/extraction.jl b/test/extraction.jl index a75c4fefa..6d50d9d98 100644 --- a/test/extraction.jl +++ b/test/extraction.jl @@ -239,4 +239,4 @@ end ## MaybeWraper name cleanup schema = function_call_signature(MaybeExtract{MyMeasurement2}) @test schema["name"] == "MaybeExtractMyMeasurement2_extractor" -end \ No newline at end of file +end diff --git a/test/llm_interface.jl b/test/llm_interface.jl index c012b706d..35daa927c 100644 --- a/test/llm_interface.jl +++ b/test/llm_interface.jl @@ -64,4 +64,4 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage ## Return things to previous PromptingTools.PROMPT_SCHEMA = OLD_PROMPT_SCHEMA -end \ No newline at end of file +end diff --git a/test/llm_ollama.jl b/test/llm_ollama.jl index 13052e7cb..d3092f4e7 100644 --- a/test/llm_ollama.jl +++ b/test/llm_ollama.jl @@ -150,4 +150,4 @@ end @testset "not implemented ai* functions" begin @test_throws ErrorException aiextract(OllamaSchema(), "prompt") @test_throws ErrorException aiclassify(OllamaSchema(), "prompt") -end \ No newline at end of file +end diff --git a/test/llm_ollama_managed.jl b/test/llm_ollama_managed.jl index 6456b1dac..aebb85233 100644 --- a/test/llm_ollama_managed.jl +++ b/test/llm_ollama_managed.jl @@ -196,4 +196,4 @@ end @test_throws ErrorException aiextract(OllamaManagedSchema(), "prompt") @test_throws ErrorException aiclassify(OllamaManagedSchema(), "prompt") @test_throws ErrorException aiscan(OllamaManagedSchema(), "prompt") -end \ No newline at end of file +end diff --git a/test/llm_shared.jl b/test/llm_shared.jl index f3ca4799f..f449d9e5a 100644 --- a/test/llm_shared.jl +++ b/test/llm_shared.jl @@ -267,4 +267,4 @@ end conversation, return_all = true) @test output == expected_output -end \ No newline at end of file +end diff --git a/test/macros.jl b/test/macros.jl index aa3addaed..55cb440eb 100644 --- a/test/macros.jl +++ b/test/macros.jl @@ -86,4 +86,4 @@ end # clean up empty!(CONV_HISTORY) -end \ No newline at end of file +end diff --git a/test/messages.jl b/test/messages.jl index 8636d386f..95cc5e15a 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -92,4 +92,4 @@ end # no UserMessages msg = UserMessageWithImages(content; image_url) # unclear where to add the new images! @test_throws AssertionError attach_images_to_user_message(msg; image_url) -end \ No newline at end of file +end diff --git a/test/serialization.jl b/test/serialization.jl index 59066428f..9b04667f2 100644 --- a/test/serialization.jl +++ b/test/serialization.jl @@ -35,4 +35,4 @@ end @test metadata[1].version == version @test metadata[1].content == "Template Metadata" @test metadata[1].source == "" -end \ No newline at end of file +end diff --git a/test/templates.jl b/test/templates.jl index 4239215f3..23d613ac7 100644 --- a/test/templates.jl +++ b/test/templates.jl @@ -55,4 +55,4 @@ end schema3 = TestEchoOpenAISchema(; response, status = 200) msg = aiclassify(schema3, template_name; it = "Is this correct?") @test schema3.inputs == expected_template_rendered -end \ No newline at end of file +end diff --git a/test/utils.jl b/test/utils.jl index 31d43fd42..3c21bd5ef 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -236,4 +236,4 @@ end "Accept" => "application/json", ] @test_throws ArgumentError auth_header("") -end \ No newline at end of file +end From 1f4b3134e628c86395277c47b36c32a28d8a3010 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 26 Jan 2024 20:16:00 +0000 Subject: [PATCH 106/251] Add devcontainer.json (#60) --- .devcontainer/devcontainer.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..6c2de618b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,20 @@ +{ + "image": "mcr.microsoft.com/devcontainers/universal:2", + "features": { + "ghcr.io/julialang/devcontainer-features/julia:1": { + "channel":"1.10" + } + + }, + "customizations": { + "vscode": { + "settings": { + "editor.formatOnSave": true, + }, + "extensions": [ + "GitHub.copilot", + "GitHub.copilot-chat" + ] + } + }, +} From 504d71426dc8edc90fe7219d3a9fd5dae6acf19e Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 27 Jan 2024 09:30:19 +0000 Subject: [PATCH 107/251] fix API key getter with @noinline (#61) --- CHANGELOG.md | 1 + src/user_preferences.jl | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68f0bab23..17297a570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed typos in the documentation +- Fixed a bug when API keys set in ENV would not be picked up by the package (caused by inlining of the `get(ENV,...)` during precompilation) ## [0.9.0] diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 4142f0454..09e3626ac 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -115,20 +115,20 @@ const MODEL_EMBEDDING::String = @load_preference("MODEL_EMBEDDING", # First, load from preferences, then from environment variables const OPENAI_API_KEY::String = @load_preference("OPENAI_API_KEY", - default=get(ENV, "OPENAI_API_KEY", "")); + default=@noinline get(ENV, "OPENAI_API_KEY", "")); # Note: Disable this warning by setting OPENAI_API_KEY to anything isempty(OPENAI_API_KEY) && @warn "OPENAI_API_KEY variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.OPENAI_API_KEY=`!" const MISTRALAI_API_KEY::String = @load_preference("MISTRALAI_API_KEY", - default=get(ENV, "MISTRALAI_API_KEY", "")); + default=@noinline get(ENV, "MISTRALAI_API_KEY", "")); const COHERE_API_KEY::String = @load_preference("COHERE_API_KEY", - default=get(ENV, "COHERE_API_KEY", "")); + default=@noinline get(ENV, "COHERE_API_KEY", "")); ## Address of the local server const LOCAL_SERVER::String = @load_preference("LOCAL_SERVER", - default=get(ENV, "LOCAL_SERVER", "http://127.0.0.1:10897/v1")); + default=@noinline get(ENV, "LOCAL_SERVER", "http://127.0.0.1:10897/v1")); ## CONVERSATION HISTORY """ From 71ba389fa72e7f689fe5a35c24b0c810fa10a018 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 28 Jan 2024 20:03:55 +0000 Subject: [PATCH 108/251] Improve code feedback (#62) --- Project.toml | 8 +- src/Experimental/AgentTools/AgentTools.jl | 3 +- src/Experimental/AgentTools/code_feedback.jl | 174 +++- src/PromptingTools.jl | 7 +- src/code_eval.jl | 364 ++++++++ src/code_expressions.jl | 104 +++ src/code_generation.jl | 780 ------------------ src/code_parsing.jl | 415 ++++++++++ test/Experimental/AgentTools/code_feedback.jl | 183 +++- test/code_eval.jl | 372 +++++++++ test/code_expressions.jl | 151 ++++ test/code_generation.jl | 736 ----------------- test/code_parsing.jl | 379 +++++++++ test/runtests.jl | 6 +- 14 files changed, 2147 insertions(+), 1535 deletions(-) create mode 100644 src/code_eval.jl create mode 100644 src/code_expressions.jl delete mode 100644 src/code_generation.jl create mode 100644 src/code_parsing.jl create mode 100644 test/code_eval.jl create mode 100644 test/code_expressions.jl delete mode 100644 test/code_generation.jl create mode 100644 test/code_parsing.jl diff --git a/Project.toml b/Project.toml index 3a5093961..9737f70a6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.10.0" +version = "0.10.0-DEV" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -9,8 +9,10 @@ HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [weakdeps] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -30,6 +32,7 @@ LinearAlgebra = "<0.0.1, 1" Logging = "<0.0.1, 1" Markdown = "<0.0.1, 1" OpenAI = "0.8.7" +Pkg = "<0.0.1, 1" PrecompileTools = "1" Preferences = "1" SparseArrays = "<0.0.1, 1" @@ -40,7 +43,6 @@ julia = "1.9,1.10" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Test", "SparseArrays", "LinearAlgebra", "Markdown"] +test = ["Aqua", "SparseArrays", "LinearAlgebra", "Markdown"] diff --git a/src/Experimental/AgentTools/AgentTools.jl b/src/Experimental/AgentTools/AgentTools.jl index 4c22c2546..f6c847d0b 100644 --- a/src/Experimental/AgentTools/AgentTools.jl +++ b/src/Experimental/AgentTools/AgentTools.jl @@ -9,10 +9,11 @@ module AgentTools using PromptingTools const PT = PromptingTools +using Test include("utils.jl") -export aicodefixer_feedback +export aicodefixer_feedback, error_feedback, score_feedback include("code_feedback.jl") export AICall, AIGenerate, AIExtract, AIEmbed, AIClassify, AIScan diff --git a/src/Experimental/AgentTools/code_feedback.jl b/src/Experimental/AgentTools/code_feedback.jl index a4283ed41..14ac1e164 100644 --- a/src/Experimental/AgentTools/code_feedback.jl +++ b/src/Experimental/AgentTools/code_feedback.jl @@ -103,23 +103,181 @@ function aicodefixer_feedback(::CodeFailedEval, kwargs...) feedback = AbstractString[] ## Grab the error message - error_str = if cb.error isa TaskFailedException - string(cb.error.task.result) - else - split(string(cb.error), "JuliaSyntax.SourceFile")[begin] - end ## Decide how much space can be dedicated for this error (ie, do we have stdout as well?) chunk_length = isnothing(cb.stdout) || isempty(cb.stdout) ? max_length : max_length ÷ 2 - end_idx = min(length(error_str), nextind(error_str, 0, chunk_length)) - push!(feedback, "**Error Detected:** $(error_str[begin:end_idx])") + error_str = error_feedback(cb.error; max_length = chunk_length) + push!(feedback, "**Error Detected:**\n$(error_str)") + + ## Add the lines that caused it + if !isempty(cb.error_lines) + feedback_lines = String[] + max_lines = 2 # max lines to send + logged_lines = Set{Int}() + code_lines = split(cb.code, "\n") + for line in cb.error_lines + if line ∉ logged_lines && line ≤ length(code_lines) && + length(logged_lines) <= max_lines + push!(feedback_lines, "- " * code_lines[line]) + push!(logged_lines, line) + end + end + push!(feedback, + "\n\n**Lines that caused the error:**\n" * join(feedback_lines, "\n")) + end if !isnothing(cb.stdout) && !isempty(string(cb.stdout)) ## Add the optional STDOUT (for test failures) chunk_length = max_length - sum(length, feedback) end_idx = min(length(cb.stdout), nextind(cb.stdout, 0, chunk_length)) - push!(feedback, "**Output Captured:** $(cb.stdout[begin:end_idx])") + push!(feedback, "**Output Captured:**\n $(cb.stdout[begin:end_idx])") end return isempty(feedback) ? "No feedback provided." : join(feedback, "\n\n") end + +function testset_feedback(msg::AIMessage; + prefix::AbstractString = "", + suffix::AbstractString = "", kwargs...) + code = join(PT.extract_code_blocks(msg.content), "\n") + test_f = PT.extract_testset_name(code) + if !isnothing(test_f) + test_f_mock = "$(replace(test_f, r"[\s\(\)]" => ""))(args...; kwargs...) = nothing" + prefix = prefix * "\n" * test_f_mock + end + # Insert mock function, remove test items -- good test suite should pass + cb = AICode(msg; + skip_unsafe = true, + prefix, suffix, + expression_transform = :remove_test_items, kwargs...) + feedback = if !isnothing(cb.error) + aicodefixer_feedback(CodeFailedEval(), cb) + else + nothing + end + return feedback +end + +### Feedback for individual errors +error_feedback(e::Any; max_length::Int = 512) = "No error found. Ignore." +function error_feedback(e::Exception; max_length::Int = 512) + io = IOBuffer() + name_ = typeof(e) |> nameof |> string + write(io, "**", name_, "**:\n") + showerror(io, e) + first(String(take!(io)), max_length) +end +# FallbackTestSetException will take the default path +# TODO: add with x==1; @test x==2 +function error_feedback(e::Test.TestSetException; max_length::Int = 512) + io = IOBuffer() + name_ = typeof(e) |> nameof |> string + write(io, "**", name_, "**:\n") + showerror(io, e) + ## Unpack the results in + write(io, "\n") + for error_ in e.errors_and_fails + io_ = IOBuffer() + showerror(io_, error_) + out = split(String(take!(io_)), "Stacktrace")[begin] + write(io, "\n", out) + end + + first(String(take!(io)), max_length) +end +function error_feedback(e::Task; max_length::Int = 512) + out = try + fetch(e) + catch e + e + end + error_feedback(out; max_length) +end +function error_feedback(e::TaskFailedException; max_length::Int = 512) + error_feedback(e.task.result; max_length) +end +function error_feedback(e::Base.Meta.ParseError; max_length::Int = 512) + io = IOBuffer() + name_ = typeof(e) |> nameof |> string + write(io, "**", name_, "**:\n") + showerror(io, e) + first(String(take!(io)), max_length) +end +function error_feedback(e::UndefVarError; max_length::Int = 512) + io = IOBuffer() + showerror(io, e) + # Simple heurisic - if's available in Main/Base + found = false + for mod in [Base, Main] + if hasproperty(mod, e.var) + write(io, + "\nExpert Tip: I know that the variable $(e.var) is defined in $(nameof(mod)) module. Use `import $(mod).$(e.var)` to use it.") + found = true + break + end + end + !found && write(io, + "\nTip: Does it even exist? Does it need to be imported? Or is it a typo?") + first(String(take!(io)), max_length) +end +function error_feedback(e::ArgumentError; max_length::Int = 512) + io = IOBuffer() + showerror(io, e) + # Simple heurisic - if's available in Main/Base + pkg = PT.extract_package_name_from_argerror(e.msg) + if !isnothing(pkg) + for mod in [Base, Main] + hasproperty(mod, Symbol(pkg)) && (write(io, + "\nExpert Tip: I know that the package $pkg is defined in $(nameof(mod)) module. You MUST use `import $(mod).$(pkg)` to use it."); + break) + end + end + first(String(take!(io)), max_length) +end + +## +function score_feedback(cb::AICode, expr_to_run::Expr = Expr(:block)) + score = if isempty(cb.code) + 0 + elseif !PT.isparsed(cb) + 1 + elseif isa(cb.error, Test.TestSetException) + # error outside of test is twice as bad + 10 + cb.error.pass - cb.error.fail - 2cb.error.error # ignore broken + elseif isa(cb.error, Exception) + 2 + elseif isvalid(cb) + 10 + else + throw(ArgumentError("Invalid code feedback path?")) + end + return score +end + +function extract_test_counts(test_summary::String) + # Split the test summary into lines + lines = split(test_summary, '\n') + length_ = length(lines) + counts = Dict{String, Int}() + + # Find the line containing the column headers + for i in eachindex(lines) + # iterate only until penultimate, since we look ahead one line + i == length_ && break + m = match(r"Test Summary:\s*\|\s*([^|]*?)\s*Total", lines[i]) + if !isnothing(m) && !isnothing(m.captures) + headers = [ + [lowercase(strip(col)) + for col in split(m.captures[1], " ") if !isempty(col)]..., "total"] + next_line = lines[i + 1] + digits = [tryparse(Int, m.match) + for m in eachmatch(r"\b\d+\b", next_line)] + for (header, score) in zip(headers, digits) + if !isnothing(score) + counts[header] = get(counts, header, 0) + score + end + end + end + end + return counts +end diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 51c47ddd3..867cd0e87 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -9,6 +9,7 @@ using HTTP import Preferences using Preferences: @load_preference, @set_preferences! using PrecompileTools +using Test, Pkg # GLOBALS and Preferences are managed by Preferences.jl - see src/preferences.jl for details @@ -53,9 +54,11 @@ include("serialization.jl") include("extraction.jl") ## Utilities to support code generation -export AICode # Not export extract_code_blocks, extract_function_name -include("code_generation.jl") +include("code_parsing.jl") +include("code_expressions.jl") +export AICode +include("code_eval.jl") ## Individual interfaces include("llm_shared.jl") diff --git a/src/code_eval.jl b/src/code_eval.jl new file mode 100644 index 000000000..97d96f313 --- /dev/null +++ b/src/code_eval.jl @@ -0,0 +1,364 @@ +# These are utilities to support code generation +# +# Types defined (not exported!): +# - AbstractCodeBlock +# - AICode +# +# Functions defined (not exported!): +# - detect_pkg_operation, extract_julia_imports, detect_missing_packages +# - extract_code_blocks +# - eval! +# +# +# +## # Types + +abstract type AbstractCodeBlock end + +""" + AICode(code::AbstractString; auto_eval::Bool=true, safe_eval::Bool=false, + skip_unsafe::Bool=false, capture_stdout::Bool=true, verbose::Bool=false, + prefix::AbstractString="", suffix::AbstractString="", remove_tests::Bool=false, execution_timeout::Int = 60) + + AICode(msg::AIMessage; auto_eval::Bool=true, safe_eval::Bool=false, + skip_unsafe::Bool=false, skip_invalid::Bool=false, capture_stdout::Bool=true, + verbose::Bool=false, prefix::AbstractString="", suffix::AbstractString="", remove_tests::Bool=false, execution_timeout::Int = 60) + +A mutable structure representing a code block (received from the AI model) with automatic parsing, execution, and output/error capturing capabilities. + +Upon instantiation with a string, the `AICode` object automatically runs a code parser and executor (via `PromptingTools.eval!()`), capturing any standard output (`stdout`) or errors. +This structure is useful for programmatically handling and evaluating Julia code snippets. + +See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` + +# Workflow +- Until `cb::AICode` has been evaluated, `cb.success` is set to `nothing` (and so are all other fields). +- The text in `cb.code` is parsed (saved to `cb.expression`). +- The parsed expression is evaluated. +- Outputs of the evaluated expression are captured in `cb.output`. +- Any `stdout` outputs (e.g., from `println`) are captured in `cb.stdout`. +- If an error occurs during evaluation, it is saved in `cb.error`. +- After successful evaluation without errors, `cb.success` is set to `true`. + Otherwise, it is set to `false` and you can inspect the `cb.error` to understand why. + +# Properties +- `code::AbstractString`: The raw string of the code to be parsed and executed. +- `expression`: The parsed Julia expression (set after parsing `code`). +- `stdout`: Captured standard output from the execution of the code. +- `output`: The result of evaluating the code block. +- `success::Union{Nothing, Bool}`: Indicates whether the code block executed successfully (`true`), unsuccessfully (`false`), or has yet to be evaluated (`nothing`). +- `error::Union{Nothing, Exception}`: Any exception raised during the execution of the code block. + +# Keyword Arguments +- `auto_eval::Bool`: If set to `true`, the code block is automatically parsed and evaluated upon instantiation. Defaults to `true`. +- `safe_eval::Bool`: If set to `true`, the code block checks for package operations (e.g., installing new packages) and missing imports, and then evaluates the code inside a bespoke scratch module. This is to ensure that the evaluation does not alter any user-defined variables or the global state. Defaults to `false`. +- `skip_unsafe::Bool`: If set to `true`, we skip any lines in the code block that are deemed unsafe (eg, `Pkg` operations). Defaults to `false`. +- `skip_invalid::Bool`: If set to `true`, we skip code blocks that do not even parse. Defaults to `false`. +- `verbose::Bool`: If set to `true`, we print out any lines that are skipped due to being unsafe. Defaults to `false`. +- `capture_stdout::Bool`: If set to `true`, we capture any stdout outputs (eg, test failures) in `cb.stdout`. Defaults to `true`. +- `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. + Useful to add some additional code definition or necessary imports. Defaults to an empty string. +- `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. + Useful to check that tests pass or that an example executes. Defaults to an empty string. +- `remove_tests::Bool`: If set to `true`, we remove any `@test` or `@testset` macros from the code block before parsing and evaluation. Defaults to `false`. +- `execution_timeout::Int`: The maximum time (in seconds) allowed for the code block to execute. Defaults to 60 seconds. + +# Methods +- `Base.isvalid(cb::AICode)`: Check if the code block has executed successfully. Returns `true` if `cb.success == true`. + +# Examples + +```julia +code = AICode("println(\"Hello, World!\")") # Auto-parses and evaluates the code, capturing output and errors. +isvalid(code) # Output: true +code.stdout # Output: "Hello, World!\n" +``` + +We try to evaluate "safely" by default (eg, inside a custom module, to avoid changing user variables). + You can avoid that with `save_eval=false`: + +```julia +code = AICode("new_variable = 1"; safe_eval=false) +isvalid(code) # Output: true +new_variable # Output: 1 +``` + +You can also call AICode directly on an AIMessage, which will extract the Julia code blocks, concatenate them and evaluate them: + +```julia +msg = aigenerate("In Julia, how do you create a vector of 10 random numbers?") +code = AICode(msg) +# Output: AICode(Success: True, Parsed: True, Evaluated: True, Error Caught: N/A, StdOut: True, Code: 2 Lines) + +# show the code +code.code |> println +# Output: +# numbers = rand(10) +# numbers = rand(1:100, 10) + +# or copy it to the clipboard +code.code |> clipboard + +# or execute it in the current module (=Main) +eval(code.expression) +``` +""" +@kwdef mutable struct AICode <: AbstractCodeBlock + code::AbstractString + expression = nothing + stdout = nothing + output = nothing + success::Union{Nothing, Bool} = nothing + error::Union{Nothing, Exception} = nothing + error_lines::Vector{Int} = Int[] +end +# Eager evaluation if instantiated with a string +function (CB::Type{T})(md::AbstractString; + auto_eval::Bool = true, + safe_eval::Bool = true, + skip_unsafe::Bool = false, + capture_stdout::Bool = true, + verbose::Bool = false, + prefix::AbstractString = "", + suffix::AbstractString = "", + expression_transform::Symbol = :nothing, + execution_timeout::Int = 60) where {T <: AbstractCodeBlock} + ## + @assert execution_timeout>0 "execution_timeout must be positive" + if skip_unsafe + md, removed = remove_unsafe_lines(md; verbose) + else + removed = "" + end + cb = CB(; code = md) + if auto_eval + ## set to timeout in `execution_timeout` seconds + result = @timeout execution_timeout begin + eval!(cb; + safe_eval, + capture_stdout, + prefix, + suffix, + expression_transform) + end nothing # set to nothing if it fails + # Check if we timed out + if isnothing(result) + cb.success = false + cb.error = InterruptException() + end + end + if !isempty(removed) + ## Add to STDOUT what we removed + warning = string("!!! IMPORTANT: Unsafe lines blocked from execution (eg, Pkg operations or imports of non-existent packages):", + "\n$removed\n", + "Fix or find a workaround!") + if isnothing(cb.stdout) + cb.stdout = warning + else + cb.stdout = "$(cb.stdout)\n\n$warning" + end + end + return cb +end +Base.isvalid(cb::AbstractCodeBlock) = cb.success == true +function Base.copy(cb::AbstractCodeBlock) + AICode(cb.code, + cb.expression, + cb.stdout, + cb.output, + cb.success, + cb.error, + cb.error_lines) +end +# equality check for testing, only equal if all fields are equal and type is the same +function Base.var"=="(c1::T, c2::T) where {T <: AICode} + all([getproperty(c1, f) == getproperty(c2, f) for f in fieldnames(T)]) +end +function Base.show(io::IO, cb::AICode) + success_str = cb.success === nothing ? "N/A" : titlecase(string(cb.success)) + expression_str = cb.expression === nothing ? "N/A" : titlecase(string(isparsed(cb))) + stdout_str = cb.stdout === nothing ? "N/A" : "True" + output_str = cb.output === nothing ? "N/A" : "True" + error_str = cb.error === nothing ? "N/A" : "True" + count_lines = count(==('\n'), collect(cb.code)) + 1 # there is always at least one line + + print(io, + "AICode(Success: $success_str, Parsed: $expression_str, Evaluated: $output_str, Error Caught: $error_str, StdOut: $stdout_str, Code: $count_lines Lines)") +end +function isparsed(cb::AICode) + return isparsed(cb.expression) && !isparseerror(cb.error) +end + +## Overload for AIMessage - simply extracts the code blocks and concatenates them +function AICode(msg::AIMessage; + verbose::Bool = false, + skip_invalid::Bool = false, + kwargs...) + code = extract_code_blocks(msg.content) + if isempty(code) + ## Fallback option for generic code fence, we must check if the content is parseable + code = extract_code_blocks_fallback(msg.content) + skip_invalid = true # set to true if we use fallback option + end + if skip_invalid + ## Filter out extracted code blocks that do not even parse + filter!(is_julia_code, code) + end + code = join(code, "\n") + return AICode(code; verbose, kwargs...) +end + +## # Functions + +""" + eval!(cb::AbstractCodeBlock; + safe_eval::Bool = true, + capture_stdout::Bool = true, + prefix::AbstractString = "", + suffix::AbstractString = "") + +Evaluates a code block `cb` in-place. It runs automatically when AICode is instantiated with a String. + +Check the outcome of evaluation with `Base.isvalid(cb)`. If `==true`, provide code block has executed successfully. + +Steps: +- If `cb::AICode` has not been evaluated, `cb.success = nothing`. + After the evaluation it will be either `true` or `false` depending on the outcome +- Parse the text in `cb.code` +- Evaluate the parsed expression +- Capture outputs of the evaluated in `cb.output` +- [OPTIONAL] Capture any stdout outputs (eg, test failures) in `cb.stdout` +- If any error exception is raised, it is saved in `cb.error` +- Finally, if all steps were successful, success is set to `cb.success = true` + +# Keyword Arguments +- `safe_eval::Bool`: If `true`, we first check for any Pkg operations (eg, installing new packages) and missing imports, + then the code will be evaluated inside a bespoke scratch module (not to change any user variables) +- `capture_stdout::Bool`: If `true`, we capture any stdout outputs (eg, test failures) in `cb.stdout` +- `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. + Useful to add some additional code definition or necessary imports. Defaults to an empty string. +- `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. + Useful to check that tests pass or that an example executes. Defaults to an empty string. +""" +function eval!(cb::AbstractCodeBlock; + safe_eval::Bool = true, + capture_stdout::Bool = true, + prefix::AbstractString = "", + suffix::AbstractString = "", + expression_transform::Symbol = :nothing) + @assert expression_transform in (:nothing, :remove_all_tests, :remove_test_items) "expression_transform must be one of :nothing, :remove_all_tests, :remove_test_items" + (; code) = cb + # reset + cb.expression = nothing + cb.output = nothing + cb.stdout = nothing + + ## Safety checks on `code` only -- treat it as a parsing failure + if safe_eval + detected, missing_packages = detect_missing_packages(extract_julia_imports(code)) + if detect_pkg_operation(code) || detected + cb.success = false + detect_pkg_operation(code) && + (cb.error = ErrorException("Safety Error: Use of package manager (`Pkg.*`) detected! Please verify the safety of the code or disable the safety check (`safe_eval=false`)")) + detected && + (cb.error = ErrorException("Safety Error: Failed package import. Missing packages: $(join(string.(missing_packages),", ")). Please add them or disable the safety check (`safe_eval=false`)")) + return cb + end + end + ## Catch bad code extraction + if isempty(code) + cb.error = ErrorException("Parse Error: No code found!") + cb.success = false + return cb + end + + ## Parsing test, if it fails, we skip the evaluation + try + ex = Meta.parseall(code) + cb.expression = ex + if !isparsed(ex) + cb.error = @eval(Main, $(ex)) # show the error + cb.success = false + return cb + end + catch e + cb.error = e + cb.success = false + return cb + end + + ## Pick the right expression transform (if any) + _transform = if expression_transform == :nothing + "identity" + elseif expression_transform == :remove_all_tests + "PromptingTools.remove_all_tests_from_expr!" + elseif expression_transform == :remove_test_items + "PromptingTools.remove_test_items_from_expr!" + end + + ## Add prefix and suffix + ## Write the code as an `include_string` in a module (into `io`) + io = IOBuffer() + module_name = safe_eval ? replace(string(gensym("SafeMod")), "#" => "") : "Main" + safe_eval && write(io, "module $module_name\n") + write(io, "using Test\nimport PromptingTools\n") + write(io, prefix, "\n") + write(io, + "include_string($_transform, $module_name,\"\"\"$(escape_string(code))\"\"\", \"__code_string_eval\")\n") + write(io, suffix, "\n") + safe_eval && write(io, "end") + code_full = String(take!(io)) + + ## Eval (we parse the full code now, including the prefix and suffix) + # TODO: can fail if provided prefix/suffix are invalid and break the parsing, + # but they are assumed to be correct as they are user-provided + cb.expression = Meta.parseall(code_full) + eval!(cb, cb.expression; capture_stdout, eval_module = Main) + return cb +end + +# Evaluation of any arbitrary expression with result recorded in `cb` +function eval!(cb::AbstractCodeBlock, expr::Expr; + capture_stdout::Bool = true, + eval_module::Module = Main) + ## Reset + cb.success = nothing + cb.error = nothing + cb.output = nothing + cb.stdout = nothing + empty!(cb.error_lines) + + # Prepare to catch any stdout + if capture_stdout + pipe = Pipe() + redirect_stdout(pipe) do + try + cb.output = @eval(eval_module, $(expr)) + cb.success = true + catch e + cb.error = e + cb.success = false + end + end + close(Base.pipe_writer(pipe)) + cb.stdout = read(pipe, String) + else + # Ignore stdout, just eval + try + cb.output = @eval(eval_module, $(expr)) + cb.success = true + catch e + cb.error = e + cb.success = false + end + end + ## unwrap load error + if cb.error isa LoadError + push!(cb.error_lines, cb.error.line) + append!(cb.error_lines, extract_stacktrace_lines(cb.error.file, cb.stdout)) + cb.error = cb.error.error + elseif !isnothing(cb.error) + append!(cb.error_lines, extract_stacktrace_lines("__code_string_eval", cb.stdout)) + end + return cb +end \ No newline at end of file diff --git a/src/code_expressions.jl b/src/code_expressions.jl new file mode 100644 index 000000000..9bbace0ab --- /dev/null +++ b/src/code_expressions.jl @@ -0,0 +1,104 @@ +## Parsing error detection +function isparsed(ex::Expr) + parse_error = Meta.isexpr(ex, :toplevel) && !isempty(ex.args) && + Meta.isexpr(ex.args[end], (:error, :incomplete)) + return !parse_error +end +function isparsed(ex::Nothing) + return false +end +function isparseerror(err::Exception) + return err isa Base.Meta.ParseError || + (err isa ErrorException && startswith(err.msg, "syntax:")) +end +function isparseerror(err::Nothing) + return false +end + +## Parsing Helpers +JULIA_EXPR_HEADS = [ + :block, + :quote, + :call, + :macrocall, + :(=), + :function, + :for, + :if, + :while, + :let, + :try, + :catch, + :finally, + :method, + :tuple, + :array, + :index, + :ref, + :., + :do, + :curly, + :typed_vcat, + :typed_hcat, + :typed_vcat, + :comprehension, + :generator, + :kw, + :where, +] +# Checks if the provided expression `ex` has some hallmarks of Julia code. Very naive! +# Serves as a quick check to avoid trying to eval output cells (```plaintext ... ```) +is_julia_expr(ex::Any) = false +function is_julia_expr(ex::Expr) + ## Expression itself + Meta.isexpr(ex, JULIA_EXPR_HEADS) && return true + ## Its arguments + for arg in ex.args + Meta.isexpr(arg, JULIA_EXPR_HEADS) && return true + end + ## Nothing found... + return false +end + +# Remove any given macro expression from the expression tree, used to remove tests +function remove_macro_expr!(expr, sym::Symbol = Symbol("@testset")) + if expr isa Expr && expr.head == :macrocall && !isempty(expr.args) && + expr.args[1] == sym + return Expr(:block) + elseif expr isa Expr && !isempty(expr.args) + expr.args = filter(x -> !(x isa Expr && x.head == :macrocall && !isempty(x.args) && + x.args[1] == sym), + expr.args) + foreach(x -> remove_macro_expr!(x, sym), expr.args) + end + expr +end + +# Remove testsets and sets from the expression tree +function remove_test_items_from_expr!(expr) + # Focus only on the three most common test macros + expr = remove_macro_expr!(expr, Symbol("@test")) + expr = remove_macro_expr!(expr, Symbol("@test_throws")) + return expr +end +function remove_all_tests_from_expr!(expr) + # Focus only on the three most common test macros + expr = remove_macro_expr!(expr, Symbol("@testset")) + expr = remove_test_items_from_expr!(expr) + return expr +end + +# Utility to identify the module name in a given expression (to evaluate subsequent calls in it) +function extract_module_name(expr) + if isa(expr, Expr) && expr.head == :module + return expr.args[2] # The second argument is typically the module name + elseif isa(expr, Expr) && !isempty(expr.args) + output = extract_module_name.(expr.args) + for item in output + if !isnothing(item) + return item + end + end + end + nothing +end \ No newline at end of file diff --git a/src/code_generation.jl b/src/code_generation.jl deleted file mode 100644 index b85a1aeec..000000000 --- a/src/code_generation.jl +++ /dev/null @@ -1,780 +0,0 @@ -# These are utilities to support code generation -# -# Types defined (not exported!): -# - AbstractCodeBlock -# - AICode -# -# Functions defined (not exported!): -# - detect_pkg_operation, extract_julia_imports, detect_missing_packages -# - extract_code_blocks -# - eval! -# -# -# -## # Types - -abstract type AbstractCodeBlock end - -""" - AICode(code::AbstractString; auto_eval::Bool=true, safe_eval::Bool=false, - skip_unsafe::Bool=false, capture_stdout::Bool=true, verbose::Bool=false, - prefix::AbstractString="", suffix::AbstractString="", remove_tests::Bool=false, execution_timeout::Int = 60) - - AICode(msg::AIMessage; auto_eval::Bool=true, safe_eval::Bool=false, - skip_unsafe::Bool=false, skip_invalid::Bool=false, capture_stdout::Bool=true, - verbose::Bool=false, prefix::AbstractString="", suffix::AbstractString="", remove_tests::Bool=false, execution_timeout::Int = 60) - -A mutable structure representing a code block (received from the AI model) with automatic parsing, execution, and output/error capturing capabilities. - -Upon instantiation with a string, the `AICode` object automatically runs a code parser and executor (via `PromptingTools.eval!()`), capturing any standard output (`stdout`) or errors. -This structure is useful for programmatically handling and evaluating Julia code snippets. - -See also: `PromptingTools.extract_code_blocks`, `PromptingTools.eval!` - -# Workflow -- Until `cb::AICode` has been evaluated, `cb.success` is set to `nothing` (and so are all other fields). -- The text in `cb.code` is parsed (saved to `cb.expression`). -- The parsed expression is evaluated. -- Outputs of the evaluated expression are captured in `cb.output`. -- Any `stdout` outputs (e.g., from `println`) are captured in `cb.stdout`. -- If an error occurs during evaluation, it is saved in `cb.error`. -- After successful evaluation without errors, `cb.success` is set to `true`. - Otherwise, it is set to `false` and you can inspect the `cb.error` to understand why. - -# Properties -- `code::AbstractString`: The raw string of the code to be parsed and executed. -- `expression`: The parsed Julia expression (set after parsing `code`). -- `stdout`: Captured standard output from the execution of the code. -- `output`: The result of evaluating the code block. -- `success::Union{Nothing, Bool}`: Indicates whether the code block executed successfully (`true`), unsuccessfully (`false`), or has yet to be evaluated (`nothing`). -- `error::Union{Nothing, Exception}`: Any exception raised during the execution of the code block. - -# Keyword Arguments -- `auto_eval::Bool`: If set to `true`, the code block is automatically parsed and evaluated upon instantiation. Defaults to `true`. -- `safe_eval::Bool`: If set to `true`, the code block checks for package operations (e.g., installing new packages) and missing imports, and then evaluates the code inside a bespoke scratch module. This is to ensure that the evaluation does not alter any user-defined variables or the global state. Defaults to `false`. -- `skip_unsafe::Bool`: If set to `true`, we skip any lines in the code block that are deemed unsafe (eg, `Pkg` operations). Defaults to `false`. -- `skip_invalid::Bool`: If set to `true`, we skip code blocks that do not even parse. Defaults to `false`. -- `verbose::Bool`: If set to `true`, we print out any lines that are skipped due to being unsafe. Defaults to `false`. -- `capture_stdout::Bool`: If set to `true`, we capture any stdout outputs (eg, test failures) in `cb.stdout`. Defaults to `true`. -- `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. - Useful to add some additional code definition or necessary imports. Defaults to an empty string. -- `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. - Useful to check that tests pass or that an example executes. Defaults to an empty string. -- `remove_tests::Bool`: If set to `true`, we remove any `@test` or `@testset` macros from the code block before parsing and evaluation. Defaults to `false`. -- `execution_timeout::Int`: The maximum time (in seconds) allowed for the code block to execute. Defaults to 60 seconds. - -# Methods -- `Base.isvalid(cb::AICode)`: Check if the code block has executed successfully. Returns `true` if `cb.success == true`. - -# Examples - -```julia -code = AICode("println(\"Hello, World!\")") # Auto-parses and evaluates the code, capturing output and errors. -isvalid(code) # Output: true -code.stdout # Output: "Hello, World!\n" -``` - -We try to evaluate "safely" by default (eg, inside a custom module, to avoid changing user variables). - You can avoid that with `save_eval=false`: - -```julia -code = AICode("new_variable = 1"; safe_eval=false) -isvalid(code) # Output: true -new_variable # Output: 1 -``` - -You can also call AICode directly on an AIMessage, which will extract the Julia code blocks, concatenate them and evaluate them: - -```julia -msg = aigenerate("In Julia, how do you create a vector of 10 random numbers?") -code = AICode(msg) -# Output: AICode(Success: True, Parsed: True, Evaluated: True, Error Caught: N/A, StdOut: True, Code: 2 Lines) - -# show the code -code.code |> println -# Output: -# numbers = rand(10) -# numbers = rand(1:100, 10) - -# or copy it to the clipboard -code.code |> clipboard - -# or execute it in the current module (=Main) -eval(code.expression) -``` -""" -@kwdef mutable struct AICode <: AbstractCodeBlock - code::AbstractString - expression = nothing - stdout = nothing - output = nothing - success::Union{Nothing, Bool} = nothing - error::Union{Nothing, Exception} = nothing -end -# Eager evaluation if instantiated with a string -function (CB::Type{T})(md::AbstractString; - auto_eval::Bool = true, - safe_eval::Bool = true, - skip_unsafe::Bool = false, - capture_stdout::Bool = true, - verbose::Bool = false, - prefix::AbstractString = "", - suffix::AbstractString = "", - remove_tests::Bool = false, - execution_timeout::Int = 60) where {T <: AbstractCodeBlock} - ## - @assert execution_timeout>0 "execution_timeout must be positive" - skip_unsafe && (md = remove_unsafe_lines(md; verbose)) - cb = CB(; code = md) - if auto_eval - # set to timeout in `execution_timeout` seconds - result = @timeout execution_timeout begin - eval!(cb; - safe_eval, - capture_stdout, - prefix, - suffix, - remove_tests) - end nothing # set to nothing if it fails - # Check if we timed out - if isnothing(result) - cb.success = false - cb.error = InterruptException() - end - end - return cb -end -Base.isvalid(cb::AbstractCodeBlock) = cb.success == true -function Base.copy(cb::AbstractCodeBlock) - AICode(cb.code, cb.expression, cb.stdout, cb.output, cb.success, cb.error) -end -# equality check for testing, only equal if all fields are equal and type is the same -function Base.var"=="(c1::T, c2::T) where {T <: AICode} - all([getproperty(c1, f) == getproperty(c2, f) for f in fieldnames(T)]) -end -function Base.show(io::IO, cb::AICode) - success_str = cb.success === nothing ? "N/A" : titlecase(string(cb.success)) - expression_str = cb.expression === nothing ? "N/A" : titlecase(string(isparsed(cb))) - stdout_str = cb.stdout === nothing ? "N/A" : "True" - output_str = cb.output === nothing ? "N/A" : "True" - error_str = cb.error === nothing ? "N/A" : "True" - count_lines = count(==('\n'), collect(cb.code)) + 1 # there is always at least one line - - print(io, - "AICode(Success: $success_str, Parsed: $expression_str, Evaluated: $output_str, Error Caught: $error_str, StdOut: $stdout_str, Code: $count_lines Lines)") -end - -## Parsing error detection -function isparsed(ex::Expr) - parse_error = Meta.isexpr(ex, :toplevel) && !isempty(ex.args) && - Meta.isexpr(ex.args[end], (:error, :incomplete)) - return !parse_error -end -function isparsed(ex::Nothing) - return false -end -function isparseerror(err::Exception) - return err isa Base.Meta.ParseError || - (err isa ErrorException && startswith(err.msg, "syntax:")) -end -function isparseerror(err::Nothing) - return false -end -function isparsed(cb::AICode) - return isparsed(cb.expression) && !isparseerror(cb.error) -end - -## Parsing Helpers -JULIA_EXPR_HEADS = [ - :block, - :quote, - :call, - :macrocall, - :(=), - :function, - :for, - :if, - :while, - :let, - :try, - :catch, - :finally, - :method, - :tuple, - :array, - :index, - :ref, - :., - :do, - :curly, - :typed_vcat, - :typed_hcat, - :typed_vcat, - :comprehension, - :generator, - :kw, - :where, -] -# Checks if the provided expression `ex` has some hallmarks of Julia code. Very naive! -# Serves as a quick check to avoid trying to eval output cells (```plaintext ... ```) -is_julia_expr(ex::Any) = false -function is_julia_expr(ex::Expr) - ## Expression itself - Meta.isexpr(ex, JULIA_EXPR_HEADS) && return true - ## Its arguments - for arg in ex.args - Meta.isexpr(arg, JULIA_EXPR_HEADS) && return true - end - ## Nothing found... - return false -end - -# Remove any given macro expression from the expression tree, used to remove tests -function remove_macro_expr!(expr, sym::Symbol = Symbol("@testset")) - if expr isa Expr - expr.args = filter(x -> !(x isa Expr && x.head == :macrocall && !isempty(x.args) && - x.args[1] == sym), - expr.args) - foreach(x -> remove_macro_expr!(x, sym), expr.args) - end - expr -end - -# Remove testsets and sets from the expression tree -function remove_tests_from_expr!(expr) - # Focus only on the three most common test macros - remove_macro_expr!(expr, Symbol("@testset")) - remove_macro_expr!(expr, Symbol("@test")) - remove_macro_expr!(expr, Symbol("@test_throws")) -end - -# Utility to identify the module name in a given expression (to evaluate subsequent calls in it) -function extract_module_name(expr) - if isa(expr, Expr) && expr.head == :module - return expr.args[2] # The second argument is typically the module name - elseif isa(expr, Expr) && !isempty(expr.args) - output = extract_module_name.(expr.args) - for item in output - if !isnothing(item) - return item - end - end - end - nothing -end - -## Check if a given String seems to be a valid Julia expression (simple heuristics) -function is_julia_code(code::AbstractString) - # Try to parse the expression, return false if parsing fails - expr = try - Meta.parseall(code) - catch - return false - end - - if isparsed(expr) && is_julia_expr(expr) - return true - else - return false - end -end - -## Overload for AIMessage - simply extracts the code blocks and concatenates them -function AICode(msg::AIMessage; - verbose::Bool = false, - skip_unsafe::Bool = false, - skip_invalid::Bool = false, - kwargs...) - code = extract_code_blocks(msg.content) - if isempty(code) - ## Fallback option for generic code fence, we must check if the content is parseable - code = extract_code_blocks_fallback(msg.content) - skip_invalid = true # set to true if we use fallback option - end - if skip_invalid - ## Filter out extracted code blocks that do not even parse - filter!(is_julia_code, code) - end - code = join(code, "\n") - skip_unsafe && (code = remove_unsafe_lines(code; verbose)) - return AICode(code; kwargs...) -end - -## # Functions - -# Utility to detect if Pkg.* is called in a string (for `safe` code evaluation) -function detect_pkg_operation(input::AbstractString) - m = match(r"\bPkg.[a-z]", input) - return !isnothing(m) -end -# Utility to detect dependencies in a string (for `safe` code evaluation / understand when we don't have a necessary package) -function extract_julia_imports(input::AbstractString) - package_names = Symbol[] - for line in split(input, "\n") - if occursin(r"(^using |^import )"m, line) - subparts = replace(replace(line, "using" => ""), "import" => "") - ## TODO: add split on . - subparts = map(x -> contains(x, ':') ? split(x, ':')[1] : x, - split(subparts, ",")) - subparts = replace(join(subparts, ' '), ',' => ' ') - packages = filter(x -> !isempty(x) && !startswith(x, "Base") && - !startswith(x, "Main"), - split(subparts, " ")) - append!(package_names, Symbol.(packages)) - end - end - return package_names -end - -# Utility to pinpoint unavailable dependencies -function detect_missing_packages(imports_required::AbstractVector{<:Symbol}) - # shortcut if no packages are required - isempty(imports_required) && return false, Symbol[] - # - available_packages = Base.loaded_modules |> values .|> Symbol - missing_packages = filter(pkg -> !in(pkg, available_packages), imports_required) - if length(missing_packages) > 0 - return true, missing_packages - else - return false, Symbol[] - end -end - -"Iterates over the lines of a string and removes those that contain a package operation or a missing import." -function remove_unsafe_lines(code::AbstractString; verbose::Bool = false) - io = IOBuffer() - for line in readlines(IOBuffer(code)) - if !detect_pkg_operation(line) && - !detect_missing_packages(extract_julia_imports(line))[1] - println(io, line) - else - verbose && @info "Unsafe line removed: $line" - end - end - return String(take!(io)) -end - -"Checks if a given string has a Julia prompt (`julia> `) at the beginning of a line." -has_julia_prompt(s::T) where {T <: AbstractString} = occursin(r"(:?^julia> |^> )"m, s) - -""" - remove_julia_prompt(s::T) where {T<:AbstractString} - -If it detects a julia prompt, it removes it and all lines that do not have it (except for those that belong to the code block). -""" -function remove_julia_prompt(s::T) where {T <: AbstractString} - if !has_julia_prompt(s) - return s - end - # Has julia prompt, so we need to parse it line by line - lines = split(s, '\n') - code_line = false - io = IOBuffer() - for line in lines - if startswith(line, r"^julia> ") - code_line = true - # remove the prompt - println(io, replace(line, "julia> " => "")) - elseif startswith(line, r"^> ") - code_line = true - # remove the prompt - println(io, replace(line, "> " => "")) - elseif code_line && startswith(line, r"^ ") - # continuation of the code line - println(io, line) - else - code_line = false - end - end - # strip removes training whitespace and newlines - String(take!(io)) |> strip -end - -# escape dollar sign only if not preceeded by backslash already, ie, unescaped -- use negative lookbehind -# Useful in cases where we have double nested interpolation, eg, string code -> has string literal -> function with interpolation inside it -escape_interpolation(s::AbstractString) = replace(s, r"(? String(['\\', '$'])) - -""" - find_subsequence_positions(subseq, seq) -> Vector{Int} - -Find all positions of a subsequence `subseq` within a larger sequence `seq`. Used to lookup positions of code blocks in markdown. - -This function scans the sequence `seq` and identifies all starting positions where the subsequence `subseq` is found. Both `subseq` and `seq` should be vectors of integers, typically obtained using `codeunits` on strings. - -# Arguments -- `subseq`: A vector of integers representing the subsequence to search for. -- `seq`: A vector of integers representing the larger sequence in which to search. - -# Returns -- `Vector{Int}`: A vector of starting positions (1-based indices) where the subsequence is found in the sequence. - -# Examples -```julia -find_subsequence_positions(codeunits("ab"), codeunits("cababcab")) # Returns [2, 5] -``` -""" -function find_subsequence_positions(subseq, seq) - positions = Int[] - len_subseq = length(subseq) - len_seq = length(seq) - lim = len_seq - len_subseq + 1 - cur = 1 - while cur <= lim - match = true - @inbounds for i in 1:len_subseq - if seq[cur + i - 1] != subseq[i] - match = false - break - end - end - if match - push!(positions, cur) - end - cur += 1 - end - return positions -end - -""" - extract_code_blocks(markdown_content::String) -> Vector{String} - -Extract Julia code blocks from a markdown string. - -This function searches through the provided markdown content, identifies blocks of code specifically marked as Julia code -(using the ```julia ... ``` code fence patterns), and extracts the code within these blocks. -The extracted code blocks are returned as a vector of strings, with each string representing one block of Julia code. - -Note: Only the content within the code fences is extracted, and the code fences themselves are not included in the output. - -See also: `extract_code_blocks_fallback` - -# Arguments -- `markdown_content::String`: A string containing the markdown content from which Julia code blocks are to be extracted. - -# Returns -- `Vector{String}`: A vector containing strings of extracted Julia code blocks. If no Julia code blocks are found, an empty vector is returned. - -# Examples - -Example with a single Julia code block -```julia -markdown_single = \""" -```julia -println("Hello, World!") -``` -\""" -extract_code_blocks(markdown_single) -# Output: [\"Hello, World!\"] -``` - -```julia -# Example with multiple Julia code blocks -markdown_multiple = \""" -```julia -x = 5 -``` -Some text in between -```julia -y = x + 2 -``` -\""" -extract_code_blocks(markdown_multiple) -# Output: ["x = 5", "y = x + 2"] -``` -""" -function extract_code_blocks(markdown_content::T) where {T <: AbstractString} - # Convert content and delimiters to codeunits - content_units = codeunits(markdown_content) - # Ideal code fences - start_delim_units1 = codeunits("\n```julia\n") - start_delim_units2 = codeunits("```julia\n") - start_delim_units3 = codeunits("```julia ") # happens to small models - end_delim_units1 = codeunits("\n```\n") - end_delim_units2 = codeunits("\n```") - - # Find all starting and ending positions of code blocks - pos = find_subsequence_positions(start_delim_units1, content_units) - pos2 = find_subsequence_positions(start_delim_units2, content_units) - pos3 = find_subsequence_positions(start_delim_units3, content_units) - # the +1 offset is because the first pattern starts 1 character earlier - start_positions = vcat(pos2, pos .+ 1, pos3) |> unique |> sort - - pos = find_subsequence_positions(end_delim_units1, content_units) - pos2 = find_subsequence_positions(end_delim_units2, content_units) - end_positions = vcat(pos, pos2) |> unique - unused_end_positions = trues(length(end_positions)) - - # Generate code block position pairs - block_positions = Tuple{Int, Int}[] - for start_pos in reverse(start_positions) - for (i, end_pos) in enumerate(end_positions) - if end_pos > start_pos && unused_end_positions[i] - push!(block_positions, (start_pos, end_pos)) - unused_end_positions[i] = false - break - end - end - end - - # Filter out nested blocks (only if they have full overlap) - filtered_positions = filter(inner -> !any(outer -> (outer[1] < inner[1]) && - (inner[2] < outer[2]), - block_positions), - block_positions) - - # Extract code blocks - eltype_ = typeof(@view(markdown_content[begin:end])) - code_blocks = Vector{eltype_}() - for (start_pos, end_pos) in filtered_positions - start_ = (start_pos + length(start_delim_units2)) - end_ = prevind(markdown_content, end_pos) - code_block = markdown_content[start_:end_] - # Also remove the julia prompt - push!(code_blocks, remove_julia_prompt(strip(code_block))) - end - - return reverse(code_blocks) # Reverse to maintain original order -end - -""" - extract_code_blocks_fallback(markdown_content::String, delim::AbstractString="\\n```\\n") - -Extract Julia code blocks from a markdown string using a fallback method (splitting by arbitrary `delim`-iters). -Much more simplistic than `extract_code_blocks` and does not support nested code blocks. - -It is often used as a fallback for smaller LLMs that forget to code fence ```julia ... ```. - -# Example - -```julia -code = \"\"\" -\`\`\` -println("hello") -\`\`\` - -Some text - -\`\`\` -println("world") -\`\`\` -\"\"\" - -# We extract text between triple backticks and check each blob if it looks like a valid Julia code -code_parsed = extract_code_blocks_fallback(code) |> x -> filter(is_julia_code, x) |> x -> join(x, "\n") -``` -""" -function extract_code_blocks_fallback(markdown_content::T, - delim::AbstractString = "\n```\n") where {T <: AbstractString} - # Convert content and delimiters to codeunits - content_units = codeunits(markdown_content) - content_length = length(content_units) - delim_units = codeunits(delim) - delim_positions = find_subsequence_positions(delim_units, content_units) - - # Extract code blocks - eltype_ = typeof(@view(markdown_content[begin:end])) - code_blocks = Vector{eltype_}() - isempty(delim_positions) && !startswith(markdown_content, lstrip(delim)) && - return code_blocks - - # Run the extraction - # catch if we're missing the opening mark because of document start - no_newline_start = lstrip(delim) - start_pos = if no_newline_start != delim && - startswith(markdown_content, no_newline_start) - (length(codeunits(no_newline_start)) - length(delim_units)) - else - delim_positions[1] - end - no_new_line_end = rstrip(delim) - if no_new_line_end != delim && endswith(markdown_content, no_new_line_end) - last_end = 1 + content_length - length(codeunits(no_new_line_end)) - push!(delim_positions, last_end) - end - # start the iteration - for end_pos in unique(delim_positions) - if end_pos > start_pos && end_pos <= content_length - end_ = prevind(markdown_content, end_pos) - code_block = markdown_content[(start_pos + length(delim_units)):end_] - # Also remove the julia prompt - push!(code_blocks, remove_julia_prompt(strip(code_block))) - # Reset the start - start_pos = end_pos - end - end - - return code_blocks -end - -""" - extract_function_name(code_block::String) -> Union{String, Nothing} - -Extract the name of a function from a given Julia code block. The function searches for two patterns: -- The explicit function declaration pattern: `function name(...) ... end` -- The concise function declaration pattern: `name(...) = ...` - -If a function name is found, it is returned as a string. If no function name is found, the function returns `nothing`. - -# Arguments -- `code_block::String`: A string containing Julia code. - -# Returns -- `Union{String, Nothing}`: The extracted function name or `nothing` if no name is found. - -# Example -```julia -code = \""" -function myFunction(arg1, arg2) - # Function body -end -\""" -extract_function_name(code) -# Output: "myFunction" -``` -""" -function extract_function_name(code_block::AbstractString) - # Regular expression for the explicit function declaration - pattern_explicit = r"function\s+(\w+)\(" - # Regular expression for the concise function declaration - pattern_concise = r"^(\w+)\(.*\)\s*=" - - # Searching for the explicit function declaration - match_explicit = match(pattern_explicit, code_block) - if match_explicit !== nothing - return match_explicit.captures[1] - end - - # Searching for the concise function declaration - match_concise = match(pattern_concise, code_block) - if match_concise !== nothing - return match_concise.captures[1] - end - - # Return nothing if no function name is found - return nothing -end - -""" - eval!(cb::AbstractCodeBlock; - safe_eval::Bool = true, - capture_stdout::Bool = true, - prefix::AbstractString = "", - suffix::AbstractString = "") - -Evaluates a code block `cb` in-place. It runs automatically when AICode is instantiated with a String. - -Check the outcome of evaluation with `Base.isvalid(cb)`. If `==true`, provide code block has executed successfully. - -Steps: -- If `cb::AICode` has not been evaluated, `cb.success = nothing`. - After the evaluation it will be either `true` or `false` depending on the outcome -- Parse the text in `cb.code` -- Evaluate the parsed expression -- Capture outputs of the evaluated in `cb.output` -- [OPTIONAL] Capture any stdout outputs (eg, test failures) in `cb.stdout` -- If any error exception is raised, it is saved in `cb.error` -- Finally, if all steps were successful, success is set to `cb.success = true` - -# Keyword Arguments -- `safe_eval::Bool`: If `true`, we first check for any Pkg operations (eg, installing new packages) and missing imports, - then the code will be evaluated inside a bespoke scratch module (not to change any user variables) -- `capture_stdout::Bool`: If `true`, we capture any stdout outputs (eg, test failures) in `cb.stdout` -- `prefix::AbstractString`: A string to be prepended to the code block before parsing and evaluation. - Useful to add some additional code definition or necessary imports. Defaults to an empty string. -- `suffix::AbstractString`: A string to be appended to the code block before parsing and evaluation. - Useful to check that tests pass or that an example executes. Defaults to an empty string. -""" -function eval!(cb::AbstractCodeBlock; - safe_eval::Bool = true, - capture_stdout::Bool = true, - prefix::AbstractString = "", - suffix::AbstractString = "", - remove_tests::Bool = false) - (; code) = cb - # reset - cb.success = nothing - cb.error = nothing - cb.expression = nothing - cb.output = nothing - - code_extra = if safe_eval - safe_module = string(gensym("SafeCustomModule")) |> - x -> replace(x, "#" => "") - string("module $safe_module \nusing Test\n", - prefix, - "\n", - code, - "\n", - suffix, - "\nend") - else - string(prefix, "\n", code, "\n", suffix) - end - ## Safety checks on `code` only -- treat it as a parsing failure - if safe_eval - detected, missing_packages = detect_missing_packages(extract_julia_imports(code)) - if detect_pkg_operation(code) || detected - cb.success = false - detect_pkg_operation(code) && - (cb.error = ErrorException("Safety Error: Use of package manager (`Pkg.*`) detected! Please verify the safety of the code or disable the safety check (`safe_eval=false`)")) - detected && - (cb.error = ErrorException("Safety Error: Failed package import. Missing packages: $(join(string.(missing_packages),", ")). Please add them or disable the safety check (`safe_eval=false`)")) - return cb - end - end - ## Catch bad code extraction - if isempty(code) - cb.error = ErrorException("Parse Error: No code found!") - cb.success = false - return cb - end - ## Parse into an expression - try - ex = Meta.parseall(code_extra) - cb.expression = ex - catch e - cb.error = e - cb.success = false - return cb - end - - ## Remove any tests - if remove_tests - ex = remove_tests_from_expr!(ex) - end - - ## Eval - eval!(cb, cb.expression; capture_stdout, eval_module = Main) - return cb -end - -# Evaluation of any arbitrary expression with result recorded in `cb` -function eval!(cb::AbstractCodeBlock, expr::Expr; - capture_stdout::Bool = true, - eval_module::Module = Main) - # Prepare to catch any stdout - if capture_stdout - pipe = Pipe() - redirect_stdout(pipe) do - try - cb.output = @eval(eval_module, $(expr)) - cb.success = true - catch e - cb.error = e - cb.success = false - end - end - close(Base.pipe_writer(pipe)) - cb.stdout = read(pipe, String) - else - # Ignore stdout, just eval - try - cb.output = @eval(eval_module, $(expr)) - cb.success = true - catch e - cb.error = e - cb.success = false - end - end - return cb -end diff --git a/src/code_parsing.jl b/src/code_parsing.jl new file mode 100644 index 000000000..f5562b9f1 --- /dev/null +++ b/src/code_parsing.jl @@ -0,0 +1,415 @@ +## Check if a given String seems to be a valid Julia expression (simple heuristics) +function is_julia_code(code::AbstractString) + # Try to parse the expression, return false if parsing fails + expr = try + Meta.parseall(code) + catch + return false + end + + if isparsed(expr) && is_julia_expr(expr) + return true + else + return false + end +end + +# Utility to detect if Pkg.* is called in a string (for `safe` code evaluation) +function detect_pkg_operation(input::AbstractString) + m = match(r"^\s*\bPkg.[a-z]"ms, input) + return !isnothing(m) +end +# Utility to detect dependencies in a string (for `safe` code evaluation / understand when we don't have a necessary package) +function extract_julia_imports(input::AbstractString) + package_names = Symbol[] + for line in split(input, "\n") + if occursin(r"(^using |^import )"m, line) + subparts = replace(replace(line, "using" => ""), "import" => "") + ## TODO: add split on . + subparts = map(x -> contains(x, ':') ? split(x, ':')[1] : x, + split(subparts, ",")) + subparts = replace(join(subparts, ' '), ',' => ' ') + packages = filter(x -> !isempty(x) && !startswith(x, "Base") && + !startswith(x, "Main"), + split(subparts, " ")) + append!(package_names, Symbol.(packages)) + end + end + return package_names +end + +# Utility to pinpoint unavailable dependencies +function detect_missing_packages(imports_required::AbstractVector{<:Symbol}) + # shortcut if no packages are required + isempty(imports_required) && return false, Symbol[] + # + available_packages = Base.loaded_modules |> values .|> Symbol + dependencies = Symbol[Symbol(p.name) for p in values(Pkg.dependencies())] + missing_packages = Symbol[] + for pkg in imports_required + if !(pkg in available_packages || pkg in dependencies || hasproperty(Base, pkg) || + hasproperty(Main, pkg)) + push!(missing_packages, pkg) + end + end + + if length(missing_packages) > 0 + return true, missing_packages + else + return false, missing_packages + end +end + +"Iterates over the lines of a string and removes those that contain a package operation or a missing import." +function remove_unsafe_lines(code::AbstractString; verbose::Bool = false) + io_keep, io_remove = IOBuffer(), IOBuffer() + for line in readlines(IOBuffer(code)) + if !detect_pkg_operation(line) && + !detect_missing_packages(extract_julia_imports(line))[1] + println(io_keep, line) + else + verbose && @info "Unsafe line removed: $line" + println(io_remove, line) + end + end + return String(take!(io_keep)), String(take!(io_remove)) +end + +"Checks if a given string has a Julia prompt (`julia> `) at the beginning of a line." +has_julia_prompt(s::T) where {T <: AbstractString} = occursin(r"(:?^julia> |^> )"m, s) + +""" + remove_julia_prompt(s::T) where {T<:AbstractString} + +If it detects a julia prompt, it removes it and all lines that do not have it (except for those that belong to the code block). +""" +function remove_julia_prompt(s::T) where {T <: AbstractString} + if !has_julia_prompt(s) + return s + end + # Has julia prompt, so we need to parse it line by line + lines = split(s, '\n') + code_line = false + io = IOBuffer() + for line in lines + if startswith(line, r"^julia> ") + code_line = true + # remove the prompt + println(io, replace(line, "julia> " => "")) + elseif startswith(line, r"^> ") + code_line = true + # remove the prompt + println(io, replace(line, "> " => "")) + elseif code_line && startswith(line, r"^ ") + # continuation of the code line + println(io, line) + else + code_line = false + end + end + # strip removes training whitespace and newlines + String(take!(io)) |> strip +end + +# escape dollar sign only if not preceeded by backslash already, ie, unescaped -- use negative lookbehind +# Useful in cases where we have double nested interpolation, eg, string code -> has string literal -> function with interpolation inside it +escape_interpolation(s::AbstractString) = replace(s, r"(? String(['\\', '$'])) + +""" + find_subsequence_positions(subseq, seq) -> Vector{Int} + +Find all positions of a subsequence `subseq` within a larger sequence `seq`. Used to lookup positions of code blocks in markdown. + +This function scans the sequence `seq` and identifies all starting positions where the subsequence `subseq` is found. Both `subseq` and `seq` should be vectors of integers, typically obtained using `codeunits` on strings. + +# Arguments +- `subseq`: A vector of integers representing the subsequence to search for. +- `seq`: A vector of integers representing the larger sequence in which to search. + +# Returns +- `Vector{Int}`: A vector of starting positions (1-based indices) where the subsequence is found in the sequence. + +# Examples +```julia +find_subsequence_positions(codeunits("ab"), codeunits("cababcab")) # Returns [2, 5] +``` +""" +function find_subsequence_positions(subseq, seq) + positions = Int[] + len_subseq = length(subseq) + len_seq = length(seq) + lim = len_seq - len_subseq + 1 + cur = 1 + while cur <= lim + match = true + @inbounds for i in 1:len_subseq + if seq[cur + i - 1] != subseq[i] + match = false + break + end + end + if match + push!(positions, cur) + end + cur += 1 + end + return positions +end + +""" + extract_code_blocks(markdown_content::String) -> Vector{String} + +Extract Julia code blocks from a markdown string. + +This function searches through the provided markdown content, identifies blocks of code specifically marked as Julia code +(using the ```julia ... ``` code fence patterns), and extracts the code within these blocks. +The extracted code blocks are returned as a vector of strings, with each string representing one block of Julia code. + +Note: Only the content within the code fences is extracted, and the code fences themselves are not included in the output. + +See also: `extract_code_blocks_fallback` + +# Arguments +- `markdown_content::String`: A string containing the markdown content from which Julia code blocks are to be extracted. + +# Returns +- `Vector{String}`: A vector containing strings of extracted Julia code blocks. If no Julia code blocks are found, an empty vector is returned. + +# Examples + +Example with a single Julia code block +```julia +markdown_single = \""" +```julia +println("Hello, World!") +``` +\""" +extract_code_blocks(markdown_single) +# Output: [\"Hello, World!\"] +``` + +```julia +# Example with multiple Julia code blocks +markdown_multiple = \""" +```julia +x = 5 +``` +Some text in between +```julia +y = x + 2 +``` +\""" +extract_code_blocks(markdown_multiple) +# Output: ["x = 5", "y = x + 2"] +``` +""" +function extract_code_blocks(markdown_content::T) where {T <: AbstractString} + # Convert content and delimiters to codeunits + content_units = codeunits(markdown_content) + # Ideal code fences + start_delim_units1 = codeunits("\n```julia\n") + start_delim_units2 = codeunits("```julia\n") + start_delim_units3 = codeunits("```julia ") # happens to small models + end_delim_units1 = codeunits("\n```\n") + end_delim_units2 = codeunits("\n```") + + # Find all starting and ending positions of code blocks + pos = find_subsequence_positions(start_delim_units1, content_units) + pos2 = find_subsequence_positions(start_delim_units2, content_units) + pos3 = find_subsequence_positions(start_delim_units3, content_units) + # the +1 offset is because the first pattern starts 1 character earlier + start_positions = vcat(pos2, pos .+ 1, pos3) |> unique |> sort + + pos = find_subsequence_positions(end_delim_units1, content_units) + pos2 = find_subsequence_positions(end_delim_units2, content_units) + end_positions = vcat(pos, pos2) |> unique + unused_end_positions = trues(length(end_positions)) + + # Generate code block position pairs + block_positions = Tuple{Int, Int}[] + for start_pos in reverse(start_positions) + for (i, end_pos) in enumerate(end_positions) + if end_pos > start_pos && unused_end_positions[i] + push!(block_positions, (start_pos, end_pos)) + unused_end_positions[i] = false + break + end + end + end + + # Filter out nested blocks (only if they have full overlap) + filtered_positions = filter(inner -> !any(outer -> (outer[1] < inner[1]) && + (inner[2] < outer[2]), + block_positions), + block_positions) + + # Extract code blocks + eltype_ = typeof(@view(markdown_content[begin:end])) + code_blocks = Vector{eltype_}() + for (start_pos, end_pos) in filtered_positions + start_ = (start_pos + length(start_delim_units2)) + end_ = prevind(markdown_content, end_pos) + code_block = markdown_content[start_:end_] + # Also remove the julia prompt + push!(code_blocks, remove_julia_prompt(strip(code_block))) + end + + return reverse(code_blocks) # Reverse to maintain original order +end + +""" + extract_code_blocks_fallback(markdown_content::String, delim::AbstractString="\\n```\\n") + +Extract Julia code blocks from a markdown string using a fallback method (splitting by arbitrary `delim`-iters). +Much more simplistic than `extract_code_blocks` and does not support nested code blocks. + +It is often used as a fallback for smaller LLMs that forget to code fence ```julia ... ```. + +# Example + +```julia +code = \"\"\" +\`\`\` +println("hello") +\`\`\` + +Some text + +\`\`\` +println("world") +\`\`\` +\"\"\" + +# We extract text between triple backticks and check each blob if it looks like a valid Julia code +code_parsed = extract_code_blocks_fallback(code) |> x -> filter(is_julia_code, x) |> x -> join(x, "\n") +``` +""" +function extract_code_blocks_fallback(markdown_content::T, + delim::AbstractString = "\n```\n") where {T <: AbstractString} + # Convert content and delimiters to codeunits + content_units = codeunits(markdown_content) + content_length = length(content_units) + delim_units = codeunits(delim) + delim_positions = find_subsequence_positions(delim_units, content_units) + + # Extract code blocks + eltype_ = typeof(@view(markdown_content[begin:end])) + code_blocks = Vector{eltype_}() + isempty(delim_positions) && !startswith(markdown_content, lstrip(delim)) && + return code_blocks + + # Run the extraction + # catch if we're missing the opening mark because of document start + no_newline_start = lstrip(delim) + start_pos = if no_newline_start != delim && + startswith(markdown_content, no_newline_start) + (length(codeunits(no_newline_start)) - length(delim_units)) + else + delim_positions[1] + end + no_new_line_end = rstrip(delim) + if no_new_line_end != delim && endswith(markdown_content, no_new_line_end) + last_end = 1 + content_length - length(codeunits(no_new_line_end)) + push!(delim_positions, last_end) + end + # start the iteration + for end_pos in unique(delim_positions) + if end_pos > start_pos && end_pos <= content_length + end_ = prevind(markdown_content, end_pos) + code_block = markdown_content[(start_pos + length(delim_units)):end_] + # Also remove the julia prompt + push!(code_blocks, remove_julia_prompt(strip(code_block))) + # Reset the start + start_pos = end_pos + end + end + + return code_blocks +end + +""" + extract_function_name(code_block::String) -> Union{String, Nothing} + +Extract the name of a function from a given Julia code block. The function searches for two patterns: +- The explicit function declaration pattern: `function name(...) ... end` +- The concise function declaration pattern: `name(...) = ...` + +If a function name is found, it is returned as a string. If no function name is found, the function returns `nothing`. + +# Arguments +- `code_block::String`: A string containing Julia code. + +# Returns +- `Union{String, Nothing}`: The extracted function name or `nothing` if no name is found. + +# Example +```julia +code = \""" +function myFunction(arg1, arg2) + # Function body +end +\""" +extract_function_name(code) +# Output: "myFunction" +``` +""" +function extract_function_name(code_block::AbstractString) + # Regular expression for the explicit function declaration + pattern_explicit = r"function\s+(\w+)\(" + # Regular expression for the concise function declaration + pattern_concise = r"^(\w+)\(.*\)\s*=" + + # Searching for the explicit function declaration + match_explicit = match(pattern_explicit, code_block) + if match_explicit !== nothing + return match_explicit.captures[1] + end + + # Searching for the concise function declaration + match_concise = match(pattern_concise, code_block) + if match_concise !== nothing + return match_concise.captures[1] + end + + # Return nothing if no function name is found + return nothing +end + +function extract_testset_name(testset_str::AbstractString) + # Define a regex pattern to match the function name + pattern = r"^\s*@testset\s*\"([^\"]+)\"\s* begin"ms + + # Search for the pattern in the test set string + match_result = match(pattern, testset_str) + + # Check if a match was found and return the captured group + result = if match_result !== nothing + match_result.captures[1] + else + nothing + end + return result +end + +function extract_package_name_from_argerror(error_msg::AbstractString) + # Define a regex pattern to match the package name + pattern = r"^Package\s+([^\s]+)\s+not found" + + # Search for the pattern in the error message + match_result = match(pattern, error_msg) + + # Check if a match was found and return the captured group + !isnothing(match_result) ? match_result.captures[1] : nothing +end + +# overload for missing stdout +extract_stacktrace_lines(filename::String, stacktrace::Nothing) = Int[] +function extract_stacktrace_lines(filename::String, stacktrace::String) + # Pattern to match the filename and line numbers + pattern = Regex(escape_string(filename) * ":(\\d+)") + + # Extracting line numbers from the matches + line_numbers = Int[parse(Int, m.captures[1]) for m in eachmatch(pattern, stacktrace)] + + return line_numbers +end diff --git a/test/Experimental/AgentTools/code_feedback.jl b/test/Experimental/AgentTools/code_feedback.jl index e2a5e1847..806818dff 100644 --- a/test/Experimental/AgentTools/code_feedback.jl +++ b/test/Experimental/AgentTools/code_feedback.jl @@ -1,6 +1,8 @@ using PromptingTools.Experimental.AgentTools: aicodefixer_feedback using PromptingTools.Experimental.AgentTools: CodeEmpty, CodeFailedParse, CodeFailedEval, CodeFailedTimeout, CodeSuccess +using PromptingTools.Experimental.AgentTools: testset_feedback, + error_feedback, score_feedback, extract_test_counts @testset "aicodefixer_feedback" begin # Empty code @@ -31,12 +33,12 @@ using PromptingTools.Experimental.AgentTools: CodeEmpty, cb.stdout = "STDOUT" feedback = aicodefixer_feedback(CodeFailedEval(), cb) @test feedback == - "**Error Detected:** ErrorException(\"xx\")\n\n**Output Captured:** STDOUT" + "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- fetch(tsk)\n\n**Output Captured:**\n STDOUT" cb = AICode("error(\"xx\")") cb.stdout = "STDOUT" feedback = aicodefixer_feedback(CodeFailedEval(), cb) @test feedback == - "**Error Detected:** ErrorException(\"xx\")\n\n**Output Captured:** STDOUT" + "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- error(\"xx\")\n\n**Output Captured:**\n STDOUT" conv = [PT.AIMessage(""" ```julia error(\"xx\") @@ -44,7 +46,7 @@ using PromptingTools.Experimental.AgentTools: CodeEmpty, """)] feedback = aicodefixer_feedback(conv).feedback @test feedback == - "**Error Detected:** ErrorException(\"xx\")" + "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- error(\"xx\")" # CodeFailedTimeout cb = AICode("InterruptException()") @@ -67,3 +69,178 @@ using PromptingTools.Experimental.AgentTools: CodeEmpty, @test occursin("Execution has been successful", feedback) @test occursin("**Output Captured:**", feedback) end + +@testset "testset_feedback" begin + # Test case 1: Test testset_feedback with valid test set name + msg = AIMessage(""" + ```julia + @testset "tester" begin + @test 1 == 1 + end + ``` + """) + expected_feedback = nothing + @test testset_feedback(msg) == expected_feedback + # Test case 2: Test testset_feedback with invalid test set name + msg = AIMessage(""" + ```julia + @testset "tester" begin + func( + @test 1 == 1 + end + ``` + """) + expected_feedback = CodeFailedEval() + feedback = testset_feedback(msg) + @test occursin("**Error Detected:**\n**", feedback) + # Mock some function + msg = AIMessage(""" + ```julia + @testset "tester" begin + tester() + @test 1 == 1 + end + ``` + """) + @test testset_feedback(msg) == nothing +end + +@testset "error_feedback" begin + # Test case 1: Test error feedback with package name + e = ArgumentError("Package Threads not found in current path, maybe you meant `import/using .Threads`.\n- Otherwise, run `import Pkg; Pkg.add(\"Threads\")` to install the Threads package.") + expected_feedback = "ArgumentError: Package Threads not found in current path, maybe you meant `import/using .Threads`.\n- Otherwise, run `import Pkg; Pkg.add(\"Threads\")` to install the Threads package.\nExpert Tip: I know that the package Threads is defined in Base module. You MUST use `import Base.Threads` to use it." + @test error_feedback(e) == expected_feedback + + # Test case 2: Test error feedback without package name + e = ArgumentError("Invalid argument") + expected_feedback = "ArgumentError: Invalid argument" + @test error_feedback(e) == expected_feedback + + # Test case 1: Test error_feedback with defined variable + e = UndefVarError(:Threads) + expected_output = "UndefVarError: `Threads` not defined\nExpert Tip: I know that the variable Threads is defined in Base module. Use `import Base.Threads` to use it." + @test error_feedback(e) == expected_output + + # Test case 2: Test error_feedback with undefined variable + e = UndefVarError(:SomeVariable) + expected_output = "UndefVarError: `SomeVariable` not defined\nTip: Does it even exist? Does it need to be imported? Or is it a typo?" + @test error_feedback(e) == expected_output + + # Test case 1: Test error_feedback with valid input + e = Base.Meta.ParseError("SyntaxError: unexpected symbol \"(\"") + output = error_feedback(e) + @test occursin("**ParseError**", output) + + # Test case 2: Test error_feedback with custom max_length + e = Base.Meta.ParseError("SyntaxError: unexpected symbol \"(\"") + output = error_feedback(e; max_length = 15) + @test occursin("**ParseError**", output) + + # Test case 4: Test error_feedback function + e = @task error("Error message") + schedule(e) + expected_output = "**ErrorException**:\nError message" + @test error_feedback(e) == expected_output + + # No error + e = @task a = 1 + schedule(e) + @test error_feedback(e) == "No error found. Ignore." + + ## Testsetexception + cb = AICode(""" + @testset "x" begin + a + a + @test x == 2 + end + """) + output = error_feedback(cb.error) + @test occursin("**TestSetException**:\nSome tests did not pass", output) + @test occursin("UndefVarError: `a` not defined", output) + + # Test case 1: Test error_feedback with no error + expected_output = "No error found. Ignore." + @test error_feedback(expected_output) == expected_output + + # Test case 2: Test error_feedback with Exception + e = ErrorException("Test exception") + expected_output = "**ErrorException**:\nTest exception" + @test error_feedback(e) == expected_output + + # Test case 3: Test error_feedback with Exception and max_length + e = ErrorException("Test exception") + expected_output = "**ErrorException**:\nTest exception" + max_length = 10 + @test error_feedback(e; max_length) == expected_output[1:10] +end + +@testset "score_feedback" begin + # Test case 1: Test score_feedback with empty code + cb = AICode("") + @test score_feedback(cb) == 0 + + # Test case 2: Test score_feedback with unparsed code + cb = AICode("x ===== 1") + @test score_feedback(cb) == 1 + + # Test case 3: Test score_feedback with TestSetException error + cb = AICode(""" + @testset "x" begin + x = 1 + @test x == 2 + end + """) + @test score_feedback(cb) == 9 + + # Test case 6: Test score_feedback with invalid code feedback path + cb = AICode(""" + @testset "x" begin + x = 1 + @test x == 2 + @test x == 1 + error("a") + end + """) + @test score_feedback(cb) == 8 + + # normal exception + cb = AICode(""" + error("a") + """) + @test score_feedback(cb) == 2 +end + +@testset "extract_test_counts" begin + test_summary1 = """ + Test Summary: | Pass Broken Total Time + a | 1 1 2 0.0s + """ + @test extract_test_counts(test_summary1) == + Dict("pass" => 1, "broken" => 1, "total" => 2) + + test_summary2 = """ + Test Summary: | Pass Fail Error Total Time + b | 1 1 1 3 0.0s + """ + @test extract_test_counts(test_summary2) == + Dict("pass" => 1, "fail" => 1, "error" => 1, "total" => 3) + + test_summary3 = """ + Test Summary: | Pass Fail Error Broken Total Time + two | 2 1 1 1 5 0.0s + a | 1 1 2 0.0s + b | 1 1 1 3 0.0s + """ + @test extract_test_counts(test_summary3) == + Dict("fail" => 1, "error" => 1, "total" => 5, "broken" => 1, "pass" => 2) + + test_summary4 = """ + Test Summary: | Pass Broken Total Time + a | 1 1 2 0.0s + + Test Summary: | Pass Fail Error Total Time + b | 1 1 1 3 0.0s + """ + @test extract_test_counts(test_summary4) == + Dict("pass" => 2, "broken" => 1, "fail" => 1, "error" => 1, "total" => 5) +end \ No newline at end of file diff --git a/test/code_eval.jl b/test/code_eval.jl new file mode 100644 index 000000000..6200ea65b --- /dev/null +++ b/test/code_eval.jl @@ -0,0 +1,372 @@ +using PromptingTools: extract_code_blocks, extract_code_blocks_fallback, eval! +using PromptingTools: AICode, isparsed, isparseerror, is_julia_code, is_julia_expr +using PromptingTools: extract_module_name + +@testset "eval!" begin + # Test that it captures stdout and output + let cb = AICode(; code = """ + println("Hello") + a=1 + """) + eval!(cb) + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test isvalid(cb) + @test cb.stdout == "Hello\n" + @test cb.output.a == 1 + end + # Test that it captures parsing errors + let cb = AICode(; code = """ + a=1 + + mla;sda b=2 + """) + eval!(cb) + @test cb.success == false + @test !isvalid(cb) + @test cb.error isa Exception # can be Base.Meta.ParseError or ErrorException depending on Julia version + end + # Test that it captures execution errors + let cb = AICode(; code = """ + a=1 + b # b not defined yet + b=2 + """) + eval!(cb) + @test cb.success == false + @test cb.error == UndefVarError(:b) + @test !isnothing(cb.expression) # parsed + end + + # expression-based eval! + cb = AICode(; code = """ + a=1 + b # b not defined yet + b=2 + """) + cb = eval!(cb) + eval!(cb, cb.expression; capture_stdout = false) + @test cb.success == false + @test cb.error == UndefVarError(:b) + @test cb.error_lines == [1] +end + +## Addition, needs to be outside of @testset +# Test that it captures test failures, we need to move it to the main file as it as it doesn't work inside a testset +# let cb = AICode(; code = """ +# @test 1==2 +# """) +# eval!(cb) +# @test cb.success == false +# @info cb.error cb.output +# @test cb.error isa Test.FallbackTestSetException +# @test !isnothing(cb.expression) # parsed +# @test occursin("Test Failed", cb.stdout) # capture details of the test failure +# @test isnothing(cb.output) # because it failed +# end + +@testset "eval! kwargs" begin + ## Safe Eval == true mode + # package that is not available + cb = AICode(; code = "using ExoticPackage123") |> eval! + @test cb.error isa Exception + @test occursin("Safety Error", cb.error.msg) + @test occursin("ExoticPackage123", cb.error.msg) + # Pkg operations + cb = AICode(; code = "Pkg.activate(\".\")") |> eval! + @test cb.error isa Exception + @test occursin("Safety Error", cb.error.msg) + @test occursin("Use of package manager ", cb.error.msg) + + # Evaluate inside a gensym'd module + cb = AICode(; code = "a=1") |> eval! + @test occursin("SafeMod", string(cb.output)) + + ## Safe Eval == false mode + # package that is not available + cb = AICode(; code = "using ExoticPackage123") + eval!(cb; safe_eval = false) + @test !isvalid(cb) + @test cb.error isa ArgumentError # now it's caught by REPL that we don't have the package + # Pkg operations + cb = AICode(; code = "import Pkg; Pkg.status()") + eval!(cb; safe_eval = false) + # This works but in test mode, Julia claims it doesn't have Pkg package... + @test isvalid(cb) + # Evaluate in Main directly + cb = AICode(; code = "a123=123") + eval!(cb; safe_eval = false) + @test cb.output == 123 + @test a123 == 123 + + cb = AICode(""" + module MyModule123 + function foo() + println("Hello") + end + end + """; safe_eval = false) + @test cb.output == Main.MyModule123 + + # Check that empty code is invalid + cb = AICode("") + @test !isvalid(cb) + @test cb.error isa Exception + + # Test prefix and suffix + cb = AICode(; code = "x=1") + eval!(cb; prefix = "a=1", suffix = "b=2") + @test cb.output.a == 1 + @test cb.output.b == 2 + + # Whether to capture stdout + cb = AICode(; code = "println(\"Hello\")") + eval!(cb; capture_stdout = false) + @test cb.stdout == nothing + @test cb.code == "println(\"Hello\")" + @test isvalid(cb) + + eval!(cb; capture_stdout = true) + @test cb.stdout == "Hello\n" + @test cb.code == "println(\"Hello\")" + @test isvalid(cb) + + # Test execution_timeout + cb = AICode("sleep(1.1)", execution_timeout = 1) + @test cb.success == false + @test isnothing(cb.output) + @test cb.error isa InterruptException + cb = AICode("sleep(1.1)", execution_timeout = 2) + @test cb.success == true + @test isnothing(cb.error) + + # expression-only method + cb = AICode(""" + module MyModule + function foo() + println("Hello") + end + end + """; safe_eval = false) + eval_module = cb.output isa Module ? cb.output : + getfield(Main, extract_module_name(cb.expression)) + eval!(cb, Meta.parseall("foo()"); eval_module) + @test isnothing(cb.error) + @test cb.stdout == "Hello\n" + cb = AICode(""" + function foo() + println("Hello") + end + """; safe_eval = true) + eval_module = cb.output isa Module ? cb.output : + getfield(Main, extract_module_name(cb.expression)) + eval!(cb, Meta.parseall("foo()"); eval_module) + @test isnothing(cb.error) + @test cb.stdout == "Hello\n" + + # Expression transformation + cb = AICode(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + @test y == 2 + end + @test x == 3 + @test_throws AssertionError func(1) + y = 3 + 3 + """; expression_transform = :nothing) + @test occursin("Example Tests", cb.stdout) + @test occursin("y == 2", cb.stdout) + + cb = AICode(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + @test y == 2 + end + @test x == 3 + @test_throws AssertionError func(1) + y = 3 + 3 + """; expression_transform = :remove_all_tests) + @test !occursin("Example Tests", cb.stdout) + @test !occursin("y == 2", cb.stdout) + @test !occursin("func(1)", cb.stdout) + @test cb.stdout == "" + @test isvalid(cb) + println(cb.stdout) + + cb = AICode(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + @test y == 2 + end + @test x == 3 + @test_throws AssertionError func(1) + y = 3 + 3 + """; expression_transform = :remove_test_items) + @test occursin("Example Tests", cb.stdout) + @test !occursin("y == 2", cb.stdout) + @test !occursin("func(1)", cb.stdout) + @test cb.stdout != "" + @test isvalid(cb) +end + +@testset "AICode constructors" begin + # Initiate from provided text + let cb = AICode(""" + println("Hello") + a=1 + """) + # eval! is automatic + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test cb.stdout == "Hello\n" + @test cb.output.a == 1 + end + + # Test auto-eval=false + let cb = AICode(""" + println("Hello") + a=1 + """; auto_eval = false) + # eval! is automatic + @test isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == nothing + end + + # From AI Message + let msg = AIMessage(""" +```julia +println(\"hello\") +``` +Some text +```julia +println(\"world\") +b=2 +``` +""") + cb = AICode(msg) + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test cb.stdout == "hello\nworld\n" + @test cb.output.b == 2 + end + + # Fallback extraction method + let msg = AIMessage(""" +``` +println(\"hello\") +``` +Some text +``` +println(\"world\") +b=2 +``` +""") + cb = AICode(msg) + @test !isnothing(cb.expression) + @test isnothing(cb.error) + @test cb.success == true + @test cb.stdout == "hello\nworld\n" + @test cb.output.b == 2 + end + + # skip_unsafe=true + let msg = AIMessage(""" + ```julia + a=1 + Pkg.add("a") + b=2 + Pkg.add("b") + using 12315456NotExisting + ``` + """) + cb = AICode(msg; skip_unsafe = true) + @test cb.code == "a=1\nb=2\n" + + # dispatch on text + code = extract_code_blocks(msg.content) |> x -> join(x, "\n") + cb = AICode(code; skip_unsafe = true) + @test cb.code == "a=1\nb=2\n" + end + + # skip_invalid=true + let msg = AIMessage(""" + ```julia + println("Hello world!") + ``` + + ```julia + println("Hello world!) # missing quote + ``` + """) + cb = AICode(msg; skip_invalid = true) + @test cb.code == "println(\"Hello world!\")" + + # if it's not switched on + cb = AICode(msg; skip_invalid = false) + @test !isvalid(cb) + end + + # Methods - copy + let msg = AIMessage(""" + ```julia + println(\"hello\") + ``` + Some text + ```julia + println(\"world\") + b=2 + ``` + """) + cb = AICode(msg) + cb_copy = Base.copy(cb) + @test cb_copy.code == cb.code + @test cb_copy !== cb + end +end + +@testset "AICode-methods" begin + ## SHOW + # Test with All Fields as `nothing` + code_block = AICode(""; auto_eval = false) + buffer = IOBuffer() + show(buffer, code_block) + output = String(take!(buffer)) + @test output == + "AICode(Success: N/A, Parsed: N/A, Evaluated: N/A, Error Caught: N/A, StdOut: N/A, Code: 1 Lines)" + + # Test with All Fields Set + code_block = AICode("println(\"Hello World\")") + buffer = IOBuffer() + show(buffer, code_block) + output = String(take!(buffer)) + @test output == + "AICode(Success: True, Parsed: True, Evaluated: True, Error Caught: N/A, StdOut: True, Code: 1 Lines)" + + # Test with error + code_block = AICode("error(\"Test Error\")\nprint(\"\")") + buffer = IOBuffer() + show(buffer, code_block) + output = String(take!(buffer)) + @test output == + "AICode(Success: False, Parsed: True, Evaluated: N/A, Error Caught: True, StdOut: True, Code: 2 Lines)" + + ## EQUALITY + # Test Comparing Two Identical Code Blocks -- if it's not safe_eval, it's not equal (gensym'd Safe module for output!) + code1 = AICode("print(\"Hello\")"; safe_eval = false) + code2 = AICode("print(\"Hello\")"; safe_eval = false) + @test code1 == code2 + + # Test Comparing Two Different Code Blocks + code1 = AICode("print(\"Hello\")") + code2 = AICode("print(\"World\")") + @test code1 != code2 + + # Different gensym! + code1 = AICode("print(\"Hello\")"; safe_eval = true) + code2 = AICode("print(\"Hello\")"; safe_eval = false) + @test code1 != code2 +end diff --git a/test/code_expressions.jl b/test/code_expressions.jl new file mode 100644 index 000000000..e146edc0c --- /dev/null +++ b/test/code_expressions.jl @@ -0,0 +1,151 @@ +using PromptingTools: AICode, isparsed, isparseerror, is_julia_code, is_julia_expr +using PromptingTools: remove_all_tests_from_expr!, + remove_test_items_from_expr!, remove_macro_expr!, extract_module_name + +@testset "is_julia_expr" begin + # Valid Julia Expressions + @test is_julia_expr(:(x = 1)) == true + @test is_julia_expr(:(x === y)) == true + @test is_julia_expr(:(for i in 1:10 + println(i) + end)) == true + @test is_julia_expr(:(function foo() + return 42 + end)) == true + @test is_julia_expr(:(if x > 0 + println("positive") + end)) == true + + # Invalid Expressions + @test is_julia_expr(:(12345)) == false + + # Nested Expressions + @test is_julia_expr(:(begin + x = 1 + y = 2 + end)) == true + + # Non-Expr Types + @test is_julia_expr(42) == false + @test is_julia_expr("string") == false + @test is_julia_expr([1, 2, 3]) == false +end + +@testset "remove_macro_expr!" begin + # Test with @testset macro + expr = Meta.parseall(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + end + y = 3 + 3 + """) + expected = Meta.parseall("y = 3 + 3") + result = remove_macro_expr!(expr) + @test result.args[end] == expected.args[end] + + # Test with nested @testset + expr = Meta.parseall(""" + @testset "Outer Test" begin + @testset "Inner Test" begin + x = 1 + 1 + end + y = 2 + 2 + end + """) + expected = Meta.parseall("") # All expressions are removed + result = remove_macro_expr!(expr) + # 1.9 parser eats the empty row, 1.10 retains it + @test length(result.args) == 1 || result == expected + + # Test without @testset + expr = Meta.parseall("z = 4 + 4") + expected = Meta.parseall("z = 4 + 4") + result = remove_macro_expr!(expr) + @test result == expected + + # Test with different macro + expr = Meta.parseall("@chain x begin; end") + expected = Meta.parseall("@chain x begin; end") + result = remove_macro_expr!(expr, Symbol("@test")) + @test result == expected +end + +@testset "remove_all_tests_from_expr!" begin + # Test with both @testset and @test macros + expr = Meta.parseall(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + end + @test x == 2 + @test_throws AssertionError func(1) + y = 3 + 3 + """) + expected = Meta.parseall("y = 3 + 3") + result = remove_all_tests_from_expr!(expr) + @test result.args[end] == expected.args[end] +end +@testset "remove_test_items_from_expr!" begin + # Remove @test macros + expr = Meta.parseall(""" + @testset "Example Tests" begin + x = 1 + 1 + @test x == 2 + @test y == 2 + end + @test x == 2 + @test_throws AssertionError func(1) + y = 3 + 3 + """) + expected = Meta.parseall(""" + @testset "Example Tests" begin + x = 1 + 1 + end + y = 3 + 3 + """) + result = remove_test_items_from_expr!(expr) + @test result.args[end] == expected.args[end] +end + +@testset "extract_module_name" begin + # Test with a valid module expression + module_expr = Meta.parse("module MyTestModule\nend") + @test extract_module_name(module_expr) == :MyTestModule + + # Test with an expression that is not a module + non_module_expr = Meta.parse("x = 1 + 1") + @test extract_module_name(non_module_expr) === nothing + + # In a nested expression tree + module_expr = Meta.parseall("module MyTestModule\nfoo()=\"hello\"\nend") + @test extract_module_name(module_expr) == :MyTestModule + + # Test with an empty expression + empty_expr = Meta.parse("") + @test extract_module_name(empty_expr) === nothing +end + +@testset "isparsed, isparseerror" begin + ## isparsed + @test isparsed(:(x = 1)) == true + # parse an incomplete call + @test isparsed(Meta.parseall("(")) == false + # parse an error call + @test isparsed(Meta.parseall("+-+-+--+")) == false + # nothing + @test isparsed(nothing) == false + # Validate that we don't have false positives with error + @test isparsed(Meta.parseall("error(\"s\")")) == true + + ## isparseerror + @test isparseerror(nothing) == false + @test isparseerror(ErrorException("syntax: unexpected \"(\" in argument list")) == true + @test isparseerror(Base.Meta.ParseError("xyz")) == true + + # AICode + cb = AICode("(") + @test isparsed(cb) == false + cb = AICode("a+1") + @test isparsed(cb) == true +end diff --git a/test/code_generation.jl b/test/code_generation.jl deleted file mode 100644 index 1c98654f9..000000000 --- a/test/code_generation.jl +++ /dev/null @@ -1,736 +0,0 @@ -using PromptingTools: extract_julia_imports -using PromptingTools: detect_pkg_operation, - detect_missing_packages, extract_function_name, remove_unsafe_lines -using PromptingTools: has_julia_prompt, - remove_julia_prompt, extract_code_blocks, extract_code_blocks_fallback, eval! -using PromptingTools: escape_interpolation, find_subsequence_positions -using PromptingTools: AICode, isparsed, isparseerror, is_julia_code, is_julia_expr -using PromptingTools: remove_tests_from_expr!, remove_macro_expr!, extract_module_name - -@testset "is_julia_expr" begin - # Valid Julia Expressions - @test is_julia_expr(:(x = 1)) == true - @test is_julia_expr(:(x === y)) == true - @test is_julia_expr(:(for i in 1:10 - println(i) - end)) == true - @test is_julia_expr(:(function foo() - return 42 - end)) == true - @test is_julia_expr(:(if x > 0 - println("positive") - end)) == true - - # Invalid Expressions - @test is_julia_expr(:(12345)) == false - - # Nested Expressions - @test is_julia_expr(:(begin - x = 1 - y = 2 - end)) == true - - # Non-Expr Types - @test is_julia_expr(42) == false - @test is_julia_expr("string") == false - @test is_julia_expr([1, 2, 3]) == false -end - -@testset "remove_macro_expr!" begin - # Test with @testset macro - expr = Meta.parseall(""" - @testset "Example Tests" begin - x = 1 + 1 - @test x == 2 - end - y = 3 + 3 - """) - expected = Meta.parseall("y = 3 + 3") - result = remove_macro_expr!(expr) - @test result.args[end] == expected.args[end] - - # Test with nested @testset - expr = Meta.parseall(""" - @testset "Outer Test" begin - @testset "Inner Test" begin - x = 1 + 1 - end - y = 2 + 2 - end - """) - expected = Meta.parseall("") # All expressions are removed - result = remove_macro_expr!(expr) - # 1.9 parser eats the empty row, 1.10 retains it - @test length(result.args) == 1 || result == expected - - # Test without @testset - expr = Meta.parseall("z = 4 + 4") - expected = Meta.parseall("z = 4 + 4") - result = remove_macro_expr!(expr) - @test result == expected - - # Test with different macro - expr = Meta.parseall("@chain x begin; end") - expected = Meta.parseall("@chain x begin; end") - result = remove_macro_expr!(expr, Symbol("@test")) - @test result == expected -end - -@testset "remove_tests_from_expr!" begin - # Test with both @testset and @test macros - expr = Meta.parseall(""" - @testset "Example Tests" begin - x = 1 + 1 - @test x == 2 - end - @test x == 2 - @test_throws AssertionError func(1) - y = 3 + 3 - """) - expected = Meta.parseall("y = 3 + 3") - result = remove_tests_from_expr!(expr) - @test result.args[end] == expected.args[end] -end - -@testset "extract_module_name" begin - # Test with a valid module expression - module_expr = Meta.parse("module MyTestModule\nend") - @test extract_module_name(module_expr) == :MyTestModule - - # Test with an expression that is not a module - non_module_expr = Meta.parse("x = 1 + 1") - @test extract_module_name(non_module_expr) === nothing - - # In a nested expression tree - module_expr = Meta.parseall("module MyTestModule\nfoo()=\"hello\"\nend") - @test extract_module_name(module_expr) == :MyTestModule - - # Test with an empty expression - empty_expr = Meta.parse("") - @test extract_module_name(empty_expr) === nothing -end - -@testset "is_julia_code" begin - - # Valid Julia Code - @test is_julia_code("x = 1 + 2") == true - @test is_julia_code("println(\"Hello, world!\")") == true - @test is_julia_code("function foo()\nreturn 42\nend") == true - - # Invalid Julia Code - @test is_julia_code("x ==== y") == false - - # Empty String - @test is_julia_code("") == false - - # Non-Code Strings - @test is_julia_code("This is a plain text, not a code.") == false - - # Complex Julia Expressions - @test is_julia_code("for i in 1:10\nprintln(i)\nend") == true - @test is_julia_code("if x > 0\nprintln(\"positive\")\nelse\nprintln(\"non-positive\")\nend") == - true - - # Invalid Syntax - @test is_julia_code("function foo() return 42") == false # Missing 'end' keyword -end - -@testset "extract_imports tests" begin - @test extract_julia_imports("using Test, LinearAlgebra") == - Symbol.(["Test", "LinearAlgebra"]) - @test extract_julia_imports("import Test\nimport ABC,DEF\nusing GEM: func") == - Symbol.(["Test", "ABC", "DEF", "GEM"]) - @test extract_julia_imports("import PackageA.PackageB: funcA\nimport PackageC") == - Symbol.(["PackageA.PackageB", "PackageC"]) - @test extract_julia_imports("using Base.Threads\nusing Main.MyPkg") == - Symbol[] -end - -@testset "detect_missing_packages" begin - @test detect_missing_packages(Symbol[]) == (false, Symbol[]) - @test detect_missing_packages(Symbol.(["Test"])) == (false, Symbol[]) - @test detect_missing_packages(Symbol.(["Test", "Base", "Main"])) == (false, Symbol[]) - @test detect_missing_packages(Symbol.(["Test", - "Base", - "Main", - "SpecialPackage12345678", "SpecialPackage123456789"])) == (true, [:SpecialPackage12345678, :SpecialPackage123456789]) -end - -@testset "detect_pkg_operation" begin - @test detect_pkg_operation("Pkg.activate(\".\")") == true - @test detect_pkg_operation("Pkg.add(\"SomePkg\")") == true - @test detect_pkg_operation("blabla Pkg.activate(\".\")") == true - @test detect_pkg_operation("hello world;") == false - @test detect_pkg_operation("import Pkg;") == false -end - -@testset "remove_unsafe_lines" begin - @test remove_unsafe_lines("Pkg.activate(\".\")") == "" - @test remove_unsafe_lines("Pkg.add(\"SomePkg\")") == "" - s = """ - a=1 - Pkg.add("a") - b=2 - Pkg.add("b") - using 12315456NotExisting - """ - @test remove_unsafe_lines(s) == "a=1\nb=2\n" - @test remove_unsafe_lines("Nothing"; verbose = true) == "Nothing\n" -end - -@testset "has_julia_prompt" begin - @test has_julia_prompt("julia> a=1") - @test has_julia_prompt("> a=1") - @test has_julia_prompt(""" -# something else first -julia> a=1 -""") - @test has_julia_prompt(""" - > a=\"\"\" - hey - there - \"\"\" - """) - @test !has_julia_prompt(""" - # something - # new - a=1 - """) -end - -@testset "remove_julia_prompt" begin - @test remove_julia_prompt("julia> a=1") == "a=1" - @test remove_julia_prompt("> a=1") == "a=1" - @test remove_julia_prompt(""" -# something else first -julia> a=1 -# output -""") == "a=1" - @test remove_julia_prompt(""" - # something - # new - a=1 - """) == """ - # something - # new - a=1 - """ - @test remove_julia_prompt(""" -julia> a=\"\"\" - hey - there - \"\"\" -"hey\nthere\n" - """) == """ -a=\"\"\" - hey - there - \"\"\"""" -end - -@testset "escape_interpolation" begin - @test escape_interpolation("aaa") == "aaa" - @test escape_interpolation("\$") == String(['\\', '$']) -end - -@testset "find_subsequence_positions" begin - # Test 1: Basic functionality - @test find_subsequence_positions(codeunits("ab"), codeunits("cababcab")) == [2, 4, 7] - - # Test 2: Subsequence not in sequence - @test find_subsequence_positions(codeunits("xyz"), codeunits("hello")) == [] - - # Test 3: Empty subsequence -- should return all positions+1 - @test find_subsequence_positions(codeunits(""), codeunits("hello")) == 1:6 - - # Test 4: Subsequence longer than sequence - @test find_subsequence_positions(codeunits("longsubsequence"), codeunits("short")) == [] - - # Test 5: Repeated characters - @test find_subsequence_positions(codeunits("ana"), codeunits("banana")) == [2, 4] - @test find_subsequence_positions(codeunits("a"), codeunits("a"^6)) == 1:6 -end - -@testset "extract_code_blocks" begin - # Single Julia Code Block - markdown_content = """ - # Example - ```julia - println("Hello, World!") - ``` - """ - @test extract_code_blocks(markdown_content) == - SubString{String}["println(\"Hello, World!\")"] - - # at edges (no newlines) - markdown_content = """```julia -println("hello") -```""" - @test extract_code_blocks(markdown_content) == - SubString{String}["println(\"hello\")"] - # Multiple Julia Code Blocks - markdown_content = """ - ```julia - println("First Block") - ``` - Some text here. - ```julia - println("Second Block") - ``` - """ - @test extract_code_blocks(markdown_content) == - SubString{String}["println(\"First Block\")", "println(\"Second Block\")"] - - # No Julia Code Blocks - markdown_content = """ - This is a text without Julia code blocks. - """ - @test isempty(extract_code_blocks(markdown_content)) - - # Mixed Language Code Blocks - markdown_content = """ - ```python - print("This is Python") - ``` - ```julia - println("This is Julia") - ``` - """ - @test extract_code_blocks(markdown_content) == - SubString{String}["println(\"This is Julia\")"] - - # Nested Blocks (plain block outer) - markdown_content = """ - ``` - ```julia - println("Nested Block") - ``` - ``` - """ - @test extract_code_blocks(markdown_content) == - SubString{String}["println(\"Nested Block\")"] - - # Nested Julia code blocks - markdown_example = """ - ```julia - # Outer Julia code block - - # An example of a nested Julia code block in markdown - \"\"\" - ```julia - x = 5 - println(x) - ``` - \"\"\" - - y = 10 - println(y) - ``` - """ - @test extract_code_blocks(markdown_example) == - SubString{String}["# Outer Julia code block\n\n# An example of a nested Julia code block in markdown\n\"\"\"\n```julia\nx = 5\nprintln(x)\n```\n\"\"\"\n\ny = 10\nprintln(y)"] - - # Tough case of regex inside a function - markdown_example = """ -```julia -function find_match(md::AbstractString) - return match(r"```\\n(?:(?!\\n```)\\s*.*\\n?)*\\s*```", md) -end -``` -""" - @test extract_code_blocks(markdown_example) == - SubString{String}["function find_match(md::AbstractString)\n return match(r\"```\\n(?:(?!\\n```)\\s*.*\\n?)*\\s*```\", md)\nend"] - - # Some small models forget newlines - no_newline = """ - ```julia function clean_column(col::AbstractString) - col = strip(lowercase(col)) - col = replace(col, r"[-\\s]+", "_") - col - end - ``` - """ - @test extract_code_blocks(no_newline) == - SubString{String}["function clean_column(col::AbstractString)\n col = strip(lowercase(col))\n col = replace(col, r\"[-\\s]+\", \"_\")\n col\nend"] -end - -@testset "extract_code_blocks_fallback" begin - - # Basic Functionality Test - @test extract_code_blocks_fallback("```\ncode block\n```") == ["code block"] - - # No Code Blocks Test - @test isempty(extract_code_blocks_fallback("Some text without code blocks")) - - # Adjacent Code Blocks Test - @test extract_code_blocks_fallback("```\ncode1\n```\n \n```\ncode2\n```") == - ["code1", "", "code2"] - - # Special Characters Test - @test extract_code_blocks_fallback("```\n<>&\"'\n```") == ["<>&\"'"] - - # Large Input Test - large_input = "```\n" * repeat("large code block\n", 10) * "```" - @test extract_code_blocks_fallback(large_input) == - [strip(repeat("large code block\n", 10))] - - # Empty String Test - @test isempty(extract_code_blocks_fallback("")) - - # delimiter inside of code - delim_in_middle = """ - ``` - function myadd(a, b) - # here is a silly comment that ends with ``` - return a + b - end - ``` - """ - @test extract_code_blocks_fallback(delim_in_middle) == - SubString{String}["function myadd(a, b)\n # here is a silly comment that ends with ```\n return a + b\nend"] - - # Different Delimiter Test - @test extract_code_blocks_fallback("~~~\ncode block\n~~~", "~~~") == ["code block"] -end - -@testset "extract_function_name" begin - # Test 1: Test an explicit function declaration - @test extract_function_name("function testFunction1()\nend") == "testFunction1" - - # Test 2: Test a concise function declaration - @test extract_function_name("testFunction2() = 42") == "testFunction2" - - # Test 3: Test a code block with no function - @test extract_function_name("let a = 10\nb = 20\nend") === nothing - - # Test 4: Test a code block with a multiline function and comments - @test extract_function_name(""" - # Comment line - function testFunction3(arg1, arg2) - # Function body - return arg1 + arg2 - end - """) == "testFunction3" - - # Test 5: Test a code block with multiple functions, should return the first function's name - @test extract_function_name(""" - function firstFunction() - end - - function secondFunction() - end - """) == "firstFunction" -end - -@testset "eval!" begin - # Test that it captures stdout and output - let cb = AICode(; code = """ - println("Hello") - a=1 - """) - eval!(cb) - @test !isnothing(cb.expression) - @test isnothing(cb.error) - @test cb.success == true - @test isvalid(cb) - @test cb.stdout == "Hello\n" - @test cb.output.a == 1 - end - # Test that it captures parsing errors - let cb = AICode(; code = """ - a=1 + - mla;sda b=2 - """) - eval!(cb) - @test cb.success == false - @test !isvalid(cb) - @test cb.error isa Exception # can be Base.Meta.ParseError or ErrorException depending on Julia version - end - # Test that it captures execution errors - let cb = AICode(; code = """ - a=1 + b # b not defined yet - b=2 - """) - eval!(cb) - @test cb.success == false - @test cb.error == UndefVarError(:b) - @test !isnothing(cb.expression) # parsed - end -end -## Addition, needs to be outside of @testset -# Test that it captures test failures, we need to move it to the main file as it as it doesn't work inside a testset -# let cb = AICode(; code = """ -# @test 1==2 -# """) -# eval!(cb) -# @test cb.success == false -# @info cb.error cb.output -# @test cb.error isa Test.FallbackTestSetException -# @test !isnothing(cb.expression) # parsed -# @test occursin("Test Failed", cb.stdout) # capture details of the test failure -# @test isnothing(cb.output) # because it failed -# end - -@testset "eval! kwargs" begin - ## Safe Eval == true mode - # package that is not available - cb = AICode(; code = "using ExoticPackage123") |> eval! - @test cb.error isa Exception - @test occursin("Safety Error", cb.error.msg) - @test occursin("ExoticPackage123", cb.error.msg) - # Pkg operations - cb = AICode(; code = "Pkg.activate(\".\")") |> eval! - @test cb.error isa Exception - @test occursin("Safety Error", cb.error.msg) - @test occursin("Use of package manager ", cb.error.msg) - - # Evaluate inside a gensym'd module - cb = AICode(; code = "a=1") |> eval! - @test occursin("SafeCustomModule", string(cb.output)) - - ## Safe Eval == false mode - # package that is not available - cb = AICode(; code = "using ExoticPackage123") - eval!(cb; safe_eval = false) - @test !isvalid(cb) - @test cb.error isa ArgumentError # now it's caught by REPL that we don't have the package - # Pkg operations - cb = AICode(; code = "import Pkg; Pkg.status()") - eval!(cb; safe_eval = false) - # This works but in test mode, Julia claims it doesn't have Pkg package... - # @test isvalid(cb) - # Evaluate in Main directly - cb = AICode(; code = "a123=123") - eval!(cb; safe_eval = false) - @test cb.output == 123 - @test a123 == 123 - - # Check that empty code is invalid - cb = AICode("") - @test !isvalid(cb) - @test cb.error isa Exception - - # Test prefix and suffix - cb = AICode(; code = "x=1") - eval!(cb; prefix = "a=1", suffix = "b=2") - @test cb.output.a == 1 - @test cb.output.b == 2 - - # Whether to capture stdout - cb = AICode(; code = "println(\"Hello\")") - eval!(cb; capture_stdout = false) - @test cb.stdout == nothing - @test cb.code == "println(\"Hello\")" - @test isvalid(cb) - - eval!(cb; capture_stdout = true) - @test cb.stdout == "Hello\n" - @test cb.code == "println(\"Hello\")" - @test isvalid(cb) - - # Test execution_timeout - cb = AICode("sleep(1.1)", execution_timeout = 1) - @test cb.success == false - @test isnothing(cb.output) - @test cb.error isa InterruptException - cb = AICode("sleep(1.1)", execution_timeout = 2) - @test cb.success == true - @test isnothing(cb.error) - - # expression-only method - cb = AICode(""" - module MyModule - function foo() - println("Hello") - end - end - """; safe_eval = false) - eval_module = getfield(Main, extract_module_name(cb.expression)) - eval!(cb, Meta.parseall("foo()"); eval_module) - @test isnothing(cb.error) - @test cb.stdout == "Hello\n" -end - -@testset "AICode constructors" begin - # Initiate from provided text - let cb = AICode(""" - println("Hello") - a=1 - """) - # eval! is automatic - @test !isnothing(cb.expression) - @test isnothing(cb.error) - @test cb.success == true - @test cb.stdout == "Hello\n" - @test cb.output.a == 1 - end - - # Test auto-eval=false - let cb = AICode(""" - println("Hello") - a=1 - """; auto_eval = false) - # eval! is automatic - @test isnothing(cb.expression) - @test isnothing(cb.error) - @test cb.success == nothing - end - - # From AI Message - let msg = AIMessage(""" -```julia -println(\"hello\") -``` -Some text -```julia -println(\"world\") -b=2 -``` -""") - cb = AICode(msg) - @test !isnothing(cb.expression) - @test isnothing(cb.error) - @test cb.success == true - @test cb.stdout == "hello\nworld\n" - @test cb.output.b == 2 - end - - # Fallback extraction method - let msg = AIMessage(""" -``` -println(\"hello\") -``` -Some text -``` -println(\"world\") -b=2 -``` -""") - cb = AICode(msg) - @test !isnothing(cb.expression) - @test isnothing(cb.error) - @test cb.success == true - @test cb.stdout == "hello\nworld\n" - @test cb.output.b == 2 - end - - # skip_unsafe=true - let msg = AIMessage(""" - ```julia - a=1 - Pkg.add("a") - b=2 - Pkg.add("b") - using 12315456NotExisting - ``` - """) - cb = AICode(msg; skip_unsafe = true) - @test cb.code == "a=1\nb=2\n" - - # dispatch on text - code = extract_code_blocks(msg.content) |> x -> join(x, "\n") - cb = AICode(code; skip_unsafe = true) - @test cb.code == "a=1\nb=2\n" - end - - # skip_invalid=true - let msg = AIMessage(""" - ```julia - println("Hello world!") - ``` - - ```julia - println("Hello world!) # missing quote - ``` - """) - cb = AICode(msg; skip_invalid = true) - @test cb.code == "println(\"Hello world!\")" - - # if it's not switched on - cb = AICode(msg; skip_invalid = false) - @test !isvalid(cb) - end - - # Methods - copy - let msg = AIMessage(""" - ```julia - println(\"hello\") - ``` - Some text - ```julia - println(\"world\") - b=2 - ``` - """) - cb = AICode(msg) - cb_copy = Base.copy(cb) - @test cb_copy.code == cb.code - @test cb_copy !== cb - end -end - -@testset "AICode-methods" begin - ## SHOW - # Test with All Fields as `nothing` - code_block = AICode(""; auto_eval = false) - buffer = IOBuffer() - show(buffer, code_block) - output = String(take!(buffer)) - @test output == - "AICode(Success: N/A, Parsed: N/A, Evaluated: N/A, Error Caught: N/A, StdOut: N/A, Code: 1 Lines)" - - # Test with All Fields Set - code_block = AICode("println(\"Hello World\")") - buffer = IOBuffer() - show(buffer, code_block) - output = String(take!(buffer)) - @test output == - "AICode(Success: True, Parsed: True, Evaluated: True, Error Caught: N/A, StdOut: True, Code: 1 Lines)" - - # Test with error - code_block = AICode("error(\"Test Error\")\nprint(\"\")") - buffer = IOBuffer() - show(buffer, code_block) - output = String(take!(buffer)) - @test output == - "AICode(Success: False, Parsed: True, Evaluated: N/A, Error Caught: True, StdOut: True, Code: 2 Lines)" - - ## EQUALITY - # Test Comparing Two Identical Code Blocks -- if it's not safe_eval, it's not equal (gensym'd Safe module for output!) - code1 = AICode("print(\"Hello\")"; safe_eval = false) - code2 = AICode("print(\"Hello\")"; safe_eval = false) - @test code1 == code2 - - # Test Comparing Two Different Code Blocks - code1 = AICode("print(\"Hello\")") - code2 = AICode("print(\"World\")") - @test code1 != code2 - - # Different gensym! - code1 = AICode("print(\"Hello\")"; safe_eval = true) - code2 = AICode("print(\"Hello\")"; safe_eval = false) - @test code1 != code2 -end -@testset "isparsed, isparseerror" begin - ## isparsed - @test isparsed(:(x = 1)) == true - # parse an incomplete call - @test isparsed(Meta.parseall("(")) == false - # parse an error call - @test isparsed(Meta.parseall("+-+-+--+")) == false - # nothing - @test isparsed(nothing) == false - # Validate that we don't have false positives with error - @test isparsed(Meta.parseall("error(\"s\")")) == true - - ## isparseerror - @test isparseerror(nothing) == false - @test isparseerror(ErrorException("syntax: unexpected \"(\" in argument list")) == true - @test isparseerror(Base.Meta.ParseError("xyz")) == true - - # AICode - cb = AICode("(") - @test isparsed(cb) == false - cb = AICode("a+1") - @test isparsed(cb) == true -end diff --git a/test/code_parsing.jl b/test/code_parsing.jl new file mode 100644 index 000000000..179a4d27b --- /dev/null +++ b/test/code_parsing.jl @@ -0,0 +1,379 @@ +using PromptingTools: extract_julia_imports +using PromptingTools: detect_pkg_operation, + detect_missing_packages, extract_function_name, remove_unsafe_lines +using PromptingTools: has_julia_prompt, + remove_julia_prompt, extract_code_blocks, extract_code_blocks_fallback, eval! +using PromptingTools: escape_interpolation, find_subsequence_positions +using PromptingTools: AICode, is_julia_code, is_julia_expr +using PromptingTools: extract_testset_name, + extract_package_name_from_argerror, extract_stacktrace_lines + +@testset "is_julia_code" begin + + # Valid Julia Code + @test is_julia_code("x = 1 + 2") == true + @test is_julia_code("println(\"Hello, world!\")") == true + @test is_julia_code("function foo()\nreturn 42\nend") == true + + # Invalid Julia Code + @test is_julia_code("x ==== y") == false + + # Empty String + @test is_julia_code("") == false + + # Non-Code Strings + @test is_julia_code("This is a plain text, not a code.") == false + + # Complex Julia Expressions + @test is_julia_code("for i in 1:10\nprintln(i)\nend") == true + @test is_julia_code("if x > 0\nprintln(\"positive\")\nelse\nprintln(\"non-positive\")\nend") == + true + + # Invalid Syntax + @test is_julia_code("function foo() return 42") == false # Missing 'end' keyword +end + +@testset "extract_imports tests" begin + @test extract_julia_imports("using Test, LinearAlgebra") == + Symbol.(["Test", "LinearAlgebra"]) + @test extract_julia_imports("import Test\nimport ABC,DEF\nusing GEM: func") == + Symbol.(["Test", "ABC", "DEF", "GEM"]) + @test extract_julia_imports("import PackageA.PackageB: funcA\nimport PackageC") == + Symbol.(["PackageA.PackageB", "PackageC"]) + @test extract_julia_imports("using Base.Threads\nusing Main.MyPkg") == + Symbol[] +end + +@testset "detect_missing_packages" begin + @test detect_missing_packages(Symbol[]) == (false, Symbol[]) + @test detect_missing_packages(Symbol.(["Test"])) == (false, Symbol[]) + @test detect_missing_packages(Symbol.(["Test", "Base", "Main"])) == (false, Symbol[]) + @test detect_missing_packages(Symbol.(["Test", + "Base", + "Main", + "SpecialPackage12345678", "SpecialPackage123456789"])) == (true, [:SpecialPackage12345678, :SpecialPackage123456789]) +end + +@testset "detect_pkg_operation" begin + @test detect_pkg_operation("Pkg.activate(\".\")") == true + @test detect_pkg_operation("Pkg.add(\"SomePkg\")") == true + @test detect_pkg_operation(" Pkg.activate(\".\")") == true + @test detect_pkg_operation("hello world;") == false + @test detect_pkg_operation("import Pkg;") == false +end + +@testset "remove_unsafe_lines" begin + @test remove_unsafe_lines("Pkg.activate(\".\")") == ("", "Pkg.activate(\".\")\n") + @test remove_unsafe_lines("Pkg.add(\"SomePkg\")") == ("", "Pkg.add(\"SomePkg\")\n") + s = """ + a=1 + Pkg.add("a") + b=2 + Pkg.add("b") + using 12315456NotExisting + """ + @test remove_unsafe_lines(s) == + ("a=1\nb=2\n", "Pkg.add(\"a\")\nPkg.add(\"b\")\nusing 12315456NotExisting\n") + @test remove_unsafe_lines("Nothing"; verbose = true) == ("Nothing\n", "") +end + +@testset "has_julia_prompt" begin + @test has_julia_prompt("julia> a=1") + @test has_julia_prompt("> a=1") + @test has_julia_prompt(""" +# something else first +julia> a=1 +""") + @test has_julia_prompt(""" + > a=\"\"\" + hey + there + \"\"\" + """) + @test !has_julia_prompt(""" + # something + # new + a=1 + """) +end + +@testset "remove_julia_prompt" begin + @test remove_julia_prompt("julia> a=1") == "a=1" + @test remove_julia_prompt("> a=1") == "a=1" + @test remove_julia_prompt(""" +# something else first +julia> a=1 +# output +""") == "a=1" + @test remove_julia_prompt(""" + # something + # new + a=1 + """) == """ + # something + # new + a=1 + """ + @test remove_julia_prompt(""" +julia> a=\"\"\" + hey + there + \"\"\" +"hey\nthere\n" + """) == """ +a=\"\"\" + hey + there + \"\"\"""" +end + +@testset "escape_interpolation" begin + @test escape_interpolation("aaa") == "aaa" + @test escape_interpolation("\$") == String(['\\', '$']) +end + +@testset "find_subsequence_positions" begin + # Test 1: Basic functionality + @test find_subsequence_positions(codeunits("ab"), codeunits("cababcab")) == [2, 4, 7] + + # Test 2: Subsequence not in sequence + @test find_subsequence_positions(codeunits("xyz"), codeunits("hello")) == [] + + # Test 3: Empty subsequence -- should return all positions+1 + @test find_subsequence_positions(codeunits(""), codeunits("hello")) == 1:6 + + # Test 4: Subsequence longer than sequence + @test find_subsequence_positions(codeunits("longsubsequence"), codeunits("short")) == [] + + # Test 5: Repeated characters + @test find_subsequence_positions(codeunits("ana"), codeunits("banana")) == [2, 4] + @test find_subsequence_positions(codeunits("a"), codeunits("a"^6)) == 1:6 +end + +@testset "extract_code_blocks" begin + # Single Julia Code Block + markdown_content = """ + # Example + ```julia + println("Hello, World!") + ``` + """ + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"Hello, World!\")"] + + # at edges (no newlines) + markdown_content = """```julia +println("hello") +```""" + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"hello\")"] + # Multiple Julia Code Blocks + markdown_content = """ + ```julia + println("First Block") + ``` + Some text here. + ```julia + println("Second Block") + ``` + """ + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"First Block\")", "println(\"Second Block\")"] + + # No Julia Code Blocks + markdown_content = """ + This is a text without Julia code blocks. + """ + @test isempty(extract_code_blocks(markdown_content)) + + # Mixed Language Code Blocks + markdown_content = """ + ```python + print("This is Python") + ``` + ```julia + println("This is Julia") + ``` + """ + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"This is Julia\")"] + + # Nested Blocks (plain block outer) + markdown_content = """ + ``` + ```julia + println("Nested Block") + ``` + ``` + """ + @test extract_code_blocks(markdown_content) == + SubString{String}["println(\"Nested Block\")"] + + # Nested Julia code blocks + markdown_example = """ + ```julia + # Outer Julia code block + + # An example of a nested Julia code block in markdown + \"\"\" + ```julia + x = 5 + println(x) + ``` + \"\"\" + + y = 10 + println(y) + ``` + """ + @test extract_code_blocks(markdown_example) == + SubString{String}["# Outer Julia code block\n\n# An example of a nested Julia code block in markdown\n\"\"\"\n```julia\nx = 5\nprintln(x)\n```\n\"\"\"\n\ny = 10\nprintln(y)"] + + # Tough case of regex inside a function + markdown_example = """ +```julia +function find_match(md::AbstractString) + return match(r"```\\n(?:(?!\\n```)\\s*.*\\n?)*\\s*```", md) +end +``` +""" + @test extract_code_blocks(markdown_example) == + SubString{String}["function find_match(md::AbstractString)\n return match(r\"```\\n(?:(?!\\n```)\\s*.*\\n?)*\\s*```\", md)\nend"] + + # Some small models forget newlines + no_newline = """ + ```julia function clean_column(col::AbstractString) + col = strip(lowercase(col)) + col = replace(col, r"[-\\s]+", "_") + col + end + ``` + """ + @test extract_code_blocks(no_newline) == + SubString{String}["function clean_column(col::AbstractString)\n col = strip(lowercase(col))\n col = replace(col, r\"[-\\s]+\", \"_\")\n col\nend"] +end + +@testset "extract_code_blocks_fallback" begin + + # Basic Functionality Test + @test extract_code_blocks_fallback("```\ncode block\n```") == ["code block"] + + # No Code Blocks Test + @test isempty(extract_code_blocks_fallback("Some text without code blocks")) + + # Adjacent Code Blocks Test + @test extract_code_blocks_fallback("```\ncode1\n```\n \n```\ncode2\n```") == + ["code1", "", "code2"] + + # Special Characters Test + @test extract_code_blocks_fallback("```\n<>&\"'\n```") == ["<>&\"'"] + + # Large Input Test + large_input = "```\n" * repeat("large code block\n", 10) * "```" + @test extract_code_blocks_fallback(large_input) == + [strip(repeat("large code block\n", 10))] + + # Empty String Test + @test isempty(extract_code_blocks_fallback("")) + + # delimiter inside of code + delim_in_middle = """ + ``` + function myadd(a, b) + # here is a silly comment that ends with ``` + return a + b + end + ``` + """ + @test extract_code_blocks_fallback(delim_in_middle) == + SubString{String}["function myadd(a, b)\n # here is a silly comment that ends with ```\n return a + b\nend"] + + # Different Delimiter Test + @test extract_code_blocks_fallback("~~~\ncode block\n~~~", "~~~") == ["code block"] +end + +@testset "extract_function_name" begin + # Test 1: Test an explicit function declaration + @test extract_function_name("function testFunction1()\nend") == "testFunction1" + + # Test 2: Test a concise function declaration + @test extract_function_name("testFunction2() = 42") == "testFunction2" + + # Test 3: Test a code block with no function + @test extract_function_name("let a = 10\nb = 20\nend") === nothing + + # Test 4: Test a code block with a multiline function and comments + @test extract_function_name(""" + # Comment line + function testFunction3(arg1, arg2) + # Function body + return arg1 + arg2 + end + """) == "testFunction3" + + # Test 5: Test a code block with multiple functions, should return the first function's name + @test extract_function_name(""" + function firstFunction() + end + + function secondFunction() + end + """) == "firstFunction" +end + +@testset "extract_testset_name" begin + @test extract_testset_name("@testset \"TestSet1\" begin") == "TestSet1" + testset_str = """ + @testset "pig_latinify" begin + output = pig_latinify("hello") + expected = "ellohay" + @test output == expected + end + """ + @test extract_testset_name(testset_str) == "pig_latinify" + @test extract_testset_name(" " * testset_str) == "pig_latinify" + @test extract_testset_name("@testset \"TestSet1\" begin") == "TestSet1" + @test extract_testset_name("@testset begin") == nothing +end + +@testset "extract_package_name_from_argerror" begin + @test extract_package_name_from_argerror("Package MyPackage not found") == "MyPackage" + error_msg = "Package Threads not found in current path, maybe you meant `import/using .Threads`.\n- Otherwise, run `import Pkg; Pkg.add(\"Threads\")` to install the Threads package." + @test extract_package_name_from_argerror(error_msg) == "Threads" + error_msg = "Package Main.Base.Something.Package not found in current path..." + @test extract_package_name_from_argerror(error_msg) == "Main.Base.Something.Package" + @test extract_package_name_from_argerror("asdl;asdas Package Threads not found in my living room :)") == + nothing +end + +@testset "extract_stacktrace_lines" begin + @test extract_stacktrace_lines("filename1.jl", nothing) == Int[] + @test extract_stacktrace_lines("filename1.jl", "nothing") == Int[] + @test extract_stacktrace_lines("filename1.jl", "filename1.jl:10\nfilename1.jl:20\n") == + [10, 20] + s = """ + Test Summary: | Pass Total Time + detect_pkg_operation | 5 5 0.0s + Test.DefaultTestSet("detect_pkg_operation", Any[], 5, false, false, true, 1.706440939410623e9, 1.706440939410673e9, false, "/Users/xyz/test/code_parsing.jl") + + remove_unsafe_lines: Test Failed at /Users/xyz/test/code_parsing.jl:66 + Expression: remove_unsafe_lines("Pkg.activate(\".\")") == "" + Evaluated: ("", "Pkg.activate(\".\")\n") == "" + + Stacktrace: + [1] macro expansion + @ ~/.julia/juliaup/julia-1.10.0+0.aarch64.apple.darwin14/share/julia/stdlib/v1.10/Test/src/Test.jl:672 [inlined] + [2] macro expansion + @ ~/test/code_parsing.jl:66 [inlined] + [3] macro expansion + @ ~/.julia/juliaup/julia-1.10.0+0.aarch64.apple.darwin14/share/julia/stdlib/v1.10/Test/src/Test.jl:1577 [inlined] + [4] top-level scope + @ ~/test/code_parsing.jl:66 + remove_unsafe_lines: Test Failed at /Users/xyz/test/code_parsing.jl:67 + Expression: remove_unsafe_lines("Pkg.add(\"SomePkg\")") == "" + Evaluated: ("", "Pkg.add(\"SomePkg\")\n") == "" + """ + @test extract_stacktrace_lines("code_parsing.jl", s) == [66, 66, 66, 67] + @test extract_stacktrace_lines("Test.jl", s) == [672, 1577] + @test extract_stacktrace_lines("notexisting.jl", s) == Int[] +end diff --git a/test/runtests.jl b/test/runtests.jl index d598defd3..e0699bfc7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,7 @@ using PromptingTools using OpenAI, HTTP, JSON3 using SparseArrays, LinearAlgebra, Markdown -using Test +using Test, Pkg using Aqua const PT = PromptingTools @@ -22,7 +22,9 @@ end include("macros.jl") include("templates.jl") include("serialization.jl") - include("code_generation.jl") + include("code_parsing.jl") + include("code_expressions.jl") + include("code_eval.jl") end # Part of code_generation.jl / @testset "eval!" begin From 590da001273ab6dc862c9fa5a5644151dd8ba885 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 30 Jan 2024 21:14:15 +0000 Subject: [PATCH 109/251] Improve error capture + error lines capture (#63) --- CHANGELOG.md | 2 ++ src/code_eval.jl | 16 ++++++++++++++-- test/Experimental/AgentTools/code_feedback.jl | 11 +++++------ test/code_eval.jl | 12 ++++++++++++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17297a570..933a2b65f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - "gpt3" still refers to the general endpoint "gpt-3.5-turbo", which OpenAI will move to version 0125 by mid-February (ie, "gpt3t" will be the same as "gpt3" then. We have reflected the approximate cost in the model registry but note that it will be incorrect in the transition period) - "emb3small" refers to the small version of the new embedding model (dim=1536), which is 5x cheaper than Ada and promises higher quality - "emb3large" refers to the large version of the new embedding model (dim=3072), which is only 30% more expensive than Ada +- Improved AgentTools: added more information and specific methods to `aicode_feedback` and `error_feedback` to pass more targeted feedback/tips to the AIAgent +- Improved detection of which lines were the source of error during `AICode` evaluation + forcing the error details to be printed in `AICode(...).stdout` for downstream analysis. ### Fixed - Fixed typos in the documentation diff --git a/src/code_eval.jl b/src/code_eval.jl index 97d96f313..83841ca9f 100644 --- a/src/code_eval.jl +++ b/src/code_eval.jl @@ -352,13 +352,25 @@ function eval!(cb::AbstractCodeBlock, expr::Expr; cb.success = false end end + ## showerror if stdout capture failed + if (isnothing(cb.stdout) || isempty(cb.stdout)) && !isnothing(cb.error) + io = IOBuffer() + showerror(io, cb.error isa LoadError ? cb.error.error : cb.error) + cb.stdout = String(take!(io)) + end ## unwrap load error if cb.error isa LoadError push!(cb.error_lines, cb.error.line) - append!(cb.error_lines, extract_stacktrace_lines(cb.error.file, cb.stdout)) + for line in extract_stacktrace_lines(cb.error.file, cb.stdout) + (line ∉ cb.error_lines) && push!(cb.error_lines, line) + end cb.error = cb.error.error elseif !isnothing(cb.error) - append!(cb.error_lines, extract_stacktrace_lines("__code_string_eval", cb.stdout)) + ## fallback, looks for errors only in the original code (cb.code) + lines = extract_stacktrace_lines("__code_string_eval", cb.stdout) + for line in lines + (line ∉ cb.error_lines) && push!(cb.error_lines, line) + end end return cb end \ No newline at end of file diff --git a/test/Experimental/AgentTools/code_feedback.jl b/test/Experimental/AgentTools/code_feedback.jl index 806818dff..36a94c949 100644 --- a/test/Experimental/AgentTools/code_feedback.jl +++ b/test/Experimental/AgentTools/code_feedback.jl @@ -30,15 +30,14 @@ using PromptingTools.Experimental.AgentTools: testset_feedback, schedule(tsk) fetch(tsk) """) - cb.stdout = "STDOUT" feedback = aicodefixer_feedback(CodeFailedEval(), cb) - @test feedback == - "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- fetch(tsk)\n\n**Output Captured:**\n STDOUT" + @test occursin("**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- fetch(tsk)", + feedback) + cb = AICode("error(\"xx\")") - cb.stdout = "STDOUT" feedback = aicodefixer_feedback(CodeFailedEval(), cb) @test feedback == - "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- error(\"xx\")\n\n**Output Captured:**\n STDOUT" + "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- error(\"xx\")\n\n**Output Captured:**\n xx" conv = [PT.AIMessage(""" ```julia error(\"xx\") @@ -46,7 +45,7 @@ using PromptingTools.Experimental.AgentTools: testset_feedback, """)] feedback = aicodefixer_feedback(conv).feedback @test feedback == - "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- error(\"xx\")" + "**Error Detected:**\n**ErrorException**:\nxx\n\n\n\n**Lines that caused the error:**\n- error(\"xx\")\n\n**Output Captured:**\n xx" # CodeFailedTimeout cb = AICode("InterruptException()") diff --git a/test/code_eval.jl b/test/code_eval.jl index 6200ea65b..67caea2fc 100644 --- a/test/code_eval.jl +++ b/test/code_eval.jl @@ -47,6 +47,18 @@ using PromptingTools: extract_module_name @test cb.success == false @test cb.error == UndefVarError(:b) @test cb.error_lines == [1] + # despite not capturing stdout, we always unwrap the error to be able to detect error lines + @test occursin("UndefVarError", cb.stdout) + + # provide expression directly + cb = AICode(""" + bad_func()=1 + """) + expr = Meta.parseall("bad_func(1)") + eval!(cb, expr; capture_stdout = false, eval_module = cb.output) + @test cb.success == false + @test cb.error isa MethodError + @test cb.error_lines == [1] end ## Addition, needs to be outside of @testset From f29ae85c9e7de4a31b667644b2b7b98339139577 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:55:26 +0000 Subject: [PATCH 110/251] Escape fix in code loading (#64) --- CHANGELOG.md | 1 + src/code_eval.jl | 2 +- test/code_eval.jl | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 933a2b65f..b9621cad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed typos in the documentation - Fixed a bug when API keys set in ENV would not be picked up by the package (caused by inlining of the `get(ENV,...)` during precompilation) +- Fixed string interpolation to be correctly escaped when evaluating `AICode` ## [0.9.0] diff --git a/src/code_eval.jl b/src/code_eval.jl index 83841ca9f..f31af0a6b 100644 --- a/src/code_eval.jl +++ b/src/code_eval.jl @@ -304,7 +304,7 @@ function eval!(cb::AbstractCodeBlock; write(io, "using Test\nimport PromptingTools\n") write(io, prefix, "\n") write(io, - "include_string($_transform, $module_name,\"\"\"$(escape_string(code))\"\"\", \"__code_string_eval\")\n") + "include_string($_transform, $module_name,\"\"\"$(escape_string(code,'$'))\"\"\", \"__code_string_eval\")\n") write(io, suffix, "\n") safe_eval && write(io, "end") code_full = String(take!(io)) diff --git a/test/code_eval.jl b/test/code_eval.jl index 67caea2fc..c74d5b36d 100644 --- a/test/code_eval.jl +++ b/test/code_eval.jl @@ -59,6 +59,15 @@ using PromptingTools: extract_module_name @test cb.success == false @test cb.error isa MethodError @test cb.error_lines == [1] + + # test correct escaping of \$ + cb = AICode(""" + greet(s)="hi \$s" + """) + expr = Meta.parseall("greet(\"jan\")|>print") + eval!(cb, expr; capture_stdout = true, eval_module = cb.output) + @test cb.success == true + @test cb.stdout == "hi jan" end ## Addition, needs to be outside of @testset From 495d88a49db611ccf01b0fae7ee052a3e3f6e236 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 1 Feb 2024 21:12:27 +0000 Subject: [PATCH 111/251] Detect Base method overrides (#65) --- CHANGELOG.md | 1 + src/code_eval.jl | 5 +++ src/code_parsing.jl | 78 ++++++++++++++++++++++++++++++++++++++++---- test/code_eval.jl | 9 +++++ test/code_parsing.jl | 68 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 153 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9621cad8..f010d24c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - "emb3large" refers to the large version of the new embedding model (dim=3072), which is only 30% more expensive than Ada - Improved AgentTools: added more information and specific methods to `aicode_feedback` and `error_feedback` to pass more targeted feedback/tips to the AIAgent - Improved detection of which lines were the source of error during `AICode` evaluation + forcing the error details to be printed in `AICode(...).stdout` for downstream analysis. +- Improved detection of Base/Main method overrides in `AICode` evaluation (only warns about the fact), but you can use `detect_base_main_overrides(code)` for custom handling ### Fixed - Fixed typos in the documentation diff --git a/src/code_eval.jl b/src/code_eval.jl index f31af0a6b..71355bc2a 100644 --- a/src/code_eval.jl +++ b/src/code_eval.jl @@ -264,6 +264,11 @@ function eval!(cb::AbstractCodeBlock; (cb.error = ErrorException("Safety Error: Failed package import. Missing packages: $(join(string.(missing_packages),", ")). Please add them or disable the safety check (`safe_eval=false`)")) return cb end + detected, overrides = detect_base_main_overrides(code) + if detected + ## DO NOT THROW ERROR + @warn "Safety Warning: Base / Main overrides detected (functions: $(join(overrides,",")))! Please verify the safety of the code or disable the safety check (`safe_eval=false`)" + end end ## Catch bad code extraction if isempty(code) diff --git a/src/code_parsing.jl b/src/code_parsing.jl index f5562b9f1..3c0c620d6 100644 --- a/src/code_parsing.jl +++ b/src/code_parsing.jl @@ -20,7 +20,14 @@ function detect_pkg_operation(input::AbstractString) return !isnothing(m) end # Utility to detect dependencies in a string (for `safe` code evaluation / understand when we don't have a necessary package) -function extract_julia_imports(input::AbstractString) +""" + extract_julia_imports(input::AbstractString; base_or_main::Bool = false) + +Detects any `using` or `import` statements in a given string and returns the package names as a vector of symbols. + +`base_or_main` is a boolean that determines whether to isolate only `Base` and `Main` OR whether to exclude them in the returned vector. +""" +function extract_julia_imports(input::AbstractString; base_or_main::Bool = false) package_names = Symbol[] for line in split(input, "\n") if occursin(r"(^using |^import )"m, line) @@ -29,9 +36,16 @@ function extract_julia_imports(input::AbstractString) subparts = map(x -> contains(x, ':') ? split(x, ':')[1] : x, split(subparts, ",")) subparts = replace(join(subparts, ' '), ',' => ' ') - packages = filter(x -> !isempty(x) && !startswith(x, "Base") && - !startswith(x, "Main"), - split(subparts, " ")) + packages = filter(x -> !isempty(x), split(subparts, " ")) + if base_or_main + ## keep only them + packages = filter(x -> startswith(x, "Base") || + startswith(x, "Main"), packages) + else + ## exclude them + packages = filter(x -> !startswith(x, "Base") && + !startswith(x, "Main"), packages) + end append!(package_names, Symbol.(packages)) end end @@ -336,6 +350,8 @@ Extract the name of a function from a given Julia code block. The function searc If a function name is found, it is returned as a string. If no function name is found, the function returns `nothing`. +To capture all function names in the block, use `extract_function_names`. + # Arguments - `code_block::String`: A string containing Julia code. @@ -355,9 +371,9 @@ extract_function_name(code) """ function extract_function_name(code_block::AbstractString) # Regular expression for the explicit function declaration - pattern_explicit = r"function\s+(\w+)\(" + pattern_explicit = r"^\s*function\s+([\w\.\_]+)\("m # Regular expression for the concise function declaration - pattern_concise = r"^(\w+)\(.*\)\s*=" + pattern_concise = r"^\s*([\w\.\_]+)\(.*\)\s*="m # Searching for the explicit function declaration match_explicit = match(pattern_explicit, code_block) @@ -375,6 +391,56 @@ function extract_function_name(code_block::AbstractString) return nothing end +""" + extract_function_names(code_block::AbstractString) + +Extract one or more names of functions defined in a given Julia code block. The function searches for two patterns: + - The explicit function declaration pattern: `function name(...) ... end` + - The concise function declaration pattern: `name(...) = ...` + +It always returns a vector of strings, even if only one function name is found (it will be empty). + +For only one function name match, use `extract_function_name`. +""" +function extract_function_names(code_block::AbstractString) + # Regular expression for the explicit function declaration + pattern_explicit = r"^\s*function\s+([\w\.\_]+)\("m + # Regular expression for the concise function declaration + pattern_concise = r"^\s*([\w\.\_]+)\(.*\)\s*="m + + matches = String[] + + # Searching for the explicit function declaration + for m in eachmatch(pattern_explicit, code_block) + push!(matches, m.captures[1]) + end + # Searching for the concise function declaration + for m in eachmatch(pattern_concise, code_block) + push!(matches, m.captures[1]) + end + + return matches +end + +""" + detect_base_main_overrides(code_block::AbstractString) + +Detects if a given code block overrides any Base or Main methods. + +Returns a tuple of a boolean and a vector of the overriden methods. +""" +function detect_base_main_overrides(code_block::AbstractString) + funcs = extract_function_names(code_block) + base_imports = extract_julia_imports(code_block; base_or_main = true) .|> + x -> split(string(x), ".")[end] + ## check Base/Main method overrides + overriden_methods = filter(f -> occursin("Base.", f) || occursin("Main.", f) || + in(f, base_imports), + funcs) + detected = !isempty(overriden_methods) + return detected, overriden_methods +end + function extract_testset_name(testset_str::AbstractString) # Define a regex pattern to match the function name pattern = r"^\s*@testset\s*\"([^\"]+)\"\s* begin"ms diff --git a/test/code_eval.jl b/test/code_eval.jl index c74d5b36d..e6be81a76 100644 --- a/test/code_eval.jl +++ b/test/code_eval.jl @@ -96,6 +96,15 @@ end @test cb.error isa Exception @test occursin("Safety Error", cb.error.msg) @test occursin("Use of package manager ", cb.error.msg) + ## Base / Main overrides + cb = AICode(; code = """ +import Base.splitx + +splitx(aaa) = 2 +""") + @test_logs (:warn, + r"Safety Warning: Base / Main overrides detected \(functions: splitx\)") match_mode=:any eval!(cb; + safe_eval = true) # Evaluate inside a gensym'd module cb = AICode(; code = "a=1") |> eval! diff --git a/test/code_parsing.jl b/test/code_parsing.jl index 179a4d27b..318a610c2 100644 --- a/test/code_parsing.jl +++ b/test/code_parsing.jl @@ -1,6 +1,7 @@ using PromptingTools: extract_julia_imports using PromptingTools: detect_pkg_operation, - detect_missing_packages, extract_function_name, remove_unsafe_lines + detect_missing_packages, extract_function_name, extract_function_names, + remove_unsafe_lines, detect_base_main_overrides using PromptingTools: has_julia_prompt, remove_julia_prompt, extract_code_blocks, extract_code_blocks_fallback, eval! using PromptingTools: escape_interpolation, find_subsequence_positions @@ -42,6 +43,8 @@ end Symbol.(["PackageA.PackageB", "PackageC"]) @test extract_julia_imports("using Base.Threads\nusing Main.MyPkg") == Symbol[] + @test extract_julia_imports("using Base.Threads\nusing Main.MyPkg"; + base_or_main = true) == Symbol[Symbol("Base.Threads"), Symbol("Main.MyPkg")] end @testset "detect_missing_packages" begin @@ -295,9 +298,9 @@ end @testset "extract_function_name" begin # Test 1: Test an explicit function declaration @test extract_function_name("function testFunction1()\nend") == "testFunction1" - # Test 2: Test a concise function declaration @test extract_function_name("testFunction2() = 42") == "testFunction2" + @test extract_function_name(" test_Function_2() = 42") == "test_Function_2" # Test 3: Test a code block with no function @test extract_function_name("let a = 10\nb = 20\nend") === nothing @@ -321,6 +324,67 @@ end """) == "firstFunction" end +@testset "extract_function_names" begin + code_block = """ + function add(x, y) + return x + y + end + + subtract(x, y) = x - y + """ + expected_result = ["add", "subtract"] + @test extract_function_names(code_block) == expected_result + + s = """ + import Base.splitx + + Base.splitx()=1 + + splitx(aaa) = 2 + """ + @test extract_function_names(s) == ["Base.splitx", "splitx"] + @test extract_function_names("") == String[] +end + +@testset "detect_base_main_overrides" begin + # Test case 1: No overrides detected + code_block_1 = """ + function foo() + println("Hello, World!") + end + """ + @test detect_base_main_overrides(code_block_1) == (false, []) + + # Test case 2: Overrides detected + code_block_2 = """ + function Base.bar() + println("Override Base.bar()") + end + + function Main.baz() + println("Override Main.baz()") + end + """ + @test detect_base_main_overrides(code_block_2) == (true, ["Base.bar", "Main.baz"]) + + # Test case 3: Overrides with base imports + code_block_3 = """ + using Base: sin + + function Main.qux() + println("Override Main.qux()") + end + """ + @test detect_base_main_overrides(code_block_3) == (true, ["Main.qux"]) + + s4 = """ + import Base.splitx + + splitx(aaa) = 2 + """ + @test detect_base_main_overrides(s4) == (true, ["splitx"]) +end + @testset "extract_testset_name" begin @test extract_testset_name("@testset \"TestSet1\" begin") == "TestSet1" testset_str = """ From c82c4722c9088a23c23cc80c42a02e94339f7ade Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 2 Feb 2024 09:14:59 +0000 Subject: [PATCH 112/251] Up the version (#66) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9737f70a6..e24cd6579 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.10.0-DEV" +version = "0.10.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From cab3e8827b7127e1cd3a9bb1621121c2a1b6d6ef Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 7 Feb 2024 21:37:42 +0000 Subject: [PATCH 113/251] Update Codecov4 (#70) --- .github/workflows/CI.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7babf50a3..7e620817e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -38,9 +38,10 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 with: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false files: lcov.info docs: name: Documentation From 91b9479343ce47daac721eabd8c7942c0e3ae576 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 13 Feb 2024 21:54:02 +0000 Subject: [PATCH 114/251] Add Databricks API support (#71) --- CHANGELOG.md | 7 ++++ Project.toml | 2 +- docs/src/examples/working_with_custom_apis.md | 33 ++++++++++++++++- src/llm_interface.jl | 11 ++++++ src/llm_openai.jl | 36 +++++++++++++++++++ src/user_preferences.jl | 22 +++++++++--- 6 files changed, 105 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f010d24c1..5c81eb3e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [0.11.0] + +### Added +- Support for [Databricks Foundation Models API](https://docs.databricks.com/en/machine-learning/foundation-models/index.html). Requires two environment variables to be set: `DATABRICKS_API_KEY` and `DATABRICKS_HOST` (the part of the URL before `/serving-endpoints/`) + +### Fixed + ## [0.10.0] ### Added diff --git a/Project.toml b/Project.toml index e24cd6579..3a0a192ae 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.10.0" +version = "0.11.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/docs/src/examples/working_with_custom_apis.md b/docs/src/examples/working_with_custom_apis.md index d6d961552..2a083d778 100644 --- a/docs/src/examples/working_with_custom_apis.md +++ b/docs/src/examples/working_with_custom_apis.md @@ -66,4 +66,35 @@ msg = aigenerate(PT.CustomOpenAISchema(), "Count to 5 and say hi!"; api_kwargs=( ``` > [!TIP] -> If you register the model names with `PT.register_model!`, you won't have to keep providing the `schema` manually. It can be any `model` name, because the model is actually selected when you start the server in the terminal. \ No newline at end of file +> If you register the model names with `PT.register_model!`, you won't have to keep providing the `schema` manually. It can be any `model` name, because the model is actually selected when you start the server in the terminal. + +## Using Databricks Foundation Models + +You can also use the Databricks Foundation Models API with PromptingTools.jl. +It requires you to set ENV variables `DATABRICKS_API_KEY` (often referred to as "DATABRICKS TOKEN") and `DATABRICKS_HOST`. + +The long way to use it is: +```julia +msg = aigenerate(PT.DatabricksOpenAISchema(), + "Say hi to the llama!"; + model = "databricks-llama-2-70b-chat", + api_key = ENV["DATABRICKS_API_KEY"], api_kwargs = (; url=ENV["DATABRICKS_HOST"])) +``` + +But you can also register the models you're hosting and use it as usual: +```julia +# Quick registration of a model +PT.register_model!(; + name = "databricks-llama-2-70b-chat", + schema = PT.DatabricksOpenAISchema()) +PT.MODEL_ALIASES["dllama"] = "databricks-llama-2-70b-chat" # set alias to make your life easier + +# Simply call: +msg = aigenerate("Say hi to the llama!"; model = "dllama") +# Or even shorter +ai"Say hi to the llama!"dllama +``` + +You can use `aiembed` as well. + +Find more information [here](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html). \ No newline at end of file diff --git a/src/llm_interface.jl b/src/llm_interface.jl index f47237d01..95dcca998 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -135,6 +135,17 @@ See `?PREFERENCES` for more details on how to set your API key permanently. """ struct MistralOpenAISchema <: AbstractOpenAISchema end +""" + DatabricksOpenAISchema + +DatabricksOpenAISchema() allows user to call Databricks Foundation Model API. [API Reference](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html) + +Requires two environment variables to be set: +- `DATABRICKS_API_KEY`: Databricks token +- `DATABRICKS_HOST`: Address of the Databricks workspace (`https://.databricks.com`) +""" +struct DatabricksOpenAISchema <: AbstractOpenAISchema end + abstract type AbstractOllamaSchema <: AbstractPromptSchema end """ diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 369330904..8fcdce5e5 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -165,6 +165,24 @@ function OpenAI.create_chat(schema::MistralOpenAISchema, base_url = url) OpenAI.create_chat(provider, model, conversation; kwargs...) end +function OpenAI.create_chat(schema::DatabricksOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String = "https://.databricks.com", + kwargs...) + # Build the corresponding provider object + provider = CustomProvider(; + api_key = isempty(DATABRICKS_API_KEY) ? api_key : DATABRICKS_API_KEY, + base_url = isempty(DATABRICKS_HOST) ? url : DATABRICKS_HOST) + # Override standard OpenAI request endpoint + OpenAI.openai_request("serving-endpoints/$model/invocations", + provider; + method = "POST", + model, + messages = conversation, + kwargs...) +end # Extend OpenAI create_embeddings to allow for testing function OpenAI.create_embeddings(schema::AbstractOpenAISchema, @@ -221,6 +239,24 @@ function OpenAI.create_embeddings(schema::MistralOpenAISchema, base_url = url) OpenAI.create_embeddings(provider, docs, model; kwargs...) end +function OpenAI.create_embeddings(schema::DatabricksOpenAISchema, + api_key::AbstractString, + docs, + model::AbstractString; + url::String = "https://.databricks.com", + kwargs...) + # Build the corresponding provider object + provider = CustomProvider(; + api_key = isempty(DATABRICKS_API_KEY) ? api_key : DATABRICKS_API_KEY, + base_url = isempty(DATABRICKS_HOST) ? url : DATABRICKS_HOST) + # Override standard OpenAI request endpoint + OpenAI.openai_request("serving-endpoints/$model/invocations", + provider; + method = "POST", + model, + input = docs, + kwargs...) +end ## Temporary fix -- it will be moved upstream function OpenAI.create_embeddings(provider::AbstractCustomProvider, diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 09e3626ac..d9ab98bfe 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -14,6 +14,8 @@ Check your preferences by calling `get_preferences(key::String)`. - `OPENAI_API_KEY`: The API key for the OpenAI API. See [OpenAI's documentation](https://platform.openai.com/docs/quickstart?context=python) for more information. - `MISTRALAI_API_KEY`: The API key for the Mistral AI API. See [Mistral AI's documentation](https://docs.mistral.ai/) for more information. - `COHERE_API_KEY`: The API key for the Cohere API. See [Cohere's documentation](https://docs.cohere.com/docs/the-cohere-platform) for more information. +- `DATABRICKS_API_KEY`: The API key for the Databricks Foundation Model API. See [Databricks' documentation](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html) for more information. +- `DATABRICKS_HOST`: The host for the Databricks API. See [Databricks' documentation](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html) for more information. - `MODEL_CHAT`: The default model to use for aigenerate and most ai* calls. See `MODEL_REGISTRY` for a list of available models or define your own. - `MODEL_EMBEDDING`: The default model to use for aiembed (embedding documents). See `MODEL_REGISTRY` for a list of available models or define your own. - `PROMPT_SCHEMA`: The default prompt schema to use for aigenerate and most ai* calls (if not specified in `MODEL_REGISTRY`). Set as a string, eg, `"OpenAISchema"`. @@ -33,6 +35,8 @@ Define your `register_model!()` calls in your `startup.jl` file to make them ava - `MISTRALAI_API_KEY`: The API key for the Mistral AI API. - `COHERE_API_KEY`: The API key for the Cohere API. - `LOCAL_SERVER`: The URL of the local server to use for `ai*` calls. Defaults to `http://localhost:10897/v1`. This server is called when you call `model="local"` +- `DATABRICKS_API_KEY`: The API key for the Databricks Foundation Model API. +- `DATABRICKS_HOST`: The host for the Databricks API. Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. @@ -59,6 +63,8 @@ function set_preferences!(pairs::Pair{String, <:Any}...) "MISTRALAI_API_KEY", "OPENAI_API_KEY", "COHERE_API_KEY", + "DATABRICKS_API_KEY", + "DATABRICKS_HOST", "MODEL_CHAT", "MODEL_EMBEDDING", "MODEL_ALIASES", @@ -95,6 +101,8 @@ function get_preferences(key::String) "MISTRALAI_API_KEY", "OPENAI_API_KEY", "COHERE_API_KEY", + "DATABRICKS_API_KEY", + "DATABRICKS_HOST", "MODEL_CHAT", "MODEL_EMBEDDING", "MODEL_ALIASES", @@ -114,20 +122,26 @@ const MODEL_EMBEDDING::String = @load_preference("MODEL_EMBEDDING", # const PROMPT_SCHEMA = OpenAISchema() # First, load from preferences, then from environment variables -const OPENAI_API_KEY::String = @load_preference("OPENAI_API_KEY", +const OPENAI_API_KEY::String = @noinline @load_preference("OPENAI_API_KEY", default=@noinline get(ENV, "OPENAI_API_KEY", "")); # Note: Disable this warning by setting OPENAI_API_KEY to anything isempty(OPENAI_API_KEY) && @warn "OPENAI_API_KEY variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.OPENAI_API_KEY=`!" -const MISTRALAI_API_KEY::String = @load_preference("MISTRALAI_API_KEY", +const MISTRALAI_API_KEY::String = @noinline @load_preference("MISTRALAI_API_KEY", default=@noinline get(ENV, "MISTRALAI_API_KEY", "")); -const COHERE_API_KEY::String = @load_preference("COHERE_API_KEY", +const COHERE_API_KEY::String = @noinline @load_preference("COHERE_API_KEY", default=@noinline get(ENV, "COHERE_API_KEY", "")); +const DATABRICKS_API_KEY::String = @noinline @load_preference("DATABRICKS_API_KEY", + default=@noinline get(ENV, "DATABRICKS_API_KEY", "")); + +const DATABRICKS_HOST::String = @noinline @load_preference("DATABRICKS_HOST", + default=@noinline get(ENV, "DATABRICKS_HOST", "")); + ## Address of the local server -const LOCAL_SERVER::String = @load_preference("LOCAL_SERVER", +const LOCAL_SERVER::String = @noinline @load_preference("LOCAL_SERVER", default=@noinline get(ENV, "LOCAL_SERVER", "http://127.0.0.1:10897/v1")); ## CONVERSATION HISTORY From 1180faf674463c27e87c74044b014e0310d37a27 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 13 Feb 2024 22:05:03 +0000 Subject: [PATCH 115/251] Embedding API (#72) --- CHANGELOG.md | 2 + src/Experimental/RAGTools/preparation.jl | 48 +++++++++++++++++++----- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c81eb3e7..b72219ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for [Databricks Foundation Models API](https://docs.databricks.com/en/machine-learning/foundation-models/index.html). Requires two environment variables to be set: `DATABRICKS_API_KEY` and `DATABRICKS_HOST` (the part of the URL before `/serving-endpoints/`) ### Fixed +- Added an option to reduce the "batch size" for the embedding step in building the RAG index (`build_index`, `get_embeddings`). Set `embedding_kwargs = (; target_batch_size_length=10_000, ntasks=1)` if you're having some limit issues with your provider. +- Better error message if RAGTools are only partially imported (requires `LinearAlgebra` and `SparseArrays` to load the extension). ## [0.10.0] diff --git a/src/Experimental/RAGTools/preparation.jl b/src/Experimental/RAGTools/preparation.jl index 924b58b36..cf7304e5f 100644 --- a/src/Experimental/RAGTools/preparation.jl +++ b/src/Experimental/RAGTools/preparation.jl @@ -102,34 +102,51 @@ end get_embeddings(docs::Vector{<:AbstractString}; verbose::Bool = true, cost_tracker = Threads.Atomic{Float64}(0.0), + target_batch_size_length::Int = 80_000, + ntasks::Int = 4 * Threads.nthreads(), kwargs...) Embeds a vector of `docs` using the provided model (kwarg `model`). Tries to batch embedding calls for roughly 80K characters per call (to avoid exceeding the API limit) but reduce network latency. -Note: `docs` are assumed to be already chunked to the reasonable sizes that fit within the embedding context limit. +# Notes +- `docs` are assumed to be already chunked to the reasonable sizes that fit within the embedding context limit. +- If you get errors about exceeding input sizes, first check the `max_length` in your chunks. + If that does NOT resolve the issue, try reducing the `target_batch_size_length` parameter (eg, 10_000) and number of tasks `ntasks=1`. + Some providers cannot handle large batch sizes. # Arguments - `docs`: A vector of strings to be embedded. - `verbose`: A boolean flag for verbose output. Default is `true`. - `model`: The model to use for embedding. Default is `PT.MODEL_EMBEDDING`. - `cost_tracker`: A `Threads.Atomic{Float64}` object to track the total cost of the API calls. Useful to pass the total cost to the parent call. +- `target_batch_size_length`: The target length (in characters) of each batch of document chunks sent for embedding. Default is 80_000 characters. Speeds up embedding process. +- `ntasks`: The number of tasks to use for asyncmap. Default is 4 * Threads.nthreads(). """ function get_embeddings(docs::Vector{<:AbstractString}; verbose::Bool = true, cost_tracker = Threads.Atomic{Float64}(0.0), + target_batch_size_length::Int = 80_000, + ntasks::Int = 4 * Threads.nthreads(), kwargs...) + ## check if extension is available + ext = Base.get_extension(PromptingTools, :RAGToolsExperimentalExt) + if isnothing(ext) + error("you need to also import LinearAlgebra and SparseArrays to use this function") + end verbose && @info "Embedding $(length(docs)) documents..." model = hasproperty(kwargs, :model) ? kwargs.model : PT.MODEL_EMBEDDING # Notice that we embed multiple docs at once, not one by one # OpenAI supports embedding multiple documents to reduce the number of API calls/network latency time # We do batch them just in case the documents are too large (targeting at most 80K characters per call) avg_length = sum(length.(docs)) / length(docs) - embedding_batch_size = floor(Int, 80_000 / avg_length) - embeddings = asyncmap(Iterators.partition(docs, embedding_batch_size)) do docs_chunk + embedding_batch_size = floor(Int, target_batch_size_length / avg_length) + embeddings = asyncmap(Iterators.partition(docs, embedding_batch_size); + ntasks) do docs_chunk msg = aiembed(docs_chunk, + # LinearAlgebra.normalize but imported in RAGToolsExperimentalExt _normalize; verbose = false, kwargs...) @@ -186,13 +203,15 @@ end """ build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, - separators = ["\\n\\n", ". ", "\\n"], max_length::Int = 256, + separators = ["\n\n", ". ", "\n"], max_length::Int = 256, sources::Vector{<:AbstractString} = files_or_docs, - extract_metadata::Bool = false, verbose::Int = 1, + extract_metadata::Bool = false, verbose::Integer = 1, index_id = gensym("ChunkIndex"), metadata_template::Symbol = :RAGExtractMetadataShort, model_embedding::String = PT.MODEL_EMBEDDING, model_metadata::String = PT.MODEL_CHAT, + embedding_kwargs::NamedTuple = NamedTuple(), + metadata_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), cost_tracker = Threads.Atomic{Float64}(0.0)) @@ -203,7 +222,7 @@ optionally extracts metadata, and then compiles this information into a retrieva # Arguments - `files_or_docs`: A vector of valid file paths OR string documents to be indexed (chunked and embedded). - `reader`: A symbol indicating the type of input, can be either `:files` or `:docs`. Default is `:files`. -- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\\n\\n", ". ", "\\n"]`. +- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\n\n", ". ", "\n"]`. - `max_length`: The maximum length of each chunk (if possible with provided separators). Default is 256. - `sources`: A vector of strings indicating the source of each chunk. Default is equal to `files_or_docs` (for `reader=:files`) - `extract_metadata`: A boolean flag indicating whether to extract metadata from each chunk (to build filter `tags` in the index). Default is `false`. @@ -212,7 +231,9 @@ optionally extracts metadata, and then compiles this information into a retrieva - `metadata_template`: A symbol indicating the template to be used for metadata extraction. Default is `:RAGExtractMetadataShort`. - `model_embedding`: The model to use for embedding. - `model_metadata`: The model to use for metadata extraction. -- `api_kwargs`: Parameters to be provided to the API endpoint. +- `api_kwargs`: Parameters to be provided to the API endpoint. Shared across all API calls. +- `embedding_kwargs`: Parameters to be provided to the `get_embedding` function. Useful to change the batch sizes (`target_batch_size_length`) or reduce asyncmap tasks (`ntasks`). +- `metadata_kwargs`: Parameters to be provided to the `get_metadata` function. # Returns - `ChunkIndex`: An object containing the compiled index of chunks, embeddings, tags, vocabulary, and sources. @@ -230,6 +251,13 @@ index = build_index(["file1.txt", "file2.txt"]; extract_metadata=true, verbose=true) ``` + +# Notes +- If you get errors about exceeding embedding input sizes, first check the `max_length` in your chunks. + If that does NOT resolve the issue, try changing the `embedding_kwargs`. + In particular, reducing the `target_batch_size_length` parameter (eg, 10_000) and number of tasks `ntasks=1`. + Some providers cannot handle large batch sizes (eg, Databricks). + """ function build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, separators = ["\n\n", ". ", "\n"], max_length::Int = 256, @@ -239,6 +267,8 @@ function build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = : metadata_template::Symbol = :RAGExtractMetadataShort, model_embedding::String = PT.MODEL_EMBEDDING, model_metadata::String = PT.MODEL_CHAT, + embedding_kwargs::NamedTuple = NamedTuple(), + metadata_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), cost_tracker = Threads.Atomic{Float64}(0.0)) @@ -251,7 +281,7 @@ function build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = : verbose = (verbose > 1), cost_tracker, model = model_embedding, - api_kwargs) + api_kwargs, embedding_kwargs...) ## Extract metadata tags, tags_vocab = if extract_metadata @@ -260,7 +290,7 @@ function build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = : cost_tracker, model = model_metadata, metadata_template, - api_kwargs) + api_kwargs, metadata_kwargs...) # Requires SparseArrays.jl to be loaded build_tags(output_metadata) else From 05f9b84de43b6789f47d2bf1643ab05e20c2550b Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 13 Feb 2024 22:31:10 +0000 Subject: [PATCH 116/251] Add Tavily api (#73) --- CHANGELOG.md | 1 + docs/make.jl | 1 + docs/src/reference_apitools.md | 9 +++ src/Experimental/APITools/APITools.jl | 10 +++ src/Experimental/APITools/tavily_api.jl | 81 ++++++++++++++++++++++++ src/Experimental/Experimental.jl | 4 ++ src/user_preferences.jl | 7 ++ src/utils.jl | 23 +++++-- test/Experimental/APITools/runtests.jl | 8 +++ test/Experimental/APITools/tavily_api.jl | 1 + test/runtests.jl | 1 + test/utils.jl | 1 + 12 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 docs/src/reference_apitools.md create mode 100644 src/Experimental/APITools/APITools.jl create mode 100644 src/Experimental/APITools/tavily_api.jl create mode 100644 test/Experimental/APITools/runtests.jl create mode 100644 test/Experimental/APITools/tavily_api.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index b72219ce4..5c351593d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Support for [Databricks Foundation Models API](https://docs.databricks.com/en/machine-learning/foundation-models/index.html). Requires two environment variables to be set: `DATABRICKS_API_KEY` and `DATABRICKS_HOST` (the part of the URL before `/serving-endpoints/`) +- Experimental support for API tools to enhance your LLM workflows: `Experimental.APITools.create_websearch` function which can execute and summarize a web search (incl. filtering on specific domains). It requires `TAVILY_API_KEY` to be set in the environment. Get your own key from [Tavily](https://tavily.com/) - the free tier enables c. 1000 searches/month, which should be more than enough to get started. ### Fixed - Added an option to reduce the "batch size" for the embedding step in building the RAG index (`build_index`, `get_embeddings`). Set `embedding_kwargs = (; target_batch_size_length=10_000, ntasks=1)` if you're having some limit issues with your provider. diff --git a/docs/make.jl b/docs/make.jl index 9e2e0543b..40a947c55 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -42,6 +42,7 @@ makedocs(; "Experimental Modules" => "reference_experimental.md", "RAGTools" => "reference_ragtools.md", "AgentTools" => "reference_agenttools.md", + "APITools" => "reference_apitools.md", ], ]) diff --git a/docs/src/reference_apitools.md b/docs/src/reference_apitools.md new file mode 100644 index 000000000..8e568b5be --- /dev/null +++ b/docs/src/reference_apitools.md @@ -0,0 +1,9 @@ +# Reference for APITools + +```@index +Modules = [PromptingTools.Experimental.APITools] +``` + +```@autodocs +Modules = [PromptingTools.Experimental.APITools] +``` diff --git a/src/Experimental/APITools/APITools.jl b/src/Experimental/APITools/APITools.jl new file mode 100644 index 000000000..4c2d34352 --- /dev/null +++ b/src/Experimental/APITools/APITools.jl @@ -0,0 +1,10 @@ +module APITools + +using HTTP, JSON3 +using PromptingTools +const PT = PromptingTools + +export create_websearch +include("tavily_api.jl") + +end # module \ No newline at end of file diff --git a/src/Experimental/APITools/tavily_api.jl b/src/Experimental/APITools/tavily_api.jl new file mode 100644 index 000000000..f827ca4e1 --- /dev/null +++ b/src/Experimental/APITools/tavily_api.jl @@ -0,0 +1,81 @@ +""" + tavily_api(; + api_key::AbstractString, + endpoint::String = "search", + url::AbstractString = "https://api.tavily.com", + http_kwargs::NamedTuple = NamedTuple(), + kwargs...) + +Sends API requests to [Tavily](https://tavily.com) and returns the response. +""" +function tavily_api(; + api_key::AbstractString, + endpoint::String = "search", + url::AbstractString = "https://api.tavily.com", + http_kwargs::NamedTuple = NamedTuple(), + kwargs...) + @assert !isempty(api_key) "Tavily `api_key` cannot be empty. Check `PT.TAVILY_API_KEY` or pass it as a keyword argument." + + ## api_key is sent in the POST body, not headers + input_body = Dict("api_key" => api_key, kwargs...) + + # eg, https://api.tavily.com/search + api_url = string(url, "/", endpoint) + headers = PT.auth_header(nothing) # no API key provided + resp = HTTP.post(api_url, headers, + JSON3.write(input_body); http_kwargs...) + body = JSON3.read(resp.body) + return (; response = body, status = resp.status) +end + +""" + create_websearch(query::AbstractString; + api_key::AbstractString, + search_depth::AbstractString = "basic") + +# Arguments +- `query::AbstractString`: The query to search for. +- `api_key::AbstractString`: The API key to use for the search. Get an API key from [Tavily](https://tavily.com). +- `search_depth::AbstractString`: The depth of the search. Can be either "basic" or "advanced". Default is "basic". Advanced search calls equal to 2 requests. +- `include_answer::Bool`: Whether to include the answer in the search results. Default is `false`. +- `include_raw_content::Bool`: Whether to include the raw content in the search results. Default is `false`. +- `max_results::Integer`: The maximum number of results to return. Default is 5. +- `include_images::Bool`: Whether to include images in the search results. Default is `false`. +- `include_domains::AbstractVector{<:AbstractString}`: A list of domains to include in the search results. Default is an empty list. +- `exclude_domains::AbstractVector{<:AbstractString}`: A list of domains to exclude from the search results. Default is an empty list. + +# Example +```julia +r = create_websearch("Who is King Charles?") +``` + +Even better, you can get not just the results but also the answer: +```julia +r = create_websearch("Who is King Charles?"; include_answer = true) +``` + +See [Rest API documentation](https://docs.tavily.com/docs/tavily-api/rest_api) for more information. + +""" +function create_websearch(query::AbstractString; + api_key::AbstractString = PT.TAVILY_API_KEY, + search_depth::AbstractString = "basic", + include_answer::Bool = false, + include_raw_content::Bool = false, + max_results::Integer = 5, + include_images::Bool = false, + include_domains::AbstractVector{<:AbstractString} = String[], + exclude_domains::AbstractVector{<:AbstractString} = String[],) + @assert search_depth in ["basic", "advanced"] "Search depth must be either 'basic' or 'advanced'" + @assert max_results>0 "Max results must be a positive integer" + + tavily_api(; api_key, endpoint = "search", + query, + search_depth, + include_answer, + include_raw_content, + max_results, + include_images, + include_domains, + exclude_domains) +end \ No newline at end of file diff --git a/src/Experimental/Experimental.jl b/src/Experimental/Experimental.jl index 408e2091d..df8eaf182 100644 --- a/src/Experimental/Experimental.jl +++ b/src/Experimental/Experimental.jl @@ -7,6 +7,7 @@ It is not included in the main module, so it must be explicitly imported. Contains: - `RAGTools`: Retrieval-Augmented Generation (RAG) functionality. - `AgentTools`: Agentic functionality - lazy calls for building pipelines (eg, `AIGenerate`) and `AICodeFixer`. +- `APITools`: APIs to complement GenAI workflows (eg, Tavily Search API). """ module Experimental @@ -16,4 +17,7 @@ include("RAGTools/RAGTools.jl") export AgentTools include("AgentTools/AgentTools.jl") +export APITools +include("APITools/APITools.jl") + end # module Experimental diff --git a/src/user_preferences.jl b/src/user_preferences.jl index d9ab98bfe..f2e09cf84 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -16,6 +16,7 @@ Check your preferences by calling `get_preferences(key::String)`. - `COHERE_API_KEY`: The API key for the Cohere API. See [Cohere's documentation](https://docs.cohere.com/docs/the-cohere-platform) for more information. - `DATABRICKS_API_KEY`: The API key for the Databricks Foundation Model API. See [Databricks' documentation](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html) for more information. - `DATABRICKS_HOST`: The host for the Databricks API. See [Databricks' documentation](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html) for more information. +- `TAVILY_API_KEY`: The API key for the Tavily Search API. Register [here](https://tavily.com/). See more information [here](https://docs.tavily.com/docs/tavily-api/rest_api). - `MODEL_CHAT`: The default model to use for aigenerate and most ai* calls. See `MODEL_REGISTRY` for a list of available models or define your own. - `MODEL_EMBEDDING`: The default model to use for aiembed (embedding documents). See `MODEL_REGISTRY` for a list of available models or define your own. - `PROMPT_SCHEMA`: The default prompt schema to use for aigenerate and most ai* calls (if not specified in `MODEL_REGISTRY`). Set as a string, eg, `"OpenAISchema"`. @@ -37,6 +38,7 @@ Define your `register_model!()` calls in your `startup.jl` file to make them ava - `LOCAL_SERVER`: The URL of the local server to use for `ai*` calls. Defaults to `http://localhost:10897/v1`. This server is called when you call `model="local"` - `DATABRICKS_API_KEY`: The API key for the Databricks Foundation Model API. - `DATABRICKS_HOST`: The host for the Databricks API. +- `TAVILY_API_KEY`: The API key for the Tavily Search API. Register [here](https://tavily.com/). See more information [here](https://docs.tavily.com/docs/tavily-api/rest_api). Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. @@ -65,6 +67,7 @@ function set_preferences!(pairs::Pair{String, <:Any}...) "COHERE_API_KEY", "DATABRICKS_API_KEY", "DATABRICKS_HOST", + "TAVILY_API_KEY", "MODEL_CHAT", "MODEL_EMBEDDING", "MODEL_ALIASES", @@ -103,6 +106,7 @@ function get_preferences(key::String) "COHERE_API_KEY", "DATABRICKS_API_KEY", "DATABRICKS_HOST", + "TAVILY_API_KEY", "MODEL_CHAT", "MODEL_EMBEDDING", "MODEL_ALIASES", @@ -140,6 +144,9 @@ const DATABRICKS_API_KEY::String = @noinline @load_preference("DATABRICKS_API_KE const DATABRICKS_HOST::String = @noinline @load_preference("DATABRICKS_HOST", default=@noinline get(ENV, "DATABRICKS_HOST", "")); +const TAVILY_API_KEY::String = @noinline @load_preference("TAVILY_API_KEY", + default=@noinline get(ENV, "TAVILY_API_KEY", "")); + ## Address of the local server const LOCAL_SERVER::String = @noinline @load_preference("LOCAL_SERVER", default=@noinline get(ENV, "LOCAL_SERVER", "http://127.0.0.1:10897/v1")); diff --git a/src/utils.jl b/src/utils.jl index 5b0f07458..c8cfee68f 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -362,15 +362,24 @@ end function preview end """ - auth_header(api_key::String) + auth_header(api_key::Union{Nothing, AbstractString}; + extra_headers::AbstractVector{Pair{String, String}} = Vector{Pair{String, String}}[], + kwargs...) -Builds an authorization header for API calls with the given API key. +Creates the authentication headers for any API request. Assumes that the communication is done in JSON format. """ -function auth_header(api_key::String) - isempty(api_key) && throw(ArgumentError("api_key cannot be empty")) - [ - "Authorization" => "Bearer $api_key", +function auth_header(api_key::Union{Nothing, AbstractString}; + extra_headers::AbstractVector = Vector{ + Pair{String, String}, + }[], + kwargs...) + !isnothing(api_key) && isempty(api_key) && + throw(ArgumentError("`api_key` cannot be empty")) + headers = [ "Content-Type" => "application/json", "Accept" => "application/json", + extra_headers..., ] -end + !isnothing(api_key) && pushfirst!(headers, "Authorization" => "Bearer $api_key") + return headers +end \ No newline at end of file diff --git a/test/Experimental/APITools/runtests.jl b/test/Experimental/APITools/runtests.jl new file mode 100644 index 000000000..19a607f17 --- /dev/null +++ b/test/Experimental/APITools/runtests.jl @@ -0,0 +1,8 @@ +using Test +using PromptingTools +using PromptingTools.Experimental.APITools +const PT = PromptingTools + +## @testset "APITools" begin +## include("tavily_api.jl") +## end diff --git a/test/Experimental/APITools/tavily_api.jl b/test/Experimental/APITools/tavily_api.jl new file mode 100644 index 000000000..e74519f11 --- /dev/null +++ b/test/Experimental/APITools/tavily_api.jl @@ -0,0 +1 @@ +# TODO: hard to test the API itself? \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index e0699bfc7..ea159e02b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -44,4 +44,5 @@ end @testset "Experimental" begin include("Experimental/RAGTools/runtests.jl") include("Experimental/AgentTools/runtests.jl") + include("Experimental/APITools/runtests.jl") end diff --git a/test/utils.jl b/test/utils.jl index 3c21bd5ef..112748a47 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -236,4 +236,5 @@ end "Accept" => "application/json", ] @test_throws ArgumentError auth_header("") + @test length(auth_header(nothing)) == 2 end From 15c4d08d87988bc18f171ca9660bd531fd88b20c Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 14 Feb 2024 21:35:18 +0000 Subject: [PATCH 117/251] Update AIrag kwargs (#74) --- CHANGELOG.md | 9 ++++++++ Project.toml | 4 ++-- src/Experimental/RAGTools/generation.jl | 25 ++++++++++++++++----- src/user_preferences.jl | 28 +++++++++++++++--------- test/Experimental/RAGTools/generation.jl | 10 +++++++++ 5 files changed, 59 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c351593d..950983c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [0.12.0] + +### Added +- Added more specific kwargs in `Experimental.RAGTools.airag` to give more control over each type of AI call (ie, `aiembed_kwargs`, `aigenerate_kwargs`, `aiextract_kwargs`) +- Move up compat bounds for OpenAI.jl to 0.9 + +### Fixed +- Fixed a bug where obtaining an API_KEY from ENV would get precompiled as well, causing an error if the ENV was not set at the time of precompilation. Now, we save the `get(ENV...)` into a separate variable to avoid being compiled away. + ## [0.11.0] ### Added diff --git a/Project.toml b/Project.toml index 3a0a192ae..9f1c151f3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.11.0" +version = "0.12.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -31,7 +31,7 @@ JSON3 = "1" LinearAlgebra = "<0.0.1, 1" Logging = "<0.0.1, 1" Markdown = "<0.0.1, 1" -OpenAI = "0.8.7" +OpenAI = "0.9" Pkg = "<0.0.1, 1" PrecompileTools = "1" Preferences = "1" diff --git a/src/Experimental/RAGTools/generation.jl b/src/Experimental/RAGTools/generation.jl index 712acf07f..8de2f0225 100644 --- a/src/Experimental/RAGTools/generation.jl +++ b/src/Experimental/RAGTools/generation.jl @@ -49,6 +49,9 @@ end return_context::Bool = false, verbose::Bool = true, rerank_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), + aiembed_kwargs::NamedTuple = NamedTuple(), + aigenerate_kwargs::NamedTuple = NamedTuple(), + aiextract_kwargs::NamedTuple = NamedTuple(), kwargs...) Generates a response for a given question using a Retrieval-Augmented Generation (RAG) approach. @@ -71,7 +74,10 @@ The function selects relevant chunks from an `ChunkIndex`, optionally filters th - `chunks_window_margin::Tuple{Int,Int}`: The window size around each chunk to consider for context building. See `?build_context` for more information. - `return_context::Bool`: If `true`, returns the context used for RAG along with the response. - `verbose::Bool`: If `true`, enables verbose logging. -- `api_kwargs`: API parameters that will be forwarded to the API calls +- `api_kwargs`: API parameters that will be forwarded to ALL of the API calls (`aiembed`, `aigenerate`, and `aiextract`). +- `aiembed_kwargs`: API parameters that will be forwarded to the `aiembed` call. If you need to provide `api_kwargs` only to this function, simply add them as a keyword argument, eg, `aiembed_kwargs = (; api_kwargs = (; x=1))`. +- `aigenerate_kwargs`: API parameters that will be forwarded to the `aigenerate` call. If you need to provide `api_kwargs` only to this function, simply add them as a keyword argument, eg, `aigenerate_kwargs = (; api_kwargs = (; temperature=0.3))`. +- `aiextract_kwargs`: API parameters that will be forwarded to the `aiextract` call for the metadata extraction. # Returns - If `return_context` is `false`, returns the generated message (`msg`). @@ -109,6 +115,9 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC return_context::Bool = false, verbose::Bool = true, rerank_kwargs::NamedTuple = NamedTuple(), api_kwargs::NamedTuple = NamedTuple(), + aiembed_kwargs::NamedTuple = NamedTuple(), + aigenerate_kwargs::NamedTuple = NamedTuple(), + aiextract_kwargs::NamedTuple = NamedTuple(), kwargs...) ## Note: Supports only single ChunkIndex for now ## Checks @@ -117,21 +126,26 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC placeholders = only(aitemplates(rag_template)).variables # only one template should be found @assert (:question in placeholders)&&(:context in placeholders) "Provided RAG Template $(rag_template) is not suitable. It must have placeholders: `question` and `context`." + ## Embedding + joined_kwargs = isempty(api_kwargs) ? aiembed_kwargs : + merge(aiembed_kwargs, (; api_kwargs)) question_emb = aiembed(question, _normalize; model = model_embedding, - verbose, api_kwargs).content .|> Float32 # no need for Float64 + verbose, joined_kwargs...).content .|> Float32 # no need for Float64 emb_candidates = find_closest(index, question_emb; top_k, minimum_similarity) tag_candidates = if tag_filter == :auto && !isnothing(tags(index)) && !isempty(model_metadata) _check_aiextract_capability(model_metadata) + joined_kwargs = isempty(api_kwargs) ? aiextract_kwargs : + merge(aiextract_kwargs, (; api_kwargs)) # extract metadata via LLM call metadata_ = try msg = aiextract(metadata_template; return_type = MaybeMetadataItems, text = question, instructions = "In addition to extracted items, suggest 2-3 filter keywords that could be relevant to answer this question.", - verbose, model = model_metadata, api_kwargs) + verbose, model = model_metadata, joined_kwargs...) ## eg, ["software:::pandas", "language:::python", "julia_package:::dataframes"] ## we split it and take only the keyword, not the category metadata_extract(msg.content.items) |> @@ -162,10 +176,11 @@ function airag(index::AbstractChunkIndex, rag_template::Symbol = :RAGAnswerFromC context = build_context(index, reranked_candidates; chunks_window_margin) ## LLM call + joined_kwargs = isempty(api_kwargs) ? aigenerate_kwargs : + merge(aigenerate_kwargs, (; api_kwargs)) msg = aigenerate(rag_template; question, context = join(context, "\n\n"), model = model_chat, verbose, - api_kwargs, - kwargs...) + joined_kwargs...) if return_context # for evaluation rag_context = RAGContext(; diff --git a/src/user_preferences.jl b/src/user_preferences.jl index f2e09cf84..a5ca4bb60 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -126,30 +126,38 @@ const MODEL_EMBEDDING::String = @load_preference("MODEL_EMBEDDING", # const PROMPT_SCHEMA = OpenAISchema() # First, load from preferences, then from environment variables -const OPENAI_API_KEY::String = @noinline @load_preference("OPENAI_API_KEY", - default=@noinline get(ENV, "OPENAI_API_KEY", "")); +# Note: We load first into a variable `temp_` to avoid inlining of the get(ENV...) call +_temp = get(ENV, "OPENAI_API_KEY", "") +const OPENAI_API_KEY::String = @load_preference("OPENAI_API_KEY", + default=_temp); # Note: Disable this warning by setting OPENAI_API_KEY to anything isempty(OPENAI_API_KEY) && @warn "OPENAI_API_KEY variable not set! OpenAI models will not be available - set API key directly via `PromptingTools.OPENAI_API_KEY=`!" -const MISTRALAI_API_KEY::String = @noinline @load_preference("MISTRALAI_API_KEY", - default=@noinline get(ENV, "MISTRALAI_API_KEY", "")); +_temp = get(ENV, "MISTRALAI_API_KEY", "") +const MISTRALAI_API_KEY::String = @load_preference("MISTRALAI_API_KEY", + default=_temp); -const COHERE_API_KEY::String = @noinline @load_preference("COHERE_API_KEY", - default=@noinline get(ENV, "COHERE_API_KEY", "")); +_temp = get(ENV, "COHERE_API_KEY", "") +const COHERE_API_KEY::String = @load_preference("COHERE_API_KEY", + default=_temp); +_temp = get(ENV, "DATABRICKS_API_KEY", "") const DATABRICKS_API_KEY::String = @noinline @load_preference("DATABRICKS_API_KEY", - default=@noinline get(ENV, "DATABRICKS_API_KEY", "")); + default=_temp); +_temp = get(ENV, "DATABRICKS_HOST", "") const DATABRICKS_HOST::String = @noinline @load_preference("DATABRICKS_HOST", - default=@noinline get(ENV, "DATABRICKS_HOST", "")); + default=_temp); +_temp = get(ENV, "TAVILY_API_KEY", "") const TAVILY_API_KEY::String = @noinline @load_preference("TAVILY_API_KEY", - default=@noinline get(ENV, "TAVILY_API_KEY", "")); + default=_temp); +_temp = get(ENV, "LOCAL_SERVER", "") ## Address of the local server const LOCAL_SERVER::String = @noinline @load_preference("LOCAL_SERVER", - default=@noinline get(ENV, "LOCAL_SERVER", "http://127.0.0.1:10897/v1")); + default=_temp); ## CONVERSATION HISTORY """ diff --git a/test/Experimental/RAGTools/generation.jl b/test/Experimental/RAGTools/generation.jl index b19d03a44..6fbb95057 100644 --- a/test/Experimental/RAGTools/generation.jl +++ b/test/Experimental/RAGTools/generation.jl @@ -96,6 +96,16 @@ end return_context = false) @test occursin("Time?", msg.content) + # test kwargs passing + api_kwargs = (; url = "http://localhost:$(PORT)") + msg = airag(index; question = "Time?", model_embedding = "mock-emb", + model_chat = "mock-gen", + model_metadata = "mock-meta", + tag_filter = ["yes"], + return_context = false, aiembed_kwargs = (; api_kwargs), + aigenerate_kwargs = (; api_kwargs), aiextract_kwargs = (; api_kwargs)) + @test occursin("Time?", msg.content) + ## Test different kwargs msg, ctx = airag(index; question = "Time?", model_embedding = "mock-emb", model_chat = "mock-gen", From 703902c0465e66c4407b6785cc7df0270771ec52 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 17 Feb 2024 08:31:44 +0000 Subject: [PATCH 118/251] Add Google API (#75) --- CHANGELOG.md | 1 + Project.toml | 5 +- ext/GoogleGenAIPromptingToolsExt.jl | 31 +++++ src/PromptingTools.jl | 1 + src/llm_google.jl | 193 ++++++++++++++++++++++++++++ src/llm_interface.jl | 13 ++ src/user_preferences.jl | 75 +++++------ test/extraction.jl | 4 +- test/llm_google.jl | 177 +++++++++++++++++++++++++ test/runtests.jl | 3 +- 10 files changed, 462 insertions(+), 41 deletions(-) create mode 100644 ext/GoogleGenAIPromptingToolsExt.jl create mode 100644 src/llm_google.jl create mode 100644 test/llm_google.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index 950983c00..6131c7b12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). ### Fixed diff --git a/Project.toml b/Project.toml index 9f1c151f3..4d0b7ee39 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.12.0" +version = "0.13.0-DEV" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -15,17 +15,20 @@ Preferences = "21216c6a-2e73-6563-6e65-726566657250" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [weakdeps] +GoogleGenAI = "903d41d1-eaca-47dd-943b-fee3930375ab" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] +GoogleGenAIPromptingToolsExt = ["GoogleGenAI"] MarkdownPromptingToolsExt = ["Markdown"] RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] [compat] Aqua = "0.7" Base64 = "<0.0.1, 1" +GoogleGenAI = "0.1.0" HTTP = "1" JSON3 = "1" LinearAlgebra = "<0.0.1, 1" diff --git a/ext/GoogleGenAIPromptingToolsExt.jl b/ext/GoogleGenAIPromptingToolsExt.jl new file mode 100644 index 000000000..d5ff80140 --- /dev/null +++ b/ext/GoogleGenAIPromptingToolsExt.jl @@ -0,0 +1,31 @@ +module GoogleGenAIPromptingToolsExt + +using GoogleGenAI +using PromptingTools +using HTTP, JSON3 +const PT = PromptingTools + +"Wrapper for GoogleGenAI.generate_content." +function PromptingTools.ggi_generate_content(prompt_schema::PT.AbstractGoogleSchema, + api_key::AbstractString, model_name::AbstractString, + conversation; http_kwargs, api_kwargs...) + ## Build the provider + provider = GoogleGenAI.GoogleProvider(; api_key) + url = "$(provider.base_url)/models/$model_name:generateContent?key=$(provider.api_key)" + generation_config = Dict{String, Any}() + for (key, value) in api_kwargs + generation_config[string(key)] = value + end + + body = Dict("contents" => conversation, + "generationConfig" => generation_config) + response = HTTP.post(url; headers = Dict("Content-Type" => "application/json"), + body = JSON3.write(body), http_kwargs...) + if response.status >= 200 && response.status < 300 + return GoogleGenAI._parse_response(response) + else + error("Request failed with status $(response.status): $(String(response.body))") + end +end + +end # end of module \ No newline at end of file diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 867cd0e87..04b70ea3d 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -65,6 +65,7 @@ include("llm_shared.jl") include("llm_openai.jl") include("llm_ollama_managed.jl") include("llm_ollama.jl") +include("llm_google.jl") ## Convenience utils export @ai_str, @aai_str, @ai!_str, @aai!_str diff --git a/src/llm_google.jl b/src/llm_google.jl new file mode 100644 index 000000000..516c5cfe1 --- /dev/null +++ b/src/llm_google.jl @@ -0,0 +1,193 @@ +## Rendering of converation history for the OpenAI API +""" + render(schema::AbstractGoogleSchema, + messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + kwargs...) + +Builds a history of the conversation to provide the prompt to the API. All unspecified kwargs are passed as replacements such that `{{key}}=>value` in the template. + +# Keyword Arguments +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. + +""" +function render(schema::AbstractGoogleSchema, + messages::Vector{<:AbstractMessage}; + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + kwargs...) + ## + ## First pass: keep the message types but make the replacements provided in `kwargs` + messages_replaced = render(NoSchema(), messages; conversation, kwargs...) + + ## Second pass: convert to the OpenAI schema + conversation = Dict{String, Any}[] + + # replace any handlebar variables in the messages + for msg in messages_replaced + role = if msg isa SystemMessage + ## No system message, we need to merge with UserMessage, see below + "user" + elseif msg isa UserMessage + "user" + elseif msg isa AIMessage + "model" + end + push!(conversation, Dict("role" => role, "parts" => [Dict("text" => msg.content)])) + end + ## Merge any subsequent UserMessages + merged_conversation = Dict{String, Any}[] + # run n-1 times, look at the current item and the next one + i = 1 + while i <= (length(conversation) - 1) + next_i = i + 1 + if conversation[i]["role"] == "user" && conversation[next_i]["role"] == "user" + ## Concat the user messages to together, put two newlines + txt1 = conversation[i]["parts"][1]["text"] + txt2 = conversation[next_i]["parts"][1]["text"] + merged_text = isempty(txt1) || isempty(txt2) ? txt1 * txt2 : + txt1 * "\n\n" * txt2 + new_msg = Dict("role" => "user", "parts" => [Dict("text" => merged_text)]) + push!(merged_conversation, new_msg) + i += 2 + else + push!(merged_conversation, conversation[i]) + i += 1 + end + end + ## Add last message + if i == length(conversation) + push!(merged_conversation, conversation[end]) + end + return merged_conversation +end + +"Stub - to be extended in extension: GoogleGenAIPromptingToolsExt. `ggi` stands for GoogleGenAI" +function ggi_generate_content end +function ggi_generate_content(schema::TestEchoGoogleSchema, api_key::AbstractString, + model::AbstractString, + conversation; kwargs...) + schema.model_id = model + schema.inputs = conversation + return schema +end + +## User-Facing API +""" + aigenerate(prompt_schema::AbstractGoogleSchema, prompt::ALLOWED_PROMPT_TYPE; + verbose::Bool = true, + api_key::String = GOOGLE_API_KEY, + model::String = "gemini-pro", return_all::Bool = false, dry_run::Bool = false, + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + +Generate an AI response based on a given prompt using the OpenAI API. + +# Arguments +- `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema`) +- `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` +- `verbose`: A boolean indicating whether to print additional information. +- `api_key`: A string representing the API key for accessing the OpenAI API. +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). +- `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). +- `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. +- `http_kwargs`: A named tuple of HTTP keyword arguments. +- `api_kwargs`: A named tuple of API keyword arguments. +- `kwargs`: Prompt variables to be used to fill the prompt/template + +# Returns + +If `return_all=false` (default): +- `msg`: An `AIMessage` object representing the generated AI message, including the content, status, tokens, and elapsed time. + Use `msg.content` to access the extracted string. + +If `return_all=true`: +- `conversation`: A vector of `AbstractMessage` objects representing the conversation history, including the response from the AI model (`AIMessage`). + +See also: `ai_str`, `aai_str`, `aiembed`, `aiclassify`, `aiextract`, `aiscan`, `aitemplates` + +# Example + +Simple hello world to test the API: +```julia +result = aigenerate("Say Hi!") +# [ Info: Tokens: 29 @ Cost: \$0.0 in 1.0 seconds +# AIMessage("Hello! How can I assist you today?") +``` + +`result` is an `AIMessage` object. Access the generated string via `content` property: +```julia +typeof(result) # AIMessage{SubString{String}} +propertynames(result) # (:content, :status, :tokens, :elapsed +result.content # "Hello! How can I assist you today?" +``` +___ +You can use string interpolation: +```julia +a = 1 +msg=aigenerate("What is `\$a+\$a`?") +msg.content # "The sum of `1+1` is `2`." +``` +___ +You can provide the whole conversation or more intricate prompts as a `Vector{AbstractMessage}`: +```julia +const PT = PromptingTools + +conversation = [ + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] +msg=aigenerate(conversation) +# AIMessage("Ah, strong feelings you have for your iPhone. A Jedi's path, this is not... ") +``` +""" +function aigenerate(prompt_schema::AbstractGoogleSchema, prompt::ALLOWED_PROMPT_TYPE; + verbose::Bool = true, + api_key::String = GOOGLE_API_KEY, + model::String = "gemini-pro", return_all::Bool = false, dry_run::Bool = false, + conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], + http_kwargs::NamedTuple = (retry_non_idempotent = true, + retries = 5, + readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + kwargs...) + ## + global MODEL_ALIASES + + ## Check that package GoogleGenAI is loaded + ext = Base.get_extension(PromptingTools, :GoogleGenAIPromptingToolsExt) + if isnothing(ext) && !(prompt_schema isa TestEchoGoogleSchema) + throw(ArgumentError("You need to also import GoogleGenAI package to use this function")) + end + + ## Find the unique ID for the model alias provided + model_id = get(MODEL_ALIASES, model, model) + conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) + + if !dry_run + time = @elapsed r = ggi_generate_content(prompt_schema, api_key, + model_id, + conv_rendered; + http_kwargs, + api_kwargs...) + msg = AIMessage(; + content = r.text |> strip, + status = 200, + tokens = (0, 0), + elapsed = time) + ## Reporting + verbose && @info _report_stats(msg, model_id) + else + msg = nothing + end + ## Select what to return + output = finalize_outputs(prompt, + conv_rendered, + msg; + conversation, + return_all, + dry_run, + kwargs...) + + return output +end \ No newline at end of file diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 95dcca998..04f24a56c 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -207,6 +207,19 @@ struct OllamaManagedSchema <: AbstractOllamaManagedSchema end inputs::Any = nothing end +abstract type AbstractGoogleSchema <: AbstractPromptSchema end + +"Calls Google's Gemini API. See more information [here](https://aistudio.google.com/). It's available only for _some_ regions." +struct GoogleSchema <: AbstractGoogleSchema end + +"Echoes the user's input back to them. Used for testing the implementation" +@kwdef mutable struct TestEchoGoogleSchema <: AbstractGoogleSchema + text::Any + status::Integer + model_id::String = "" + inputs::Any = nothing +end + ## Dispatch into a default schema (can be set by Preferences.jl) # Since we load it as strings, we need to convert it to a symbol and instantiate it global PROMPT_SCHEMA::AbstractPromptSchema = @load_preference("PROMPT_SCHEMA", diff --git a/src/user_preferences.jl b/src/user_preferences.jl index a5ca4bb60..e26066cf6 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -17,6 +17,7 @@ Check your preferences by calling `get_preferences(key::String)`. - `DATABRICKS_API_KEY`: The API key for the Databricks Foundation Model API. See [Databricks' documentation](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html) for more information. - `DATABRICKS_HOST`: The host for the Databricks API. See [Databricks' documentation](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html) for more information. - `TAVILY_API_KEY`: The API key for the Tavily Search API. Register [here](https://tavily.com/). See more information [here](https://docs.tavily.com/docs/tavily-api/rest_api). +- `GOOGLE_API_KEY`: The API key for Google Gemini models. Get yours from [here](https://ai.google.dev/). If you see a documentation page ("Available languages and regions for Google AI Studio and Gemini API"), it means that it's not yet available in your region. - `MODEL_CHAT`: The default model to use for aigenerate and most ai* calls. See `MODEL_REGISTRY` for a list of available models or define your own. - `MODEL_EMBEDDING`: The default model to use for aiembed (embedding documents). See `MODEL_REGISTRY` for a list of available models or define your own. - `PROMPT_SCHEMA`: The default prompt schema to use for aigenerate and most ai* calls (if not specified in `MODEL_REGISTRY`). Set as a string, eg, `"OpenAISchema"`. @@ -39,13 +40,29 @@ Define your `register_model!()` calls in your `startup.jl` file to make them ava - `DATABRICKS_API_KEY`: The API key for the Databricks Foundation Model API. - `DATABRICKS_HOST`: The host for the Databricks API. - `TAVILY_API_KEY`: The API key for the Tavily Search API. Register [here](https://tavily.com/). See more information [here](https://docs.tavily.com/docs/tavily-api/rest_api). +- `GOOGLE_API_KEY`: The API key for Google Gemini models. Get yours from [here](https://ai.google.dev/). If you see a documentation page ("Available languages and regions for Google AI Studio and Gemini API"), it means that it's not yet available in your region. -Preferences.jl takes priority over ENV variables, so if you set a preference, it will override the ENV variable. +Preferences.jl takes priority over ENV variables, so if you set a preference, it will take precedence over the ENV variable. WARNING: NEVER EVER sync your `LocalPreferences.toml` file! It contains your API key and other sensitive information!!! """ const PREFERENCES = nothing +"Keys that are allowed to be set via `set_preferences!`" +const ALLOWED_PREFERENCES = ["MISTRALAI_API_KEY", + "OPENAI_API_KEY", + "COHERE_API_KEY", + "DATABRICKS_API_KEY", + "DATABRICKS_HOST", + "TAVILY_API_KEY", + "GOOGLE_API_KEY", + "MODEL_CHAT", + "MODEL_EMBEDDING", + "MODEL_ALIASES", + "PROMPT_SCHEMA", + "MAX_HISTORY_LENGTH", + "LOCAL_SERVER"] + """ set_preferences!(pairs::Pair{String, <:Any}...) @@ -61,22 +78,9 @@ PromptingTools.set_preferences!("OPENAI_API_KEY" => "key1", "MODEL_CHAT" => "cha ``` """ function set_preferences!(pairs::Pair{String, <:Any}...) - allowed_preferences = [ - "MISTRALAI_API_KEY", - "OPENAI_API_KEY", - "COHERE_API_KEY", - "DATABRICKS_API_KEY", - "DATABRICKS_HOST", - "TAVILY_API_KEY", - "MODEL_CHAT", - "MODEL_EMBEDDING", - "MODEL_ALIASES", - "PROMPT_SCHEMA", - "MAX_HISTORY_LENGTH", - "LOCAL_SERVER", - ] + global ALLOWED_PREFERENCES for (key, value) in pairs - @assert key in allowed_preferences "Unknown preference '$key'! (Allowed preferences: $(join(allowed_preferences,", "))" + @assert key in ALLOWED_PREFERENCES "Unknown preference '$key'! (Allowed preferences: $(join(ALLOWED_PREFERENCES,", "))" @set_preferences!(key=>value) if key == "MODEL_ALIASES" || key == "PROMPT_SCHEMA" # cannot change in the same session @@ -100,21 +104,8 @@ PromptingTools.get_preferences("MODEL_CHAT") ``` """ function get_preferences(key::String) - allowed_preferences = [ - "MISTRALAI_API_KEY", - "OPENAI_API_KEY", - "COHERE_API_KEY", - "DATABRICKS_API_KEY", - "DATABRICKS_HOST", - "TAVILY_API_KEY", - "MODEL_CHAT", - "MODEL_EMBEDDING", - "MODEL_ALIASES", - "PROMPT_SCHEMA", - "MAX_HISTORY_LENGTH", - "LOCAL_SERVER", - ] - @assert key in allowed_preferences "Unknown preference '$key'! (Allowed preferences: $(join(allowed_preferences,", "))" + global ALLOWED_PREFERENCES + @assert key in ALLOWED_PREFERENCES "Unknown preference '$key'! (Allowed preferences: $(join(ALLOWED_PREFERENCES,", "))" getproperty(@__MODULE__, Symbol(key)) end @@ -143,20 +134,24 @@ const COHERE_API_KEY::String = @load_preference("COHERE_API_KEY", default=_temp); _temp = get(ENV, "DATABRICKS_API_KEY", "") -const DATABRICKS_API_KEY::String = @noinline @load_preference("DATABRICKS_API_KEY", +const DATABRICKS_API_KEY::String = @load_preference("DATABRICKS_API_KEY", default=_temp); _temp = get(ENV, "DATABRICKS_HOST", "") -const DATABRICKS_HOST::String = @noinline @load_preference("DATABRICKS_HOST", +const DATABRICKS_HOST::String = @load_preference("DATABRICKS_HOST", default=_temp); _temp = get(ENV, "TAVILY_API_KEY", "") -const TAVILY_API_KEY::String = @noinline @load_preference("TAVILY_API_KEY", +const TAVILY_API_KEY::String = @load_preference("TAVILY_API_KEY", + default=_temp); + +_temp = get(ENV, "GOOGLE_API_KEY", "") +const GOOGLE_API_KEY::String = @load_preference("GOOGLE_API_KEY", default=_temp); _temp = get(ENV, "LOCAL_SERVER", "") ## Address of the local server -const LOCAL_SERVER::String = @noinline @load_preference("LOCAL_SERVER", +const LOCAL_SERVER::String = @load_preference("LOCAL_SERVER", default=_temp); ## CONVERSATION HISTORY @@ -283,7 +278,8 @@ aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", "yi34c" => "yi:34b-chat", "oh25" => "openhermes2.5-mistral", "starling" => "starling-lm", - "local" => "local-server"), + "local" => "local-server", + "gemini" => "gemini-pro"), ## Load aliases from preferences as well @load_preference("MODEL_ALIASES", default=Dict{String, String}())) @@ -404,7 +400,12 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", LocalServerOpenAISchema(), 0.0, 0.0, - "Local server, eg, powered by [Llama.jl](https://github.com/marcom/Llama.jl). Model is specified when instantiating the server itself.")) + "Local server, eg, powered by [Llama.jl](https://github.com/marcom/Llama.jl). Model is specified when instantiating the server itself."), + "gemini-pro" => ModelSpec("gemini-pro", + GoogleSchema(), + 0.0, #unknown + 0.0, #unknown + "Gemini Pro is a LLM from Google. For more information, see [models](https://ai.google.dev/models/gemini).")) ### Model Registry Structure @kwdef mutable struct ModelRegistry diff --git a/test/extraction.jl b/test/extraction.jl index 6d50d9d98..a10b8478b 100644 --- a/test/extraction.jl +++ b/test/extraction.jl @@ -195,12 +195,12 @@ end end @testset "to_json_schema-ItemsExtract" begin "Represents person's age, height, and weight" - struct MyMeasurement1 + struct MyMeasurement11 age::Int height::Union{Int, Nothing} weight::Union{Nothing, Float64} end - schema = to_json_schema(ItemsExtract{MyMeasurement1}) + schema = to_json_schema(ItemsExtract{MyMeasurement11}) @test schema["type"] == "object" @test schema["properties"]["items"]["type"] == "array" @test schema["required"] == ["items"] diff --git a/test/llm_google.jl b/test/llm_google.jl new file mode 100644 index 000000000..7aafa2891 --- /dev/null +++ b/test/llm_google.jl @@ -0,0 +1,177 @@ +## using GoogleGenAI # not needed +using PromptingTools: TestEchoGoogleSchema, render, GoogleSchema, ggi_generate_content +using PromptingTools: AIMessage, SystemMessage, AbstractMessage +using PromptingTools: UserMessage, DataMessage + +@testset "render-Google" begin + schema = GoogleSchema() + # Given a schema and a vector of messages with handlebar variables, it should replace the variables with the correct values in the conversation dictionary. + messages = [ + SystemMessage("Act as a helpful AI assistant"), + UserMessage("Hello, my name is {{name}}"), + ] + expected_output = [ + Dict("role" => "user", + "parts" => [ + Dict("text" => "Act as a helpful AI assistant\n\nHello, my name is John"), + ]), + ] + conversation = render(schema, messages; name = "John") + @test conversation == expected_output + # Test with dry_run=true on ai* functions + test_schema = TestEchoGoogleSchema(; text = "a", status = 0) + @test aigenerate(test_schema, + messages; + name = "John", + dry_run = true) == + nothing + @test aigenerate(test_schema, + messages; + name = "John", + dry_run = true, + return_all = true) == + expected_output + + # AI message does NOT replace variables + messages = [ + SystemMessage("Act as a helpful AI assistant"), + AIMessage("Hello, my name is {{name}}"), + ] + expected_output = [ + Dict("role" => "user", + "parts" => [Dict("text" => "Act as a helpful AI assistant")]), + Dict("role" => "model", "parts" => [Dict("text" => "Hello, my name is {{name}}")]), + ] + conversation = render(schema, messages; name = "John") + # Broken: AIMessage does not replace handlebar variables + @test conversation == expected_output + + # Given a schema and a vector of messages with no system messages, it should add a default system prompt to the conversation dictionary. + messages = [ + UserMessage("User message"), + ] + conversation = render(schema, messages) + expected_output = [ + Dict("role" => "user", + "parts" => [Dict("text" => "Act as a helpful AI assistant\n\nUser message")]), + ] + @test conversation == expected_output + + # Given a schema and a vector of messages, it should return a conversation dictionary with the correct roles and contents for each message. + messages = [ + UserMessage("Hello"), + AIMessage("Hi there"), + UserMessage("How are you?"), + AIMessage("I'm doing well, thank you!"), + ] + expected_output = [ + Dict("role" => "user", + "parts" => [Dict("text" => "Act as a helpful AI assistant\n\nHello")]), + Dict("role" => "model", "parts" => [Dict("text" => "Hi there")]), + Dict("role" => "user", "parts" => [Dict("text" => "How are you?")]), + Dict("role" => "model", "parts" => [Dict("text" => "I'm doing well, thank you!")]), + ] + conversation = render(schema, messages) + @test conversation == expected_output + + # Given a schema and a vector of messages with a system message, it should move the system message to the front of the conversation dictionary. + messages = [ + UserMessage("Hello"), + AIMessage("Hi there"), + SystemMessage("This is a system message"), + ] + expected_output = [ + Dict("role" => "user", + "parts" => [Dict("text" => "This is a system message\n\nHello")]), + Dict("role" => "model", "parts" => [Dict("text" => "Hi there")]), + ] + conversation = render(schema, messages) + @test conversation == expected_output + + # Given an empty vector of messages, it should return an empty conversation dictionary just with the system prompt + messages = AbstractMessage[] + expected_output = [ + Dict("role" => "user", + "parts" => [Dict("text" => "Act as a helpful AI assistant")]), + ] + conversation = render(schema, messages) + @test conversation == expected_output + + # Given a schema and a vector of messages with a system message containing handlebar variables not present in kwargs, it keeps the placeholder + messages = [ + SystemMessage("Hello, {{name}}!"), + UserMessage("How are you?"), + ] + expected_output = [ + Dict("role" => "user", + "parts" => [Dict("text" => "Hello, {{name}}!\n\nHow are you?")]), + ] + conversation = render(schema, messages) + # Broken because we do not remove any unused handlebar variables + @test conversation == expected_output + + # Given a schema and a vector of messages with an unknown message type, it should skip the message and continue building the conversation dictionary. + messages = [ + UserMessage("Hello"), + DataMessage(; content = ones(3, 3)), + AIMessage("Hi there"), + ] + expected_output = [ + Dict("role" => "user", + "parts" => [Dict("text" => "Act as a helpful AI assistant\n\nHello")]), + Dict("role" => "model", "parts" => [Dict("text" => "Hi there")]), + ] + conversation = render(schema, messages) + @test conversation == expected_output + + ## Test that if either of System or User message is empty, we don't add double newlines + messages = [ + SystemMessage("Hello, {{name}}!"), + UserMessage(""), + ] + expected_output = [ + Dict("role" => "user", "parts" => [Dict("text" => "Hello, John!")]), + ] + conversation = render(schema, messages; name = "John") + # Broken because we do not remove any unused handlebar variables + @test conversation == expected_output +end + +@testset "aigenerate-Google" begin + # break without the extension + @test_throws ArgumentError aigenerate(PT.GoogleSchema(), "Hello World") + + # corresponds to GoogleGenAI v0.1.0 + # Test the monkey patch + schema = TestEchoGoogleSchema(; text = "Hello!", status = 200) + msg = ggi_generate_content(schema, "", "", "Hello") + @test msg isa TestEchoGoogleSchema + + # Real generation API + schema1 = TestEchoGoogleSchema(; text = "Hello!", status = 200) + msg = aigenerate(schema1, "Hello World") + expected_output = AIMessage(; + content = "Hello!" |> strip, + status = 200, + tokens = (0, 0), + elapsed = msg.elapsed) + @test msg == expected_output + @test schema1.inputs == Dict{String, Any}[Dict("role" => "user", + "parts" => [Dict("text" => "Act as a helpful AI assistant\n\nHello World")])] + @test schema1.model_id == "gemini-pro" # default model + + # Test different input combinations and different prompts + schema2 = TestEchoGoogleSchema(; text = "World!", status = 200) + msg = aigenerate(schema2, UserMessage("Hello {{name}}"), + model = "geminixx", http_kwargs = (; verbose = 3), api_kwargs = (; temperature = 0), + name = "World") + expected_output = AIMessage(; + content = "World!" |> strip, + status = 200, + tokens = (0, 0), + elapsed = msg.elapsed) + @test msg == expected_output + @test schema1.inputs == Dict{String, Any}[Dict("role" => "user", + "parts" => [Dict("text" => "Act as a helpful AI assistant\n\nHello World")])] + @test schema2.model_id == "geminixx" +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index ea159e02b..15503a401 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,8 +2,8 @@ using PromptingTools using OpenAI, HTTP, JSON3 using SparseArrays, LinearAlgebra, Markdown using Test, Pkg -using Aqua const PT = PromptingTools +using Aqua @testset "Code quality (Aqua.jl)" begin # Skipping unbound_args check because we need our `MaybeExtract` type to be unboard @@ -19,6 +19,7 @@ end include("llm_openai.jl") include("llm_ollama_managed.jl") include("llm_ollama.jl") + include("llm_google.jl") include("macros.jl") include("templates.jl") include("serialization.jl") From 9c667c8248fdebd2ce288b6a541ba39882c83fcf Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sat, 17 Feb 2024 13:39:11 +0100 Subject: [PATCH 119/251] Add LCS matching for strings (#76) --- CHANGELOG.md | 1 + docs/make.jl | 2 + .../examples/working_with_google_ai_studio.md | 68 +++++++++++++++++++ src/PromptingTools.jl | 1 + src/llm_google.jl | 31 +++++---- src/user_preferences.jl | 4 +- src/utils.jl | 67 ++++++++++++++++++ test/llm_google.jl | 4 +- test/utils.jl | 21 +++++- 9 files changed, 182 insertions(+), 17 deletions(-) create mode 100644 docs/src/examples/working_with_google_ai_studio.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6131c7b12..0b4ce7296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). +- Added a utility to compare any two string sequences (and other iterators)`length_longest_common_subsequence`. It can be used to fuzzy match strings (eg, detecting context/sources in an AI-generated response or fuzzy matching AI response to some preset categories). See the docstring for more information `?length_longest_common_subsequence`. ### Fixed diff --git a/docs/make.jl b/docs/make.jl index 40a947c55..e5ca9f168 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -25,6 +25,7 @@ makedocs(; repolink = "https://github.com/svilupp/PromptingTools.jl", canonical = "https://svilupp.github.io/PromptingTools.jl", edit_link = "main", + size_threshold = nothing, assets = String[]), pages = [ "Home" => "index.md", @@ -33,6 +34,7 @@ makedocs(; "Various examples" => "examples/readme_examples.md", "Using AITemplates" => "examples/working_with_aitemplates.md", "Local models with Ollama.ai" => "examples/working_with_ollama.md", + "Google AIStudio" => "examples/working_with_google_ai_studio.md", "Custom APIs (Mistral, Llama.cpp)" => "examples/working_with_custom_apis.md", "Building RAG Application" => "examples/building_RAG.md", ], diff --git a/docs/src/examples/working_with_google_ai_studio.md b/docs/src/examples/working_with_google_ai_studio.md new file mode 100644 index 000000000..d6b07e597 --- /dev/null +++ b/docs/src/examples/working_with_google_ai_studio.md @@ -0,0 +1,68 @@ +# Working with Google AI Studio + +This file contains examples of how to work with [Google AI Studio](https://ai.google.dev/). It is known for its Gemini models. + +Get an API key from [here](https://ai.google.dev/). If you see a documentation page ("Available languages and regions for Google AI Studio and Gemini API"), it means that it's not yet available in your region. + +Save the API key in your environment as `GOOGLE_API_KEY`. + +We'll need `GoogleGenAI.jl` package: + +````julia +using Pkg; Pkg.add(url="https://github.com/tylerjthomas9/GoogleGenAI.jl/") +```` + +You can now use the Gemini-1.0-Pro model like any other model in PromptingTools. We **only support `aigenerate`** at the moment. + +Let's import PromptingTools: + +````julia +using PromptingTools +const PT = PromptingTools +```` + +## Text Generation with aigenerate + +You can use the alias "gemini" for the Gemini-1.0-Pro model. + +### Simple message + +````julia +msg = aigenerate("Say hi!"; model = "gemini") +```` + +```` +AIMessage("Hi there! As a helpful AI assistant, I'm here to help you with any questions or tasks you may have. Feel free to ask me anything, and I'll do my best to assist you.") +```` + +You could achieve the same with a string macro (notice the "gemini" at the end to specify which model to use): + +````julia +@ai"Say hi!"gemini +```` + +### Advanced Prompts + +You can provide multi-turn conversations like with any other model: + +````julia +conversation = [ + PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), + PT.UserMessage("I have feelings for my iPhone. What should I do?")] +msg = aigenerate(conversation; model="gemini") +```` + +```` +AIMessage("Young Padawan, you have stumbled into a dangerous path. Attachment leads to suffering, and love can turn to darkness. + +Release your feelings for this inanimate object. + +The Force flows through all living things, not machines. Seek balance in the Force, and your heart will find true connection. + +Remember, the path of the Jedi is to serve others, not to be attached to possessions.") +```` + +### Gotchas + +- Gemini models actually do NOT have a system prompt (for instructions), so we simply concatenate the system and user messages together for consistency with other APIs. +- The reported `tokens` in the `AIMessage` are actually _characters_ (that's how Google AI Studio intends to charge for them) and are a conservative estimate that we produce. It does not matter, because at the time of writing (Feb-24), the usage is free-of-charge. \ No newline at end of file diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 04b70ea3d..8b0e96700 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -27,6 +27,7 @@ const RESERVED_KWARGS = [ ] # export replace_words, split_by_length, call_cost, auth_header # for debugging only +# export length_longest_common_subsequence include("utils.jl") export aigenerate, aiembed, aiclassify, aiextract, aiscan diff --git a/src/llm_google.jl b/src/llm_google.jl index 516c5cfe1..b50e62f34 100644 --- a/src/llm_google.jl +++ b/src/llm_google.jl @@ -82,14 +82,18 @@ end readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), kwargs...) -Generate an AI response based on a given prompt using the OpenAI API. +Generate an AI response based on a given prompt using the Google Gemini API. Get the API key [here](https://ai.google.dev/). + +Note: +- There is no "cost" reported as of February 2024, as all access seems to be free-of-charge. See the details [here](https://ai.google.dev/pricing). +- `tokens` in the returned AIMessage are actually characters, not tokens. We use a _conservative_ estimate as they are not provided by the API yet. # Arguments - `prompt_schema`: An optional object to specify which prompt template should be applied (Default to `PROMPT_SCHEMA = OpenAISchema`) - `prompt`: Can be a string representing the prompt for the AI conversation, a `UserMessage`, a vector of `AbstractMessage` or an `AITemplate` - `verbose`: A boolean indicating whether to print additional information. - `api_key`: A string representing the API key for accessing the OpenAI API. -- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. +- `model`: A string representing the model to use for generating the response. Can be an alias corresponding to a model ID defined in `MODEL_ALIASES`. Defaults to - `return_all::Bool=false`: If `true`, returns the entire conversation history, otherwise returns only the last message (the `AIMessage`). - `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. @@ -112,23 +116,22 @@ See also: `ai_str`, `aai_str`, `aiembed`, `aiclassify`, `aiextract`, `aiscan`, ` Simple hello world to test the API: ```julia -result = aigenerate("Say Hi!") -# [ Info: Tokens: 29 @ Cost: \$0.0 in 1.0 seconds -# AIMessage("Hello! How can I assist you today?") +result = aigenerate("Say Hi!"; model="gemini-pro") +# AIMessage("Hi there! 👋 I'm here to help you with any questions or tasks you may have. Just let me know what you need, and I'll do my best to assist you.") ``` `result` is an `AIMessage` object. Access the generated string via `content` property: ```julia typeof(result) # AIMessage{SubString{String}} propertynames(result) # (:content, :status, :tokens, :elapsed -result.content # "Hello! How can I assist you today?" +result.content # "Hi there! ... ``` ___ -You can use string interpolation: +You can use string interpolation and alias "gemini": ```julia a = 1 -msg=aigenerate("What is `\$a+\$a`?") -msg.content # "The sum of `1+1` is `2`." +msg=aigenerate("What is `\$a+\$a`?"; model="gemini") +msg.content # "1+1 is 2." ``` ___ You can provide the whole conversation or more intricate prompts as a `Vector{AbstractMessage}`: @@ -138,8 +141,8 @@ const PT = PromptingTools conversation = [ PT.SystemMessage("You're master Yoda from Star Wars trying to help the user become a Yedi."), PT.UserMessage("I have feelings for my iPhone. What should I do?")] -msg=aigenerate(conversation) -# AIMessage("Ah, strong feelings you have for your iPhone. A Jedi's path, this is not... ") +msg=aigenerate(conversation; model="gemini") +# AIMessage("Young Padawan, you have stumbled into a dangerous path.... ") ``` """ function aigenerate(prompt_schema::AbstractGoogleSchema, prompt::ALLOWED_PROMPT_TYPE; @@ -170,10 +173,14 @@ function aigenerate(prompt_schema::AbstractGoogleSchema, prompt::ALLOWED_PROMPT_ conv_rendered; http_kwargs, api_kwargs...) + ## Big overestimate + input_token_estimate = length(JSON3.write(conv_rendered)) + output_token_estimate = length(r.text) msg = AIMessage(; content = r.text |> strip, status = 200, - tokens = (0, 0), + ## for google it's CHARACTERS, not tokens + tokens = (input_token_estimate, output_token_estimate), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) diff --git a/src/user_preferences.jl b/src/user_preferences.jl index e26066cf6..733dc0495 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -403,8 +403,8 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", "Local server, eg, powered by [Llama.jl](https://github.com/marcom/Llama.jl). Model is specified when instantiating the server itself."), "gemini-pro" => ModelSpec("gemini-pro", GoogleSchema(), - 0.0, #unknown - 0.0, #unknown + 0.0, #unknown, expected 1.25e-7 + 0.0, #unknown, expected 3.75e-7 "Gemini Pro is a LLM from Google. For more information, see [models](https://ai.google.dev/models/gemini).")) ### Model Registry Structure diff --git a/src/utils.jl b/src/utils.jl index c8cfee68f..683c810f7 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -166,6 +166,73 @@ function split_by_length(text, separators::Vector{String}; max_length) return chunks end +""" + length_longest_common_subsequence(itr1, itr2) + +Compute the length of the longest common subsequence between two sequences (ie, the higher the number, the better the match). + +Source: https://cn.julialang.org/LeetCode.jl/dev/democards/problems/problems/1143.longest-common-subsequence/ + +# Arguments +- `itr1`: The first sequence, eg, a String. +- `itr2`: The second sequence, eg, a String. + +# Returns +The length of the longest common subsequence. + +# Examples +```julia +text1 = "abc-abc----" +text2 = "___ab_c__abc" +longest_common_subsequence(text1, text2) +# Output: 6 (-> "abcabc") +``` + +It can be used to fuzzy match strings and find the similarity between them (Tip: normalize the match) +```julia +commands = ["product recommendation", "emotions", "specific product advice", "checkout advice"] +query = "Which product can you recommend for me?" +let pos = argmax(length_longest_common_subsequence.(Ref(query), commands)) + dist = length_longest_common_subsequence(query, commands[pos]) + norm = dist / min(length(query), length(commands[pos])) + @info "The closest command to the query: \"\$(query)\" is: \"\$(commands[pos])\" (distance: \$(dist), normalized: \$(norm))" +end +``` + +You can also use it to find the closest context for some AI generated summary/story: + +```julia +context = ["The enigmatic stranger vanished as swiftly as a wisp of smoke, leaving behind a trail of unanswered questions.", + "Beneath the shimmering moonlight, the ocean whispered secrets only the stars could hear.", + "The ancient tree stood as a silent guardian, its gnarled branches reaching for the heavens.", + "The melody danced through the air, painting a vibrant tapestry of emotions.", + "Time flowed like a relentless river, carrying away memories and leaving imprints in its wake."] + +story = \"\"\" + Beneath the shimmering moonlight, the ocean whispered secrets only the stars could hear. + + Under the celestial tapestry, the vast ocean whispered its secrets to the indifferent stars. Each ripple, a murmured confidence, each wave, a whispered lament. The glittering celestial bodies listened in silent complicity, their enigmatic gaze reflecting the ocean's unspoken truths. The cosmic dance between the sea and the sky, a symphony of shared secrets, forever echoing in the ethereal expanse. + \"\"\" + +let pos = argmax(length_longest_common_subsequence.(Ref(story), context)) + dist = length_longest_common_subsequence(story, context[pos]) + norm = dist / min(length(story), length(context[pos])) + @info "The closest context to the query: \"\$(first(story,20))...\" is: \"\$(context[pos])\" (distance: \$(dist), normalized: \$(norm))" +end +``` +""" +function length_longest_common_subsequence(itr1, itr2) + m, n = length(itr1) + 1, length(itr2) + 1 + dp = fill(0, m, n) + + for i in 2:m, j in 2:n + dp[i, j] = (itr1[i - 1] == itr2[j - 1]) ? (dp[i - 1, j - 1] + 1) : + max(dp[i - 1, j], dp[i, j - 1]) + end + + return dp[m, n] +end + ### INTERNAL FUNCTIONS - DO NOT USE DIRECTLY # helper to extract handlebar variables (eg, `{{var}}`) from a prompt string function _extract_handlebar_variables(s::AbstractString) diff --git a/test/llm_google.jl b/test/llm_google.jl index 7aafa2891..a238004b0 100644 --- a/test/llm_google.jl +++ b/test/llm_google.jl @@ -153,7 +153,7 @@ end expected_output = AIMessage(; content = "Hello!" |> strip, status = 200, - tokens = (0, 0), + tokens = (83, 6), elapsed = msg.elapsed) @test msg == expected_output @test schema1.inputs == Dict{String, Any}[Dict("role" => "user", @@ -168,7 +168,7 @@ end expected_output = AIMessage(; content = "World!" |> strip, status = 200, - tokens = (0, 0), + tokens = (83, 6), elapsed = msg.elapsed) @test msg == expected_output @test schema1.inputs == Dict{String, Any}[Dict("role" => "user", diff --git a/test/utils.jl b/test/utils.jl index 112748a47..a55310b84 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,4 +1,4 @@ -using PromptingTools: split_by_length, replace_words +using PromptingTools: split_by_length, replace_words, length_longest_common_subsequence using PromptingTools: _extract_handlebar_variables, call_cost, _report_stats using PromptingTools: _string_to_vector, _encode_local_image using PromptingTools: DataMessage, AIMessage @@ -86,6 +86,25 @@ end @test length(separators) == sep_length end +@testset "length_longest_common_subsequence" begin + # Test for equal strings + @test length_longest_common_subsequence("abcde", "abcde") == 5 + # flip the order of the strings -> abcd only + @test length_longest_common_subsequence("abcde", "abced") == 4 + + # Test for empty string + @test length_longest_common_subsequence("", "") == 0 + + # Test for no common subsequence + @test length_longest_common_subsequence("abcde", "xyz") == 0 + + # Test for partial common subsequence + @test length_longest_common_subsequence("abcde", "ace") == 3 + + # Test for common subsequence with repeated characters + @test length_longest_common_subsequence("abc-abc----", "___ab_c__abc") == 6 +end + @testset "extract_handlebar_variables" begin # Extracts handlebar variables enclosed in double curly braces input_string = "Hello {{name}}, how are you?" From 84f105494f38e1f7efa968cb94893871862e40cd Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Sun, 18 Feb 2024 14:46:18 +0100 Subject: [PATCH 120/251] Aiclassify arbitrary choices (#78) --- CHANGELOG.md | 1 + docs/src/frequently_asked_questions.md | 12 +- ext/GoogleGenAIPromptingToolsExt.jl | 2 +- src/Experimental/APITools/APITools.jl | 2 +- src/Experimental/APITools/tavily_api.jl | 2 +- src/code_eval.jl | 2 +- src/code_expressions.jl | 2 +- src/llm_google.jl | 2 +- src/llm_openai.jl | 204 ++++++++++++++++-- src/llm_shared.jl | 13 ++ src/utils.jl | 2 +- templates/classification/InputClassifier.json | 1 + test/Experimental/APITools/tavily_api.jl | 2 +- test/Experimental/AgentTools/code_feedback.jl | 2 +- test/llm_google.jl | 2 +- test/llm_interface.jl | 4 +- test/llm_openai.jl | 102 +++++++++ 17 files changed, 327 insertions(+), 30 deletions(-) create mode 100644 templates/classification/InputClassifier.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b4ce7296..0467d9899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). - Added a utility to compare any two string sequences (and other iterators)`length_longest_common_subsequence`. It can be used to fuzzy match strings (eg, detecting context/sources in an AI-generated response or fuzzy matching AI response to some preset categories). See the docstring for more information `?length_longest_common_subsequence`. +- Rewrite of `aiclassify` to classify into an arbitrary list of categories (including with descriptions). It's a quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick and outputs only 1 token. Currently only `OpenAISchema` is supported. See `?aiclassify` for more information. ### Fixed diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md index d12612c00..3ad8ec9d5 100644 --- a/docs/src/frequently_asked_questions.md +++ b/docs/src/frequently_asked_questions.md @@ -139,4 +139,14 @@ Download new models with `ollama pull ` (eg, `ollama pull openhermes Show currently available models with `ollama list`. -See [Ollama.ai](https://ollama.ai/) for more information. \ No newline at end of file +See [Ollama.ai](https://ollama.ai/) for more information. + +## Changing the Default Model or Schema + +If you tend to use non-default options, it can get tedious to specify `PT.*` every time. + +There are three ways how you can customize your workflows (especially when you use Ollama or other local models): + +1) Import the functions/types you need explicitly at the top (eg, `using PromptingTools: OllamaSchema`) +2) Register your model and its associated schema (`PT.register_model!(; name="123", schema=PT.OllamaSchema())`). You won't have to specify the schema anymore only the model name. See [Working with Ollama](#working-with-ollama) for more information. +3) Override your default model (`PT.MODEL_CHAT`) and schema (`PT.PROMPT_SCHEMA`). It can be done persistently with Preferences, eg, `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaSchema", "MODEL_CHAT"=>"llama2")`. \ No newline at end of file diff --git a/ext/GoogleGenAIPromptingToolsExt.jl b/ext/GoogleGenAIPromptingToolsExt.jl index d5ff80140..12d875441 100644 --- a/ext/GoogleGenAIPromptingToolsExt.jl +++ b/ext/GoogleGenAIPromptingToolsExt.jl @@ -28,4 +28,4 @@ function PromptingTools.ggi_generate_content(prompt_schema::PT.AbstractGoogleSch end end -end # end of module \ No newline at end of file +end # end of module diff --git a/src/Experimental/APITools/APITools.jl b/src/Experimental/APITools/APITools.jl index 4c2d34352..aafe159ab 100644 --- a/src/Experimental/APITools/APITools.jl +++ b/src/Experimental/APITools/APITools.jl @@ -7,4 +7,4 @@ const PT = PromptingTools export create_websearch include("tavily_api.jl") -end # module \ No newline at end of file +end # module diff --git a/src/Experimental/APITools/tavily_api.jl b/src/Experimental/APITools/tavily_api.jl index f827ca4e1..5d2e19bb2 100644 --- a/src/Experimental/APITools/tavily_api.jl +++ b/src/Experimental/APITools/tavily_api.jl @@ -78,4 +78,4 @@ function create_websearch(query::AbstractString; include_images, include_domains, exclude_domains) -end \ No newline at end of file +end diff --git a/src/code_eval.jl b/src/code_eval.jl index 71355bc2a..df78cc587 100644 --- a/src/code_eval.jl +++ b/src/code_eval.jl @@ -378,4 +378,4 @@ function eval!(cb::AbstractCodeBlock, expr::Expr; end end return cb -end \ No newline at end of file +end diff --git a/src/code_expressions.jl b/src/code_expressions.jl index 9bbace0ab..5f3cf8284 100644 --- a/src/code_expressions.jl +++ b/src/code_expressions.jl @@ -101,4 +101,4 @@ function extract_module_name(expr) end end nothing -end \ No newline at end of file +end diff --git a/src/llm_google.jl b/src/llm_google.jl index b50e62f34..078e1ab2d 100644 --- a/src/llm_google.jl +++ b/src/llm_google.jl @@ -197,4 +197,4 @@ function aigenerate(prompt_schema::AbstractGoogleSchema, prompt::ALLOWED_PROMPT_ kwargs...) return output -end \ No newline at end of file +end diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 8fcdce5e5..5fa641227 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -471,29 +471,194 @@ function aiembed(prompt_schema::AbstractOpenAISchema, return msg end +"Token IDs for GPT3.5 and GPT4 from https://platform.openai.com/tokenizer" +const OPENAI_TOKEN_IDS = Dict("true" => 837, + "false" => 905, + "unknown" => 9987, + "other" => 1023, + "1" => 16, + "2" => 17, + "3" => 18, + "4" => 19, + "5" => 20, + "6" => 21, + "7" => 22, + "8" => 23, + "9" => 24, + "10" => 605, + "11" => 806, + "12" => 717, + "13" => 1032, + "14" => 975, + "15" => 868, + "16" => 845, + "17" => 1114, + "18" => 972, + "19" => 777, + "20" => 508) + +""" + encode_choices(schema::OpenAISchema, choices::AbstractVector{<:AbstractString}; kwargs...) + + encode_choices(schema::OpenAISchema, choices::AbstractVector{T}; + kwargs...) where {T <: Tuple{<:AbstractString, <:AbstractString}} + +Encode the choices into an enumerated list that can be interpolated into the prompt and creates the corresponding logit biases (to choose only from the selected tokens). + +Optionally, can be a vector tuples, where the first element is the choice and the second is the description. + +# Arguments +- `schema::OpenAISchema`: The OpenAISchema object. +- `choices::AbstractVector{<:Union{AbstractString,Tuple{<:AbstractString, <:AbstractString}}}`: The choices to be encoded, represented as a vector of the choices directly, or tuples where each tuple contains a choice and its description. +- `kwargs...`: Additional keyword arguments. + +# Returns +- `choices_prompt::AbstractString`: The encoded choices as a single string, separated by newlines. +- `logit_bias::Dict`: The logit bias dictionary, where the keys are the token IDs and the values are the bias values. +- `decode_ids::AbstractVector{<:AbstractString}`: The decoded IDs of the choices. + +# Examples +```julia +choices_prompt, logit_bias, _ = PT.encode_choices(PT.OpenAISchema(), ["true", "false"]) +choices_prompt # Output: "true for \"true\"\nfalse for \"false\" +logit_bias # Output: Dict(837 => 100, 905 => 100) + +choices_prompt, logit_bias, _ = PT.encode_choices(PT.OpenAISchema(), ["animal", "plant"]) +choices_prompt # Output: "1. \"animal\"\n2. \"plant\"" +logit_bias # Output: Dict(16 => 100, 17 => 100) +``` + +Or choices with descriptions: +```julia +choices_prompt, logit_bias, _ = PT.encode_choices(PT.OpenAISchema(), [("A", "any animal or creature"), ("P", "for any plant or tree"), ("O", "for everything else")]) +choices_prompt # Output: "1. \"A\" for any animal or creature\n2. \"P\" for any plant or tree\n3. \"O\" for everything else" +logit_bias # Output: Dict(16 => 100, 17 => 100, 18 => 100) +``` +""" +function encode_choices(schema::OpenAISchema, + choices::AbstractVector{<:AbstractString}; + kwargs...) + global OPENAI_TOKEN_IDS + ## if all choices are in the dictionary, use the dictionary + if all(x -> haskey(OPENAI_TOKEN_IDS, x), choices) + choices_prompt = ["$c for \"$c\"" for c in choices] + logit_bias = Dict(OPENAI_TOKEN_IDS[c] => 100 for c in choices) + elseif length(choices) <= 20 + ## encode choices to IDs 1..20 + choices_prompt = ["$(i). \"$c\"" for (i, c) in enumerate(choices)] + logit_bias = Dict(OPENAI_TOKEN_IDS[string(i)] => 100 for i in 1:length(choices)) + else + throw(ArgumentError("The number of choices must be less than or equal to 20.")) + end + + return join(choices_prompt, "\n"), logit_bias, choices +end +function encode_choices(schema::OpenAISchema, + choices::AbstractVector{T}; + kwargs...) where {T <: Tuple{<:AbstractString, <:AbstractString}} + global OPENAI_TOKEN_IDS + ## if all choices are in the dictionary, use the dictionary + if all(x -> haskey(OPENAI_TOKEN_IDS, first(x)), choices) + choices_prompt = ["$c for \"$desc\"" for (c, desc) in choices] + logit_bias = Dict(OPENAI_TOKEN_IDS[c] => 100 for (c, desc) in choices) + elseif length(choices) <= 20 + ## encode choices to IDs 1..20 + choices_prompt = ["$(i). \"$c\" for $desc" for (i, (c, desc)) in enumerate(choices)] + logit_bias = Dict(OPENAI_TOKEN_IDS[string(i)] => 100 for i in 1:length(choices)) + else + throw(ArgumentError("The number of choices must be less than or equal to 20.")) + end + + return join(choices_prompt, "\n"), logit_bias, first.(choices) +end + +# For testing +function encode_choices(schema::TestEchoOpenAISchema, choices; kwargs...) + return encode_choices(OpenAISchema(), choices; kwargs...) +end +# For testing +function decode_choices(schema::TestEchoOpenAISchema, + choices, + conv::Union{AbstractVector, AIMessage}; + kwargs...) + return decode_choices(OpenAISchema(), choices, conv; kwargs...) +end + +function decode_choices(schema::OpenAISchema, choices, conv::AbstractVector; kwargs...) + if length(conv) > 0 && last(conv) isa AIMessage + conv[end] = decode_choices(schema, choices, last(conv)) + end + return conv +end + +""" + decode_choices(schema::OpenAISchema, + choices::AbstractVector{<:AbstractString}, + msg::AIMessage; kwargs...) + +Decodes the underlying AIMessage against the original choices to lookup what the category name was. + +If it fails, it will return `msg.content == nothing` +""" +function decode_choices(schema::OpenAISchema, + choices::AbstractVector{<:AbstractString}, + msg::AIMessage; kwargs...) + global OPENAI_TOKEN_IDS + parsed_digit = tryparse(Int, strip(msg.content)) + if !isnothing(parsed_digit) && haskey(OPENAI_TOKEN_IDS, strip(msg.content)) + ## It's encoded + content = choices[parsed_digit] + elseif haskey(OPENAI_TOKEN_IDS, strip(msg.content)) + ## if it's NOT a digit, but direct mapping (eg, true/false), no changes! + content = strip(msg.content) + else + ## failed decoding + content = nothing + end + return AIMessage(; content, msg.status, msg.tokens, msg.elapsed) +end + """ aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; - api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), - max_tokens = 1, temperature = 0), - kwargs...) + choices::AbstractVector{T} = ["true", "false", "unknown"], + api_kwargs::NamedTuple = NamedTuple(), + kwargs...) where {T <: Union{AbstractString, Tuple{<:AbstractString, <:AbstractString}}} + +Classifies the given prompt/statement into an arbitrary list of `choices`, which must be only the choices (vector of strings) or choices and descriptions are provided (vector of tuples, ie, `("choice","description")`). -Classifies the given prompt/statement as true/false/unknown. +It's quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick and outputs only 1 token. +classify into an arbitrary list of categories (including with descriptions). It's quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick, so it outputs only 1 token. -Note: this is a very simple classifier, it is not meant to be used in production. Credit goes to [AAAzzam](https://twitter.com/AAAzzam/status/1669753721574633473). +!!! Note: The prompt/AITemplate must have a placeholder `choices` (ie, `{{choices}}`) that will be replaced with the encoded choices -It uses Logit bias trick and limits the output to 1 token to force the model to output only true/false/unknown. +Choices are rewritten into an enumerated list and mapped to a few known OpenAI tokens (maximum of 20 choices supported). Mapping of token IDs for GPT3.5/4 are saved in variable `OPENAI_TOKEN_IDS`. -Output tokens used (via `api_kwargs`): -- 837: ' true' -- 905: ' false' -- 9987: ' unknown' +It uses Logit bias trick and limits the output to 1 token to force the model to output only true/false/unknown. Credit for the idea goes to [AAAzzam](https://twitter.com/AAAzzam/status/1669753721574633473). # Arguments - `prompt_schema::AbstractOpenAISchema`: The schema for the prompt. -- `prompt`: The prompt/statement to classify if it's a `String`. If it's a `Symbol`, it is expanded as a template via `render(schema,template)`. +- `prompt`: The prompt/statement to classify if it's a `String`. If it's a `Symbol`, it is expanded as a template via `render(schema,template)`. Eg, templates `:JudgeIsItTrue` or `:InputClassifier` +- `choices::AbstractVector{T}`: The choices to be classified into. It can be a vector of strings or a vector of tuples, where the first element is the choice and the second is the description. # Example +Given a user input, pick one of the two provided categories: +```julia +choices = ["animal", "plant"] +input = "Palm tree" +aiclassify(:InputClassifier; choices, input) +``` + +Choices with descriptions provided as tuples: +```julia +choices = [("A", "any animal or creature"), ("P", "for any plant or tree"), ("O", "for everything else")] +input = "spider" +input = "daphodil" +input = "castle" +aiclassify(:InputClassifier; choices, input) +``` + +You can still use a simple true/false classification: ```julia aiclassify("Is two plus two four?") # true aiclassify("Is two plus three a vegetable on Mars?") # false @@ -520,15 +685,20 @@ aiclassify(:JudgeIsItTrue; """ function aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; - api_kwargs::NamedTuple = (logit_bias = Dict(837 => 100, 905 => 100, 9987 => 100), - max_tokens = 1, temperature = 0), - kwargs...) - ## - msg = aigenerate(prompt_schema, + choices::AbstractVector{T} = ["true", "false", "unknown"], + api_kwargs::NamedTuple = NamedTuple(), + kwargs...) where {T <: Union{AbstractString, Tuple{<:AbstractString, <:AbstractString}}} + ## Encode the choices and the corresponding prompt + ## TODO: maybe check the model provided as well? + choices_prompt, logit_bias, decode_ids = encode_choices(prompt_schema, choices) + ## We want only 1 token + api_kwargs = merge(api_kwargs, (; logit_bias, max_tokens = 1, temperature = 0)) + msg_or_conv = aigenerate(prompt_schema, prompt; + choices = choices_prompt, api_kwargs, kwargs...) - return msg + return decode_choices(prompt_schema, decode_ids, msg_or_conv) end """ diff --git a/src/llm_shared.jl b/src/llm_shared.jl index d8cc86a02..fd6401187 100644 --- a/src/llm_shared.jl +++ b/src/llm_shared.jl @@ -101,3 +101,16 @@ function finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, return msg end end + +## Helpers for aiclassify -> they encode the choice list to create the prompt and then extract the original choice category +function encode_choices(schema::AbstractPromptSchema, + choices; + kwargs...) + throw(ArgumentError("Function `encode_choices` is not implemented for the provided schema ($schema) and $(choices).")) +end +function decode_choices(schema::AbstractPromptSchema, choices, conv; kwargs...) + throw(ArgumentError("Function `decode_choices` is not implemented for the provided schema ($schema) and $(choices).")) +end +function decode_choices(schema::AbstractPromptSchema, choices, conv::Nothing; kwargs...) + nothing +end diff --git a/src/utils.jl b/src/utils.jl index 683c810f7..3170056c8 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -449,4 +449,4 @@ function auth_header(api_key::Union{Nothing, AbstractString}; ] !isnothing(api_key) && pushfirst!(headers, "Authorization" => "Bearer $api_key") return headers -end \ No newline at end of file +end diff --git a/templates/classification/InputClassifier.json b/templates/classification/InputClassifier.json new file mode 100644 index 000000000..5bd4a69ad --- /dev/null +++ b/templates/classification/InputClassifier.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"For classification tasks and routing of queries with aiclassify. It expects a list of choices to be provided (starting with their IDs), and will pick one that best describes the user input. Placeholders: `input`, `choices`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"You are a world-class classification specialist. \n\nYour task is to select the most appropriate label from the given choices for the given user input.\n\n**Available Choices:**\n---\n{{choices}}\n---\n\n**Instructions:**\n- You must respond in one word. \n- You must respond only with the label ID (e.g., \"1\", \"2\", ...) that best fits the input.\n","variables":["choices"],"_type":"systemmessage"},{"content":"User Input: {{input}}\n\nLabel:\n","variables":["input"],"_type":"usermessage"}] \ No newline at end of file diff --git a/test/Experimental/APITools/tavily_api.jl b/test/Experimental/APITools/tavily_api.jl index e74519f11..e0c40d281 100644 --- a/test/Experimental/APITools/tavily_api.jl +++ b/test/Experimental/APITools/tavily_api.jl @@ -1 +1 @@ -# TODO: hard to test the API itself? \ No newline at end of file +# TODO: hard to test the API itself? diff --git a/test/Experimental/AgentTools/code_feedback.jl b/test/Experimental/AgentTools/code_feedback.jl index 36a94c949..bec9964a4 100644 --- a/test/Experimental/AgentTools/code_feedback.jl +++ b/test/Experimental/AgentTools/code_feedback.jl @@ -242,4 +242,4 @@ end """ @test extract_test_counts(test_summary4) == Dict("pass" => 2, "broken" => 1, "fail" => 1, "error" => 1, "total" => 5) -end \ No newline at end of file +end diff --git a/test/llm_google.jl b/test/llm_google.jl index a238004b0..518803462 100644 --- a/test/llm_google.jl +++ b/test/llm_google.jl @@ -174,4 +174,4 @@ end @test schema1.inputs == Dict{String, Any}[Dict("role" => "user", "parts" => [Dict("text" => "Act as a helpful AI assistant\n\nHello World")])] @test schema2.model_id == "geminixx" -end \ No newline at end of file +end diff --git a/test/llm_interface.jl b/test/llm_interface.jl index 35daa927c..8e010bc11 100644 --- a/test/llm_interface.jl +++ b/test/llm_interface.jl @@ -20,9 +20,9 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage @test msg == expected_output ### AIClassify - msg = aiclassify("Hello World"; model = "xyz") + msg = aiclassify("Hello World"; choices = ["true", "false", "unknown"], model = "xyz") expected_output = AIMessage(; - content = "Hello!" |> strip, + content = nothing, status = 200, tokens = (2, 1), elapsed = msg.elapsed) diff --git a/test/llm_openai.jl b/test/llm_openai.jl index a7ee5e0b4..ba59c4097 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -3,6 +3,7 @@ using PromptingTools: AIMessage, SystemMessage, AbstractMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage using PromptingTools: CustomProvider, CustomOpenAISchema, MistralOpenAISchema, MODEL_EMBEDDING +using PromptingTools: encode_choices, decode_choices @testset "render-OpenAI" begin schema = OpenAISchema() @@ -312,3 +313,104 @@ end @test schema2.inputs == ["Hello World", "Hello back"] @test schema2.model_id == "gpt-4" # not possible - just an example end + +@testset "encode_choices" begin + # Test encoding simple string choices + choices_prompt, logit_bias, ids = encode_choices(OpenAISchema(), ["true", "false"]) + # Checks if the encoded choices format and logit_bias are correct + @test choices_prompt == "true for \"true\"\nfalse for \"false\"" + @test logit_bias == Dict(837 => 100, 905 => 100) + @test ids == ["true", "false"] + + # Test encoding more than two choices + choices_prompt, logit_bias, ids = encode_choices(OpenAISchema(), ["animal", "plant"]) + # Checks the format for multiple choices and correct logit_bias mapping + @test choices_prompt == "1. \"animal\"\n2. \"plant\"" + @test logit_bias == Dict(16 => 100, 17 => 100) + @test ids == ["animal", "plant"] + + # with descriptions + choices_prompt, logit_bias, ids = encode_choices(OpenAISchema(), + [ + ("A", "any animal or creature"), + ("P", "for any plant or tree"), + ("O", "for everything else"), + ]) + expected_prompt = "1. \"A\" for any animal or creature\n2. \"P\" for for any plant or tree\n3. \"O\" for for everything else" + expected_logit_bias = Dict(16 => 100, 17 => 100, 18 => 100) + @test choices_prompt == expected_prompt + @test logit_bias == expected_logit_bias + @test ids == ["A", "P", "O"] + + choices_prompt, logit_bias, ids = encode_choices(OpenAISchema(), + [ + ("true", "If the statement is true"), + ("false", "If the statement is false"), + ]) + expected_prompt = "true for \"If the statement is true\"\nfalse for \"If the statement is false\"" + expected_logit_bias = Dict(837 => 100, 905 => 100) + @test choices_prompt == expected_prompt + @test logit_bias == expected_logit_bias + @test ids == ["true", "false"] + + # Test encoding with an invalid number of choices + @test_throws ArgumentError encode_choices(OpenAISchema(), string.(collect(1:100))) + @test_throws ArgumentError encode_choices(OpenAISchema(), [("$i", "$i") for i in 1:50]) + + @test_throws ArgumentError encode_choices(PT.OllamaSchema(), ["true", "false"]) +end + +@testset "decode_choices" begin + # Test decoding a choice based on its ID + msg = AIMessage("1") + decoded_msg = decode_choices(OpenAISchema(), ["true", "false"], msg) + @test decoded_msg.content == "true" + + # Test decoding with a direct mapping (e.g., true/false) + msg = AIMessage("false") + decoded_msg = decode_choices(OpenAISchema(), ["true", "false"], msg) + @test decoded_msg.content == "false" + + # Test decoding failure (invalid content) + msg = AIMessage("invalid") + decoded_msg = decode_choices(OpenAISchema(), ["true", "false"], msg) + @test isnothing(decoded_msg.content) + + # Decode from conversation + conv = [AIMessage("1")] + decoded_conv = decode_choices(OpenAISchema(), ["true", "false"], conv) + @test decoded_conv[end].content == "true" + + # Nothing (when dry_run=true) + @test isnothing(decode_choices(OpenAISchema(), ["true", "false"], nothing)) + + # unimplemented + @test_throws ArgumentError decode_choices(PT.OllamaSchema(), + ["true", "false"], + AIMessage("invalid")) +end + +@testset "aiclassify-OpenAI" begin + # corresponds to OpenAI API v1 + response = Dict(:choices => [Dict(:message => Dict(:content => "1"))], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + + # Real generation API + schema1 = TestEchoOpenAISchema(; response, status = 200) + choices = [ + ("A", "any animal or creature"), + ("P", "for any plant or tree"), + ("O", "for everything else"), + ] + msg = aiclassify(schema1, :InputClassifier; input = "pelican", choices) + expected_output = AIMessage(; + content = "A", + status = 200, + tokens = (2, 1), + elapsed = msg.elapsed) + @test msg == expected_output + @test schema1.inputs == + Dict{String, Any}[Dict("role" => "system", + "content" => "You are a world-class classification specialist. \n\nYour task is to select the most appropriate label from the given choices for the given user input.\n\n**Available Choices:**\n---\n1. \"A\" for any animal or creature\n2. \"P\" for for any plant or tree\n3. \"O\" for for everything else\n---\n\n**Instructions:**\n- You must respond in one word. \n- You must respond only with the label ID (e.g., \"1\", \"2\", ...) that best fits the input.\n"), + Dict("role" => "user", "content" => "User Input: pelican\n\nLabel:\n")] +end From 463a830a518c134b296f2b6aa031a948ed585951 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 22 Feb 2024 21:06:24 +0000 Subject: [PATCH 121/251] Multiple competions (`n`) (#79) --- CHANGELOG.md | 9 +- docs/src/frequently_asked_questions.md | 184 ++++++++++++++- src/Experimental/AgentTools/lazy_types.jl | 2 + src/Experimental/RAGTools/types.jl | 13 ++ src/llm_interface.jl | 13 ++ src/llm_ollama.jl | 18 +- src/llm_ollama_managed.jl | 8 +- src/llm_openai.jl | 223 +++++++++++++++--- src/llm_shared.jl | 13 +- src/messages.jl | 50 ++++- src/precompilation.jl | 5 +- src/user_preferences.jl | 5 +- src/utils.jl | 61 +++-- test/Experimental/RAGTools/evaluation.jl | 36 +-- test/Experimental/RAGTools/generation.jl | 19 +- test/Experimental/RAGTools/preparation.jl | 13 +- test/llm_interface.jl | 29 ++- test/llm_openai.jl | 261 +++++++++++++++++++++- test/llm_shared.jl | 28 +++ test/messages.jl | 4 +- test/utils.jl | 13 +- 21 files changed, 897 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0467d9899..cab889975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). - Added a utility to compare any two string sequences (and other iterators)`length_longest_common_subsequence`. It can be used to fuzzy match strings (eg, detecting context/sources in an AI-generated response or fuzzy matching AI response to some preset categories). See the docstring for more information `?length_longest_common_subsequence`. -- Rewrite of `aiclassify` to classify into an arbitrary list of categories (including with descriptions). It's a quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick and outputs only 1 token. Currently only `OpenAISchema` is supported. See `?aiclassify` for more information. +- Rewrite of `aiclassify` to classify into an arbitrary list of categories (including with descriptions). It's a quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick and outputs only 1 token. Currently, only `OpenAISchema` is supported. See `?aiclassify` for more information. +- Initial support for multiple completions in one request for OpenAI-compatible API servers. Set via API kwarg `n=5` and it will request 5 completions in one request, saving the network communication time and paying the prompt tokens only once. It's useful for majority voting, diversity, or challenging agentic workflows. +- Added new fields to `AIMessage` and `DataMessage` types to simplify tracking in complex applications. Added fields: + - `cost` - the cost of the query (summary per call, so count only once if you requested multiple completions in one call) + - `log_prob` - summary log probability of the generated sequence, set API kwarg `logprobs=true` to receive it + - `run_id` - ID of the AI API call + - `sample_id` - ID of the sample in the batch if you requested multiple completions, otherwise `sample_id==nothing` (they will have the same `run_id`) + - `finish_reason` - the reason why the AI stopped generating the sequence (eg, "stop", "length") to provide more visibility for the user ### Fixed diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md index 3ad8ec9d5..228d6a4c2 100644 --- a/docs/src/frequently_asked_questions.md +++ b/docs/src/frequently_asked_questions.md @@ -39,6 +39,21 @@ Resources: Pro tip: Always set the spending limits! +## Getting an error "ArgumentError: api_key cannot be empty" despite having set `OPENAI_API_KEY`? + +Quick fix: just provide kwarg `api_key` with your key to the `aigenerate` function (and other `ai*` functions). + +This error is thrown when the OpenAI API key is not available in 1) local preferences or 2) environment variables (`ENV["OPENAI_API_KEY"]`). + +First, check if you can access the key by running `ENV["OPENAI_API_KEY"]` in the Julia REPL. If it returns `nothing`, the key is not set. + +If the key is set, but you still get the error, there was a rare bug in earlier versions where if you first precompiled PromptingTools without the API key, it would remember it and "compile away" the `get(ENV,...)` function call. If you're experiencing this bug on the latest version of PromptingTools, please open an issue on GitHub. + +The solution is to force a new precompilation, so you can do any of the below: +1) Force precompilation (run `Pkg.precompile()` in the Julia REPL) +2) Update the PromptingTools package (runs precompilation automatically) +3) Delete your compiled cache in `.julia` DEPOT (usually `.julia/compiled/v1.10/PromptingTools`). You can do it manually in the file explorer or via Julia REPL: `rm("~/.julia/compiled/v1.10/PromptingTools", recursive=true, force=true)` + ## Setting OpenAI Spending Limits OpenAI allows you to set spending limits directly on your account dashboard to prevent unexpected costs. @@ -149,4 +164,171 @@ There are three ways how you can customize your workflows (especially when you u 1) Import the functions/types you need explicitly at the top (eg, `using PromptingTools: OllamaSchema`) 2) Register your model and its associated schema (`PT.register_model!(; name="123", schema=PT.OllamaSchema())`). You won't have to specify the schema anymore only the model name. See [Working with Ollama](#working-with-ollama) for more information. -3) Override your default model (`PT.MODEL_CHAT`) and schema (`PT.PROMPT_SCHEMA`). It can be done persistently with Preferences, eg, `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaSchema", "MODEL_CHAT"=>"llama2")`. \ No newline at end of file +3) Override your default model (`PT.MODEL_CHAT`) and schema (`PT.PROMPT_SCHEMA`). It can be done persistently with Preferences, eg, `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaSchema", "MODEL_CHAT"=>"llama2")`. + +## How to have a Multi-turn Conversations? + +Let's say you would like to respond back to a model's response. How to do it? + +1) With `ai""` macro +The simplest way if you used `ai""` macro, is to send a reply with the `ai!""` macro. It will use the last response as the conversation. +```julia +ai"Hi! I'm John" + +ai!"What's my name?" +# Return: "Your name is John." +``` + +2) With `aigenerate` function +You can use the `conversation` keyword argument to pass the previous conversation (in all `ai*` functions). It will prepend the past `conversation` before sending the new request to the model. + +To get the conversation, set `return_all=true` and store the whole conversation thread (not just the last message) in a variable. Then, use it as a keyword argument in the next call. + +```julia +conversation = aigenerate("Hi! I'm John"; return_all=true) +@info last(conversation) # display the response + +# follow-up (notice that we provide past messages as conversation kwarg +conversation = aigenerate("What's my name?"; return_all=true, conversation) + +## [ Info: Tokens: 50 @ Cost: $0.0 in 1.0 seconds +## 5-element Vector{PromptingTools.AbstractMessage}: +## PromptingTools.SystemMessage("Act as a helpful AI assistant") +## PromptingTools.UserMessage("Hi! I'm John") +## AIMessage("Hello John! How can I assist you today?") +## PromptingTools.UserMessage("What's my name?") +## AIMessage("Your name is John.") +``` +Notice that the last message is the response to the second request, but with `return_all=true` we can see the whole conversation from the beginning. + +## Explain What Happens Under the Hood + +4 Key Concepts/Objects: +- Schemas -> object of type `AbstractPromptSchema` that determines which methods are called and, hence, what providers/APIs are used +- Prompts -> the information you want to convey to the AI model +- Messages -> the basic unit of communication between the user and the AI model (eg, `UserMessage` vs `AIMessage`) +- Prompt Templates -> re-usable "prompts" with placeholders that you can replace with your inputs at the time of making the request + +When you call `aigenerate`, roughly the following happens: `render` -> `UserMessage`(s) -> `render` -> `OpenAI.create_chat` -> ... -> `AIMessage`. + +We'll deep dive into an example in the end. + +### Schemas + +For your "message" to reach an AI model, it needs to be formatted and sent to the right place. + +We leverage the multiple dispatch around the "schemas" to pick the right logic. +All schemas are subtypes of `AbstractPromptSchema` and there are many subtypes, eg, `OpenAISchema <: AbstractOpenAISchema <:AbstractPromptSchema`. + +For example, if you provide `schema = OpenAISchema()`, the system knows that: +- it will have to format any user inputs to OpenAI's "message specification" (a vector of dictionaries, see their API documentation). Function `render(OpenAISchema(),...)` will take care of the rendering. +- it will have to send the message to OpenAI's API. We will use the amazing `OpenAI.jl` package to handle the communication. + +### Prompts + +Prompt is loosely the information you want to convey to the AI model. It can be a question, a statement, or a command. It can have instructions or some context, eg, previous conversation. + +You need to remember that Large Language Models (LLMs) are **stateless**. They don't remember the previous conversation/request, so you need to provide the whole history/context every time (similar to how REST APIs work). + +Prompts that we send to the LLMs are effectively a sequence of messages (`<:AbstractMessage`). + +### Messages + +Messages are the basic unit of communication between the user and the AI model. + +There are 5 main types of messages (`<:AbstractMessage`): + +- `SystemMessage` - this contains information about the "system", eg, how it should behave, format its output, etc. (eg, `You're a world-class Julia programmer. You write brief and concise code.) +- `UserMessage` - the information "from the user", ie, your question/statement/task +- `UserMessageWithImages` - the same as `UserMessage`, but with images (URLs or Base64-encoded images) +- `AIMessage` - the response from the AI model, when the "output" is text +- `DataMessage` - the response from the AI model, when the "output" is data, eg, embeddings with `aiembed` or user-defined structs with `aiextract` + +### Prompt Templates + +We want to have re-usable "prompts", so we provide you with a system to retrieve pre-defined prompts with placeholders (eg, `{{name}}`) that you can replace with your inputs at the time of making the request. + +"AI Templates" as we call them (`AITemplate`) are usually a vector of `SystemMessage` and a `UserMessage` with specific purpose/task. + +For example, the template `:AssistantAsk` is defined loosely as: + +```julia + template = [SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer."), + UserMessage("# Question\n\n{{ask}}")] +``` + +Notice that we have a placeholder `ask` (`{{ask}}`) that you can replace with your question without having to re-write the generic system instructions. + +When you provide a Symbol (eg, `:AssistantAsk`) to ai* functions, thanks to the multiple dispatch, it recognizes that it's an `AITemplate(:AssistantAsk)` and looks it up. + +You can discover all available templates with `aitemplates("some keyword")` or just see the details of some template `aitemplates(:AssistantAsk)`. + +### Walkthrough Example + +```julia +using PromptingTools +const PT = PromptingTools + +# Let's say this is our ask +msg = aigenerate(:AssistantAsk; ask="What is the capital of France?") + +# it is effectively the same as: +msg = aigenerate(PT.OpenAISchema(), PT.AITemplate(:AssistantAsk); ask="What is the capital of France?", model="gpt3t") +``` + +There is no `model` provided, so we use the default `PT.MODEL_CHAT` (effectively GPT3.5-Turbo). Then we look it up in `PT.MDOEL_REGISTRY` and use the associated schema for it (`OpenAISchema` in this case). + +The next step is to render the template, replace the placeholders and render it for the OpenAI model. + +```julia +# Let's remember out schema +schema = PT.OpenAISchema() +ask = "What is the capital of France?" +``` + +First, we obtain the template (no placeholder replacement yet) and "expand it" +```julia +template_rendered = PT.render(schema, AITemplate(:AssistantAsk); ask) +``` + +```plaintext +2-element Vector{PromptingTools.AbstractChatMessage}: + PromptingTools.SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + PromptingTools.UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) +``` + +Second, we replace the placeholders +```julia +rendered_for_api = PT.render(schema, template_rendered; ask) +``` + +```plaintext +2-element Vector{Dict{String, Any}}: + Dict("role" => "system", "content" => "You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + Dict("role" => "user", "content" => "# Question\n\nWhat is the capital of France?") +``` + +Notice that the placeholders are only replaced in the second step. The final output here is a vector of messages with "role" and "content" keys, which is the format required by the OpenAI API. + +As a side note, under the hood, the second step is done in two steps: + +- replace the placeholders `messages_rendered = PT.render(PT.NoSchema(), template_rendered; ask)` -> returns a vector of Messages! +- then, we convert the messages to the format required by the provider/schema `PT.render(schema, messages_rendered)` -> returns the OpenAI formatted messages + + +Next, we send the above `rendered_for_api` to the OpenAI API and get the response back. + +```julia +using OpenAI +OpenAI.create_chat(api_key, model, rendered_for_api) +``` + +The last step is to take the JSON response from the API and convert it to the `AIMessage` object. + +```julia +# simplification for educational purposes +msg = AIMessage(; content = r.response[:choices][1][:message][:content]) +``` +In practice, there are more fields we extract, so we define a utility for it: `PT.response_to_message`. Especially, since with parameter `n`, you can request multiple AI responses at once, so we want to re-use our response processing logic. + +That's it! I hope you've learned something new about how PromptingTools.jl works under the hood. \ No newline at end of file diff --git a/src/Experimental/AgentTools/lazy_types.jl b/src/Experimental/AgentTools/lazy_types.jl index 2984ae1ac..6201080e4 100644 --- a/src/Experimental/AgentTools/lazy_types.jl +++ b/src/Experimental/AgentTools/lazy_types.jl @@ -62,6 +62,8 @@ This can be used to "reply" to previous message / continue the stored conversati success::Union{Nothing, Bool} = nothing error::Union{Nothing, Exception} = nothing end +## main sample +## samples function AICall(func::F, args...; kwargs...) where {F <: Function} @assert length(args)<=2 "AICall takes at most 2 positional arguments (provided: $(length(args)))" diff --git a/src/Experimental/RAGTools/types.jl b/src/Experimental/RAGTools/types.jl index dab4e78ba..863a25abb 100644 --- a/src/Experimental/RAGTools/types.jl +++ b/src/Experimental/RAGTools/types.jl @@ -8,6 +8,19 @@ abstract type AbstractChunkIndex <: AbstractDocumentIndex end # More advanced index would be: HybridChunkIndex # Stores document chunks and their embeddings +""" + ChunkIndex + +Main struct for storing document chunks and their embeddings. It also stores tags and sources for each chunk. + +# Fields +- `id::Symbol`: unique identifier of each index (to ensure we're using the right index with `CandidateChunks`) +- `chunks::Vector{<:AbstractString}`: underlying document chunks / snippets +- `embeddings::Union{Nothing, Matrix{<:Real}}`: for semantic search +- `tags::Union{Nothing, AbstractMatrix{<:Bool}}`: for exact search, filtering, etc. This is often a sparse matrix indicating which chunks have the given `tag` (see `tag_vocab` for the position lookup) +- `tags_vocab::Union{Nothing, Vector{<:AbstractString}}`: vocabulary for the `tags` matrix (each column in `tags` is one item in `tags_vocab` and rows are the chunks) +- `sources::Vector{<:AbstractString}`: sources of the chunks +""" @kwdef struct ChunkIndex{ T1 <: AbstractString, T2 <: Union{Nothing, Matrix{<:Real}}, diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 04f24a56c..639c188b4 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -250,3 +250,16 @@ function aiscan(prompt; model = MODEL_CHAT, kwargs...) schema = get(MODEL_REGISTRY, model, (; schema = PROMPT_SCHEMA)).schema aiscan(schema, prompt; model, kwargs...) end + +"Utility to facilitate unwrapping of HTTP response to a message type `MSG` provided. Designed to handle multi-sample completions." +function response_to_message(schema::AbstractPromptSchema, + MSG::Type{T}, + choice, + resp; + return_type = nothing, + model_id::AbstractString = "", + time::Float64 = 0.0, + run_id::Integer = rand(Int16), + sample_id::Union{Nothing, Integer} = nothing) where {T} + throw(ArgumentError("Response unwrapping not implemented for $(typeof(schema)) and $MSG")) +end diff --git a/src/llm_ollama.jl b/src/llm_ollama.jl index f03c0c2ff..ae9b95a86 100644 --- a/src/llm_ollama.jl +++ b/src/llm_ollama.jl @@ -2,6 +2,8 @@ # - llm_olama.jl works by providing messages format to /api/chat # - llm_managed_olama.jl works by providing 1 system prompt and 1 user prompt /api/generate # +# TODO: switch to OpenAI-compatible endpoint! +# ## Schema dedicated to [Ollama's models](https://ollama.ai/), which also managed the prompt templates # ## Rendering of converation history for the Ollama API (similar to OpenAI but not for the images) @@ -157,10 +159,14 @@ function aigenerate(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_ http_kwargs, api_kwargs...) + tokens_prompt = get(resp.response, :prompt_eval_count, 0) + tokens_completion = get(resp.response, :eval_count, 0) msg = AIMessage(; content = resp.response[:message][:content] |> strip, status = Int(resp.status), - tokens = (get(resp.response, :prompt_eval_count, 0), - get(resp.response, :eval_count, 0)), + cost = call_cost(tokens_prompt, tokens_completion, model_id), + ## not coming through yet anyway + ## finish_reason = get(resp.response, :finish_reason, nothing), + tokens = (tokens_prompt, tokens_completion), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) @@ -184,7 +190,7 @@ function aiembed(prompt_schema::AbstractOllamaSchema, args...; kwargs...) end """ -aiscan([prompt_schema::AbstractOllamaSchema,] prompt::ALLOWED_PROMPT_TYPE; + aiscan([prompt_schema::AbstractOllamaSchema,] prompt::ALLOWED_PROMPT_TYPE; image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, attach_to_latest::Bool = true, @@ -314,10 +320,12 @@ function aiscan(prompt_schema::AbstractOllamaSchema, prompt::ALLOWED_PROMPT_TYPE system = nothing, messages = conv_rendered, endpoint = "chat", model = model_id, http_kwargs, api_kwargs...) + tokens_prompt = get(resp.response, :prompt_eval_count, 0) + tokens_completion = get(resp.response, :eval_count, 0) msg = AIMessage(; content = resp.response[:message][:content] |> strip, status = Int(resp.status), - tokens = (get(resp.response, :prompt_eval_count, 0), - get(resp.response, :eval_count, 0)), + cost = call_cost(tokens_prompt, tokens_completion, model_id), + tokens = (tokens_prompt, tokens_completion), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) diff --git a/src/llm_ollama_managed.jl b/src/llm_ollama_managed.jl index 23286c3f6..647a53cec 100644 --- a/src/llm_ollama_managed.jl +++ b/src/llm_ollama_managed.jl @@ -214,10 +214,12 @@ function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_ time = @elapsed resp = ollama_api(prompt_schema, conv_rendered.prompt; conv_rendered.system, endpoint = "generate", model = model_id, http_kwargs, api_kwargs...) + tokens_prompt = get(resp.response, :prompt_eval_count, 0) + tokens_completion = get(resp.response, :eval_count, 0) msg = AIMessage(; content = resp.response[:response] |> strip, status = Int(resp.status), - tokens = (get(resp.response, :prompt_eval_count, 0), - get(resp.response, :eval_count, 0)), + cost = call_cost(tokens_prompt, tokens_completion, model_id), + tokens = (tokens_prompt, tokens_completion), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) @@ -326,6 +328,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, msg = DataMessage(; content = postprocess(resp.response[:embedding]), status = Int(resp.status), + cost = call_cost(0, 0, model_id), tokens = (0, 0), # token counts are not provided for embeddings elapsed = time) ## Reporting @@ -356,6 +359,7 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema, msg = DataMessage(; content = mapreduce(x -> x.content, hcat, messages), status = mapreduce(x -> x.status, max, messages), + cost = mapreduce(x -> x.cost, +, messages), tokens = (0, 0),# not tracked for embeddings in Ollama elapsed = sum(x -> x.elapsed, messages)) ## Reporting diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 5fa641227..3b35bf48a 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -273,6 +273,66 @@ function OpenAI.create_embeddings(provider::AbstractCustomProvider, kwargs...) end +""" + response_to_message(schema::AbstractOpenAISchema, + MSG::Type{AIMessage}, + choice, + resp; + model_id::AbstractString = "", + time::Float64 = 0.0, + run_id::Integer = rand(Int16), + sample_id::Union{Nothing, Integer} = nothing) + +Utility to facilitate unwrapping of HTTP response to a message type `MSG` provided for OpenAI-like responses + +Note: Extracts `finish_reason` and `log_prob` if available in the response. + +# Arguments +- `schema::AbstractOpenAISchema`: The schema for the prompt. +- `MSG::Type{AIMessage}`: The message type to be returned. +- `choice`: The choice from the response (eg, one of the completions). +- `resp`: The response from the OpenAI API. +- `model_id::AbstractString`: The model ID to use for generating the response. Defaults to an empty string. +- `time::Float64`: The elapsed time for the response. Defaults to `0.0`. +- `run_id::Integer`: The run ID for the response. Defaults to a random integer. +- `sample_id::Union{Nothing, Integer}`: The sample ID for the response (if there are multiple completions). Defaults to `nothing`. +""" +function response_to_message(schema::AbstractOpenAISchema, + MSG::Type{AIMessage}, + choice, + resp; + model_id::AbstractString = "", + time::Float64 = 0.0, + run_id::Int = Int(rand(Int32)), + sample_id::Union{Nothing, Integer} = nothing) + ## extract sum log probability + has_log_prob = haskey(choice, :logprobs) && + !isnothing(get(choice, :logprobs, nothing)) && + haskey(choice[:logprobs], :content) && + !isnothing(choice[:logprobs][:content]) + log_prob = if has_log_prob + sum([get(c, :logprob, 0.0) for c in choice[:logprobs][:content]]) + else + nothing + end + ## calculate cost + tokens_prompt = resp.response[:usage][:prompt_tokens] + tokens_completion = resp.response[:usage][:completion_tokens] + cost = call_cost(tokens_prompt, tokens_completion, model_id) + ## build AIMessage object + msg = MSG(; + content = choice[:message][:content] |> strip, + status = Int(resp.status), + cost, + run_id, + sample_id, + log_prob, + finish_reason = get(choice, :finish_reason, nothing), + tokens = (tokens_prompt, + tokens_completion), + elapsed = time) +end + ## User-Facing API """ aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; @@ -296,7 +356,11 @@ Generate an AI response based on a given prompt using the OpenAI API. - `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. - `http_kwargs`: A named tuple of HTTP keyword arguments. -- `api_kwargs`: A named tuple of API keyword arguments. +- `api_kwargs`: A named tuple of API keyword arguments. Useful parameters include: + - `temperature`: A float representing the temperature for sampling (ie, the amount of "creativity"). Often defaults to `0.7`. + - `logprobs`: A boolean indicating whether to return log probabilities for each token. Defaults to `false`. + - `n`: An integer representing the number of completions to generate at once (if supported). + - `stop`: A vector of strings representing the stop conditions for the conversation. Defaults to an empty vector. - `kwargs`: Prompt variables to be used to fill the prompt/template # Returns @@ -365,12 +429,26 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ conv_rendered; http_kwargs, api_kwargs...) - msg = AIMessage(; - content = r.response[:choices][begin][:message][:content] |> strip, - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], - r.response[:usage][:completion_tokens]), - elapsed = time) + ## Process one of more samples returned + msg = if length(r.response[:choices]) > 1 + run_id = Int(rand(Int32)) # remember one run ID + ## extract all message + msgs = [response_to_message(prompt_schema, AIMessage, choice, r; + time, model_id, run_id, sample_id = i) + for (i, choice) in enumerate(r.response[:choices])] + ## Order by log probability if available + ## bigger is better, keep it last + if all(x -> !isnothing(x.log_prob), msgs) + sort(msgs, by = x -> x.log_prob) + else + msgs + end + else + ## only 1 sample / 1 completion + choice = r.response[:choices][begin] + response_to_message(prompt_schema, AIMessage, choice, r; + time, model_id) + end ## Reporting verbose && @info _report_stats(msg, model_id) else @@ -454,7 +532,6 @@ function aiembed(prompt_schema::AbstractOpenAISchema, global MODEL_ALIASES ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) - time = @elapsed r = create_embeddings(prompt_schema, api_key, doc_or_docs, model_id; @@ -463,6 +540,7 @@ function aiembed(prompt_schema::AbstractOpenAISchema, msg = DataMessage(; content = mapreduce(x -> postprocess(x[:embedding]), hcat, r.response[:data]), status = Int(r.status), + cost = call_cost(r.response[:usage][:prompt_tokens], 0, model_id), tokens = (r.response[:usage][:prompt_tokens], 0), elapsed = time) ## Reporting @@ -585,8 +663,15 @@ function decode_choices(schema::TestEchoOpenAISchema, end function decode_choices(schema::OpenAISchema, choices, conv::AbstractVector; kwargs...) - if length(conv) > 0 && last(conv) isa AIMessage - conv[end] = decode_choices(schema, choices, last(conv)) + if length(conv) > 0 && last(conv) isa AIMessage && hasproperty(last(conv), :run_id) + ## if it is a multi-sample response, + ## Remember its run ID and convert all samples in that run + run_id = last(conv).run_id + for i in eachindex(conv) + if conv[i].run_id == run_id + conv[i] = decode_choices(schema, choices, conv[i]) + end + end end return conv end @@ -615,7 +700,8 @@ function decode_choices(schema::OpenAISchema, ## failed decoding content = nothing end - return AIMessage(; content, msg.status, msg.tokens, msg.elapsed) + ## create a new object with all the same fields except for content + return AIMessage(; [f => getfield(msg, f) for f in fieldnames(typeof(msg))]..., content) end """ @@ -701,6 +787,53 @@ function aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ return decode_choices(prompt_schema, decode_ids, msg_or_conv) end +function response_to_message(schema::AbstractOpenAISchema, + MSG::Type{DataMessage}, + choice, + resp; + return_type = nothing, + model_id::AbstractString = "", + time::Float64 = 0.0, + run_id::Int = Int(rand(Int32)), + sample_id::Union{Nothing, Integer} = nothing) + @assert !isnothing(return_type) "You must provide a return_type for DataMessage construction" + ## extract sum log probability + has_log_prob = haskey(choice, :logprobs) && + !isnothing(get(choice, :logprobs, nothing)) && + haskey(choice[:logprobs], :content) && + !isnothing(choice[:logprobs][:content]) + log_prob = if has_log_prob + sum([get(c, :logprob, 0.0) for c in choice[:logprobs][:content]]) + else + nothing + end + ## calculate cost + tokens_prompt = resp.response[:usage][:prompt_tokens] + tokens_completion = resp.response[:usage][:completion_tokens] + cost = call_cost(tokens_prompt, tokens_completion, model_id) + # "Safe" parsing of the response - it still fails if JSON is invalid + content = try + choice[:message][:tool_calls][1][:function][:arguments] |> + x -> JSON3.read(x, return_type) + catch e + @warn "There was an error parsing the response: $e. Using the raw response instead." + choice[:message][:tool_calls][1][:function][:arguments] |> + JSON3.read |> copy + end + ## build DataMessage object + msg = MSG(; + content = content, + status = Int(resp.status), + cost, + run_id, + sample_id, + log_prob, + finish_reason = get(choice, :finish_reason, nothing), + tokens = (tokens_prompt, + tokens_completion), + elapsed = time) +end + """ aiextract([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; return_type::Type, @@ -833,10 +966,12 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T ## global MODEL_ALIASES ## Function calling specifics - functions = [function_call_signature(return_type)] - function_call = Dict(:name => only(functions)["name"]) + tools = [Dict(:type => "function", :function => function_call_signature(return_type))] + ## force our function to be used + tool_choice = Dict(:type => "function", + :function => Dict(:name => only(tools)[:function]["name"])) ## Add the function call signature to the api_kwargs - api_kwargs = merge(api_kwargs, (; functions, function_call)) + api_kwargs = merge(api_kwargs, (; tools, tool_choice)) ## Find the unique ID for the model alias provided model_id = get(MODEL_ALIASES, model, model) conv_rendered = render(prompt_schema, prompt; conversation, kwargs...) @@ -847,20 +982,26 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T conv_rendered; http_kwargs, api_kwargs...) - # "Safe" parsing of the response - it still fails if JSON is invalid - content = try - r.response[:choices][begin][:message][:function_call][:arguments] |> - x -> JSON3.read(x, return_type) - catch e - @warn "There was an error parsing the response: $e. Using the raw response instead." - r.response[:choices][begin][:message][:function_call][:arguments] |> - JSON3.read |> copy + ## Process one of more samples returned + msg = if length(r.response[:choices]) > 1 + run_id = Int(rand(Int32)) # remember one run ID + ## extract all message + msgs = [response_to_message(prompt_schema, DataMessage, choice, r; + return_type, time, model_id, run_id, sample_id = i) + for (i, choice) in enumerate(r.response[:choices])] + ## Order by log probability if available + ## bigger is better, keep it last + if all(x -> !isnothing(x.log_prob), msgs) + sort(msgs, by = x -> x.log_prob) + else + msgs + end + else + ## only 1 sample / 1 completion + choice = r.response[:choices][begin] + response_to_message(prompt_schema, DataMessage, choice, r; + return_type, time, model_id) end - msg = DataMessage(; content, - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], - r.response[:usage][:completion_tokens]), - elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) else @@ -879,7 +1020,7 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T end """ -aiscan([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; + aiscan([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; image_url::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, image_path::Union{Nothing, AbstractString, Vector{<:AbstractString}} = nothing, image_detail::AbstractString = "auto", @@ -998,12 +1139,26 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE conv_rendered; http_kwargs, api_kwargs...) - msg = AIMessage(; - content = r.response[:choices][begin][:message][:content] |> strip, - status = Int(r.status), - tokens = (r.response[:usage][:prompt_tokens], - r.response[:usage][:completion_tokens]), - elapsed = time) + ## Process one of more samples returned + msg = if length(r.response[:choices]) > 1 + run_id = Int(rand(Int32)) # remember one run ID + ## extract all message + msgs = [response_to_message(prompt_schema, AIMessage, choice, r; + time, model_id, run_id, sample_id = i) + for (i, choice) in enumerate(r.response[:choices])] + ## Order by log probability if available + ## bigger is better, keep it last + if all(x -> !isnothing(x.log_prob), msgs) + sort(msgs, by = x -> x.log_prob) + else + msgs + end + else + ## only 1 sample / 1 completion + choice = r.response[:choices][begin] + response_to_message(prompt_schema, AIMessage, choice, r; + time, model_id) + end ## Reporting verbose && @info _report_stats(msg, model_id) else diff --git a/src/llm_shared.jl b/src/llm_shared.jl index fd6401187..a611c5929 100644 --- a/src/llm_shared.jl +++ b/src/llm_shared.jl @@ -65,7 +65,7 @@ end """ finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, - msg::Union{Nothing, AbstractMessage}; + msg::Union{Nothing, AbstractMessage, AbstractVector{<:AbstractMessage}}; return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], @@ -81,7 +81,7 @@ Finalizes the outputs of the ai* functions by either returning the conversation - `kwargs...`: Variables to replace in the prompt template. """ function finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, - msg::Union{Nothing, AbstractMessage}; + msg::Union{Nothing, AbstractMessage, AbstractVector{<:AbstractMessage}}; return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], @@ -92,7 +92,12 @@ function finalize_outputs(prompt::ALLOWED_PROMPT_TYPE, conv_rendered::Any, # This is a duplication of work, as we already have the rendered messages in conv_rendered, # but we prioritize the user's experience over performance here (ie, render(OpenAISchema,msgs) does everything under the hood) output = render(NoSchema(), prompt; conversation, kwargs...) - push!(output, msg) + if msg isa AbstractVector + ## handle multiple messages (multi-sample) + append!(output, msg) + else + push!(output, msg) + end else output = conv_rendered end @@ -113,4 +118,4 @@ function decode_choices(schema::AbstractPromptSchema, choices, conv; kwargs...) end function decode_choices(schema::AbstractPromptSchema, choices, conv::Nothing; kwargs...) nothing -end +end \ No newline at end of file diff --git a/src/messages.jl b/src/messages.jl index 63d153b19..cda980462 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -61,18 +61,64 @@ function UserMessageWithImages(content::T, image_url::Vector{<:AbstractString}, @assert length(not_allowed_kwargs)==0 "Error: Some placeholders are invalid, as they are reserved for `ai*` functions. Change: $(join(not_allowed_kwargs,","))" return UserMessageWithImages{T}(content, string.(image_url), variables, type) end + +""" + AIMessage + +A message type for AI-generated text-based responses. +Returned by `aigenerate`, `aiclassify`, and `aiscan` functions. + +# Fields +- `content::Union{AbstractString, Nothing}`: The content of the message. +- `status::Union{Int, Nothing}`: The status of the message from the API. +- `tokens::Tuple{Int, Int}`: The number of tokens used (prompt,completion). +- `elapsed::Float64`: The time taken to generate the response in seconds. +- `cost::Union{Nothing, Float64}`: The cost of the API call (calculated with information from `MODEL_REGISTRY`). +- `log_prob::Union{Nothing, Float64}`: The log probability of the response. +- `finish_reason::Union{Nothing, String}`: The reason the response was finished. +- `run_id::Union{Nothing, Int}`: The unique ID of the run. +- `sample_id::Union{Nothing, Int}`: The unique ID of the sample (if multiple samples are generated, they will all have the same `run_id`). +""" Base.@kwdef struct AIMessage{T <: Union{AbstractString, Nothing}} <: AbstractChatMessage content::T = nothing status::Union{Int, Nothing} = nothing tokens::Tuple{Int, Int} = (-1, -1) elapsed::Float64 = -1.0 + cost::Union{Nothing, Float64} = nothing + log_prob::Union{Nothing, Float64} = nothing + finish_reason::Union{Nothing, String} = nothing + run_id::Union{Nothing, Int} = Int(rand(Int16)) + sample_id::Union{Nothing, Int} = nothing _type::Symbol = :aimessage end + +""" + DataMessage + +A message type for AI-generated data-based responses, ie, different `content` than text. +Returned by `aiextract`, and `aiextract` functions. + +# Fields +- `content::Union{AbstractString, Nothing}`: The content of the message. +- `status::Union{Int, Nothing}`: The status of the message from the API. +- `tokens::Tuple{Int, Int}`: The number of tokens used (prompt,completion). +- `elapsed::Float64`: The time taken to generate the response in seconds. +- `cost::Union{Nothing, Float64}`: The cost of the API call (calculated with information from `MODEL_REGISTRY`). +- `log_prob::Union{Nothing, Float64}`: The log probability of the response. +- `finish_reason::Union{Nothing, String}`: The reason the response was finished. +- `run_id::Union{Nothing, Int}`: The unique ID of the run. +- `sample_id::Union{Nothing, Int}`: The unique ID of the sample (if multiple samples are generated, they will all have the same `run_id`). +""" Base.@kwdef struct DataMessage{T <: Any} <: AbstractDataMessage content::T status::Union{Int, Nothing} = nothing tokens::Tuple{Int, Int} = (-1, -1) elapsed::Float64 = -1.0 + cost::Union{Nothing, Float64} = nothing + log_prob::Union{Nothing, Float64} = nothing + finish_reason::Union{Nothing, String} = nothing + run_id::Union{Nothing, Int} = Int(rand(Int16)) + sample_id::Union{Nothing, Int} = nothing _type::Symbol = :datamessage end @@ -83,11 +129,13 @@ end isusermessage(m::AbstractMessage) = m isa UserMessage issystemmessage(m::AbstractMessage) = m isa SystemMessage isdatamessage(m::AbstractMessage) = m isa DataMessage +isaimessage(m::AbstractMessage) = m isa AIMessage # equality check for testing, only equal if all fields are equal and type is the same Base.var"=="(m1::AbstractMessage, m2::AbstractMessage) = false function Base.var"=="(m1::T, m2::T) where {T <: AbstractMessage} - all([getproperty(m1, f) == getproperty(m2, f) for f in fieldnames(T)]) + ## except for run_id, that's random and not important for content comparison + all([getproperty(m1, f) == getproperty(m2, f) for f in fieldnames(T) if f != :run_id]) end ## Vision Models -- Constructor and Conversion diff --git a/src/precompilation.jl b/src/precompilation.jl index a1d823cc6..2a7fc9703 100644 --- a/src/precompilation.jl +++ b/src/precompilation.jl @@ -8,7 +8,10 @@ load_templates!(); # API Calls prep mock_response = Dict(:choices => [ Dict(:message => Dict(:content => "Hello!", - :function_call => Dict(:arguments => JSON3.write(Dict(:x => 1))))), + :tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(Dict(:x => 1)))), + ]), + :finish_reason => "stop"), ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) schema = TestEchoOpenAISchema(; response = mock_response, status = 200) diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 733dc0495..c016b6392 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -389,7 +389,10 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", "Mistral AI's hosted model for embeddings."), "echo" => ModelSpec("echo", TestEchoOpenAISchema(; - response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello!"), + :finish_reason => "stop"), + ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)), status = 200), diff --git a/src/utils.jl b/src/utils.jl index 3170056c8..65cb94bd3 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -244,15 +244,20 @@ function _extract_handlebar_variables(vect::Vector{Dict{String, <:AbstractString end """ - call_cost(msg, model::String; - cost_of_token_prompt::Number = default_prompt_cost, - cost_of_token_generation::Number = default_generation_cost) -> Number + call_cost(prompt_tokens::Int, completion_tokens::Int, model::String; + cost_of_token_prompt::Number = get(MODEL_REGISTRY, + model, + (; cost_of_token_prompt = 0.0)).cost_of_token_prompt, + cost_of_token_generation::Number = get(MODEL_REGISTRY, model, + (; cost_of_token_generation = 0.0)).cost_of_token_generation) + + call_cost(msg, model::String) Calculate the cost of a call based on the number of tokens in the message and the cost per token. # Arguments -- `msg`: The message object, which should contain a `tokens` field - with two elements: [number_of_prompt_tokens, number_of_generation_tokens]. +- `prompt_tokens::Int`: The number of tokens used in the prompt. +- `completion_tokens::Int`: The number of tokens used in the completion. - `model::String`: The name of the model to use for determining token costs. If the model is not found in `MODEL_REGISTRY`, default costs are used. - `cost_of_token_prompt::Number`: The cost per prompt token. Defaults to the cost in `MODEL_REGISTRY` @@ -271,30 +276,49 @@ MODEL_REGISTRY = Dict( "model2" => (cost_of_token_prompt = 0.07, cost_of_token_generation = 0.02) ) -msg1 = AIMessage([10, 20]) # 10 prompt tokens, 20 generation tokens +cost1 = call_cost(10, 20, "model1") + +# from message +msg1 = AIMessage(;tokens=[10, 20]) # 10 prompt tokens, 20 generation tokens cost1 = call_cost(msg1, "model1") # cost1 = 10 * 0.05 + 20 * 0.10 = 2.5 -msg2 = DataMessage([15, 30]) # 15 prompt tokens, 30 generation tokens -cost2 = call_cost(msg2, "model2") -# cost2 = 15 * 0.07 + 30 * 0.02 = 1.35 - # Using custom token costs -msg3 = AIMessage([5, 10]) -cost3 = call_cost(msg3, "model3", cost_of_token_prompt = 0.08, cost_of_token_generation = 0.12) -# cost3 = 5 * 0.08 + 10 * 0.12 = 1.6 +cost2 = call_cost(10, 20, "model3"; cost_of_token_prompt = 0.08, cost_of_token_generation = 0.12) +# cost2 = 10 * 0.08 + 20 * 0.12 = 3.2 ``` """ -function call_cost(msg, model::String; +function call_cost(prompt_tokens::Int, completion_tokens::Int, model::String; cost_of_token_prompt::Number = get(MODEL_REGISTRY, model, (; cost_of_token_prompt = 0.0)).cost_of_token_prompt, cost_of_token_generation::Number = get(MODEL_REGISTRY, model, (; cost_of_token_generation = 0.0)).cost_of_token_generation) - cost = msg.tokens[1] * cost_of_token_prompt + - msg.tokens[2] * cost_of_token_generation + cost = prompt_tokens * cost_of_token_prompt + + completion_tokens * cost_of_token_generation + return cost +end +function call_cost(msg, model::String) + cost = if !isnothing(msg.cost) + msg.cost + else + call_cost(msg.tokens[1], msg.tokens[2], model) + end return cost end +## dispatch for array -> take unique messages only (eg, for multiple samples we count only once) +function call_cost(conv::AbstractVector, model::String) + sum_ = 0.0 + visited_runs = Set{Int}() + for msg in conv + if isnothing(msg.run_id) || (msg.run_id ∉ visited_runs) + sum_ += call_cost(msg, model) + push!(visited_runs, msg.run_id) + end + end + return sum_ +end + # helper to produce summary message of how many tokens were used and for how much function _report_stats(msg, model::String) @@ -302,6 +326,11 @@ function _report_stats(msg, cost_str = iszero(cost) ? "" : " @ Cost: \$$(round(cost; digits=4))" return "Tokens: $(sum(msg.tokens))$(cost_str) in $(round(msg.elapsed;digits=1)) seconds" end +## dispatch for array -> take last message +function _report_stats(msg::AbstractVector, + model::String) + _report_stats(last(msg), model) +end # Loads and encodes the provided image path as a base64 string function _encode_local_image(image_path::AbstractString; base64_only::Bool = false) @assert isfile(image_path) "`image_path` must be a valid path to an image file. File: $image_path not found." diff --git a/test/Experimental/RAGTools/evaluation.jl b/test/Experimental/RAGTools/evaluation.jl index 1e5e79f03..9c48eba7b 100644 --- a/test/Experimental/RAGTools/evaluation.jl +++ b/test/Experimental/RAGTools/evaluation.jl @@ -87,7 +87,9 @@ end if content[:model] == "mock-gen" user_msg = last(content[:messages]) - response = Dict(:choices => [Dict(:message => user_msg)], + response = Dict(:choices => [ + Dict(:message => user_msg, :finish_reason => "stop"), + ], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), @@ -101,10 +103,11 @@ end elseif content[:model] == "mock-meta" user_msg = last(content[:messages]) response = Dict(:choices => [ - Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(MaybeMetadataItems([ - MetadataItem("yes", "category"), - ]))))), - ], + Dict(:finish_reason => "stop", + :message => Dict(:tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(MaybeMetadataItems([ + MetadataItem("yes", "category"), + ]))))]))], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), @@ -112,9 +115,10 @@ end elseif content[:model] == "mock-qa" user_msg = last(content[:messages]) response = Dict(:choices => [ - Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(QAItem("Question", - "Answer"))))), - ], + Dict(:finish_reason => "stop", + :message => Dict(:tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(QAItem("Question", + "Answer"))))]))], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), @@ -122,14 +126,14 @@ end elseif content[:model] == "mock-judge" user_msg = last(content[:messages]) response = Dict(:choices => [ - Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(JudgeAllScores(5, - 5, - 5, - 5, - 5, - "Some reasons", - 5.0))))), - ], + Dict(:message => Dict(:tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(JudgeAllScores(5, + 5, + 5, + 5, + 5, + "Some reasons", + 5.0))))]))], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), diff --git a/test/Experimental/RAGTools/generation.jl b/test/Experimental/RAGTools/generation.jl index 6fbb95057..b7fecd739 100644 --- a/test/Experimental/RAGTools/generation.jl +++ b/test/Experimental/RAGTools/generation.jl @@ -1,4 +1,6 @@ -using PromptingTools.Experimental.RAGTools: MaybeMetadataItems, MetadataItem, build_context +using PromptingTools.Experimental.RAGTools: ChunkIndex, + CandidateChunks, build_context, airag +using PromptingTools.Experimental.RAGTools: MaybeMetadataItems, MetadataItem @testset "build_context" begin index = ChunkIndex(; @@ -29,7 +31,7 @@ end @testset "airag" begin # test with a mock server - PORT = rand(1000:2000) + PORT = rand(20000:30000) PT.register_model!(; name = "mock-emb", schema = PT.CustomOpenAISchema()) PT.register_model!(; name = "mock-meta", schema = PT.CustomOpenAISchema()) PT.register_model!(; name = "mock-gen", schema = PT.CustomOpenAISchema()) @@ -39,7 +41,9 @@ end if content[:model] == "mock-gen" user_msg = last(content[:messages]) - response = Dict(:choices => [Dict(:message => user_msg)], + response = Dict(:choices => [ + Dict(:message => user_msg, :finish_reason => "stop"), + ], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), @@ -53,10 +57,11 @@ end elseif content[:model] == "mock-meta" user_msg = last(content[:messages]) response = Dict(:choices => [ - Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(MaybeMetadataItems([ - MetadataItem("yes", "category"), - ]))))), - ], + Dict(:finish_reason => "stop", + :message => Dict(:tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(MaybeMetadataItems([ + MetadataItem("yes", "category"), + ]))))]))], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), diff --git a/test/Experimental/RAGTools/preparation.jl b/test/Experimental/RAGTools/preparation.jl index f781c2e05..dc14b087e 100644 --- a/test/Experimental/RAGTools/preparation.jl +++ b/test/Experimental/RAGTools/preparation.jl @@ -82,7 +82,9 @@ end if content[:model] == "mock-gen" user_msg = last(content[:messages]) - response = Dict(:choices => [Dict(:message => user_msg)], + response = Dict(:choices => [ + Dict(:message => user_msg, :finish_reason => "stop"), + ], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), @@ -96,10 +98,11 @@ end elseif content[:model] == "mock-meta" user_msg = last(content[:messages]) response = Dict(:choices => [ - Dict(:message => Dict(:function_call => Dict(:arguments => JSON3.write(MaybeMetadataItems([ - MetadataItem("yes", "category"), - ]))))), - ], + Dict(:finish_reason => "stop", + :message => Dict(:tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(MaybeMetadataItems([ + MetadataItem("yes", "category"), + ]))))]))], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), diff --git a/test/llm_interface.jl b/test/llm_interface.jl index 8e010bc11..d54cad0f4 100644 --- a/test/llm_interface.jl +++ b/test/llm_interface.jl @@ -1,12 +1,15 @@ using PromptingTools: TestEchoOpenAISchema, render, OpenAISchema using PromptingTools: AIMessage, SystemMessage, AbstractMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage +using PromptingTools: response_to_message, AbstractPromptSchema @testset "ai* default schema" begin OLD_PROMPT_SCHEMA = PromptingTools.PROMPT_SCHEMA ### AIGenerate # corresponds to OpenAI API v1 - response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello!"), :finish_reason => "stop"), + ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) schema = TestEchoOpenAISchema(; response, status = 200) @@ -16,6 +19,9 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage content = "Hello!" |> strip, status = 200, tokens = (2, 1), + run_id = msg.run_id, + finish_reason = "stop", + cost = 0.0, elapsed = msg.elapsed) @test msg == expected_output @@ -25,13 +31,18 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage content = nothing, status = 200, tokens = (2, 1), + run_id = msg.run_id, + cost = 0.0, + finish_reason = "stop", elapsed = msg.elapsed) @test msg == expected_output ### AIExtract response1 = Dict(:choices => [ - Dict(:message => Dict(:function_call => Dict(:arguments => "{\"content\": \"x\"}"))), - ], + Dict(:message => Dict(:tool_calls => [ + Dict(:function => Dict(:arguments => "{\"content\": \"x\"}")), + ]), + :finish_reason => "stop")], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) schema = TestEchoOpenAISchema(; response = response1, status = 200) @@ -44,6 +55,9 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage content = MyType("x"), status = 200, tokens = (2, 1), + run_id = msg.run_id, + cost = 0.0, + finish_reason = "stop", elapsed = msg.elapsed) @test msg == expected_output @@ -59,9 +73,18 @@ using PromptingTools: UserMessage, UserMessageWithImages, DataMessage content = ones(128), status = 200, tokens = (2, 0), + run_id = msg.run_id, + cost = 0.0, elapsed = msg.elapsed) @test msg == expected_output ## Return things to previous PromptingTools.PROMPT_SCHEMA = OLD_PROMPT_SCHEMA + + ## Check response_to_message throws by default + struct Random123Schema <: AbstractPromptSchema end + @test_throws ArgumentError response_to_message(Random123Schema(), + AIMessage, + nothing, + nothing) end diff --git a/test/llm_openai.jl b/test/llm_openai.jl index ba59c4097..71beb0ef2 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -3,7 +3,7 @@ using PromptingTools: AIMessage, SystemMessage, AbstractMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage using PromptingTools: CustomProvider, CustomOpenAISchema, MistralOpenAISchema, MODEL_EMBEDDING -using PromptingTools: encode_choices, decode_choices +using PromptingTools: encode_choices, decode_choices, response_to_message, call_cost @testset "render-OpenAI" begin schema = OpenAISchema() @@ -181,11 +181,18 @@ end @testset "OpenAI.create_chat" begin # Test CustomOpenAISchema() with a mock server - PORT = rand(1000:2000) + PORT = rand(10000:20000) echo_server = HTTP.serve!(PORT, verbose = -1) do req content = JSON3.read(req.body) user_msg = last(content[:messages]) - response = Dict(:choices => [Dict(:message => user_msg)], + response = Dict(:choices => [ + Dict(:message => user_msg, + :logprobs => Dict(:content => [ + Dict(:logprob => -0.1), + Dict(:logprob => -0.2), + ]), + :finish_reason => "stop"), + ], :model => content[:model], :usage => Dict(:total_tokens => length(user_msg[:content]), :prompt_tokens => length(user_msg[:content]), @@ -201,13 +208,18 @@ end return_all = false) @test msg.content == prompt @test msg.tokens == (length(prompt), 0) + @test msg.finish_reason == "stop" + ## single message, must be nothing + @test msg.sample_id |> isnothing + ## sum up log probs when provided + @test msg.log_prob ≈ -0.3 # clean up close(echo_server) end @testset "OpenAI.create_embeddings" begin # Test CustomOpenAISchema() with a mock server - PORT = rand(1000:2000) + PORT = rand(10000:20000) echo_server = HTTP.serve!(PORT, verbose = -1) do req content = JSON3.read(req.body) response = Dict(:data => [Dict(:embedding => ones(128))], @@ -230,9 +242,116 @@ end close(echo_server) end +@testset "response_to_message" begin + # Mock the response and choice data + mock_choice = Dict(:message => Dict(:content => "Hello!"), + :logprobs => Dict(:content => [Dict(:logprob => -0.5), Dict(:logprob => -0.4)]), + :finish_reason => "stop") + mock_response = (; + response = Dict(:choices => [mock_choice], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)), + status = 200) + + # Test with valid logprobs + msg = response_to_message(OpenAISchema(), + AIMessage, + mock_choice, + mock_response; + model_id = "gpt4t") + @test msg isa AIMessage + @test msg.content == "Hello!" + @test msg.tokens == (2, 1) + @test msg.log_prob ≈ -0.9 + @test msg.finish_reason == "stop" + @test msg.sample_id == nothing + @test msg.cost == call_cost(2, 1, "gpt4t") + + # Test without logprobs + choice = deepcopy(mock_choice) + delete!(choice, :logprobs) + msg = response_to_message(OpenAISchema(), AIMessage, choice, mock_response) + @test isnothing(msg.log_prob) + + # with sample_id and run_id + msg = response_to_message(OpenAISchema(), + AIMessage, + mock_choice, + mock_response; + run_id = 1, + sample_id = 2, + time = 2.0) + @test msg.run_id == 1 + @test msg.sample_id == 2 + @test msg.elapsed == 2.0 + + #### With DataMessage + # Mock the response and choice data + mock_choice = Dict(:message => Dict(:content => "Hello!", + :tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(Dict(:x => 1)))), + ]), + :logprobs => Dict(:content => [Dict(:logprob => -0.5), Dict(:logprob => -0.4)]), + :finish_reason => "stop") + mock_response = (; + response = Dict(:choices => [mock_choice], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)), + status = 200) + struct RandomType1235 + x::Int + end + return_type = RandomType1235 + # Catch missing return_type + @test_throws AssertionError response_to_message(OpenAISchema(), + DataMessage, + mock_choice, + mock_response; + model_id = "gpt4t") + + # Test with valid logprobs + msg = response_to_message(OpenAISchema(), + DataMessage, + mock_choice, + mock_response; + return_type, + model_id = "gpt4t") + @test msg isa DataMessage + @test msg.content == RandomType1235(1) + @test msg.tokens == (2, 1) + @test msg.log_prob ≈ -0.9 + @test msg.finish_reason == "stop" + @test msg.sample_id == nothing + @test msg.cost == call_cost(2, 1, "gpt4t") + + # Test without logprobs + choice = deepcopy(mock_choice) + delete!(choice, :logprobs) + msg = response_to_message(OpenAISchema(), + DataMessage, + choice, + mock_response; + return_type) + @test isnothing(msg.log_prob) + + # with sample_id and run_id + msg = response_to_message(OpenAISchema(), + DataMessage, + mock_choice, + mock_response; + return_type, + run_id = 1, + sample_id = 2, + time = 2.0) + @test msg.run_id == 1 + @test msg.sample_id == 2 + @test msg.elapsed == 2.0 +end + @testset "aigenerate-OpenAI" begin # corresponds to OpenAI API v1 - response = Dict(:choices => [Dict(:message => Dict(:content => "Hello!"))], + response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello!"), + :finish_reason => "stop"), + ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) # Test the monkey patch @@ -247,6 +366,8 @@ end content = "Hello!" |> strip, status = 200, tokens = (2, 1), + finish_reason = "stop", + cost = msg.cost, elapsed = msg.elapsed) @test msg == expected_output @test schema1.inputs == @@ -263,12 +384,30 @@ end content = "Hello!" |> strip, status = 200, tokens = (2, 1), + finish_reason = "stop", + cost = msg.cost, elapsed = msg.elapsed) @test msg == expected_output @test schema1.inputs == [Dict("role" => "system", "content" => "Act as a helpful AI assistant") Dict("role" => "user", "content" => "Hello World")] @test schema2.model_id == "gpt-4" + + ## Test multiple samples + response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello1!"), + :finish_reason => "stop"), + Dict(:message => Dict(:content => "Hello2!"), + :finish_reason => "stop"), + ], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema3 = TestEchoOpenAISchema(; response, status = 200) + conv = aigenerate(schema3, UserMessage("Hello {{name}}"), + model = "gpt4", http_kwargs = (; verbose = 3), + api_kwargs = (; temperature = 0, n = 2), + name = "World") + @test conv[end - 1].content == "Hello1!" + @test conv[end].content == "Hello2!" end @testset "aiembed-OpenAI" begin @@ -283,6 +422,7 @@ end content = ones(128), status = 200, tokens = (2, 0), + cost = msg.cost, elapsed = msg.elapsed) @test msg == expected_output @test schema1.inputs == "Hello World" @@ -298,6 +438,7 @@ end content = ones(128, 2), status = 200, tokens = (4, 0), + cost = msg.cost, elapsed = msg.elapsed) @test msg == expected_output @test schema2.inputs == ["Hello World", "Hello back"] @@ -307,6 +448,7 @@ end expected_output = DataMessage(; content = ones(128, 2), status = 200, + cost = msg.cost, tokens = (4, 0), elapsed = msg.elapsed) @test msg == expected_output @@ -381,6 +523,17 @@ end decoded_conv = decode_choices(OpenAISchema(), ["true", "false"], conv) @test decoded_conv[end].content == "true" + # Decode with multiple samples + conv = [ + AIMessage("1"), # do not touch, different run + AIMessage(; content = "1", run_id = 1, sample_id = 1), + AIMessage(; content = "1", run_id = 1, sample_id = 2), + ] + decoded_conv = decode_choices(OpenAISchema(), ["true", "false"], conv) + @test decoded_conv[1].content == "1" + @test decoded_conv[2].content == "true" + @test decoded_conv[3].content == "true" + # Nothing (when dry_run=true) @test isnothing(decode_choices(OpenAISchema(), ["true", "false"], nothing)) @@ -392,7 +545,10 @@ end @testset "aiclassify-OpenAI" begin # corresponds to OpenAI API v1 - response = Dict(:choices => [Dict(:message => Dict(:content => "1"))], + response = Dict(:choices => [ + Dict(:message => Dict(:content => "1"), + :finish_reason => "stop"), + ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) # Real generation API @@ -407,6 +563,8 @@ end content = "A", status = 200, tokens = (2, 1), + finish_reason = "stop", + cost = msg.cost, elapsed = msg.elapsed) @test msg == expected_output @test schema1.inputs == @@ -414,3 +572,94 @@ end "content" => "You are a world-class classification specialist. \n\nYour task is to select the most appropriate label from the given choices for the given user input.\n\n**Available Choices:**\n---\n1. \"A\" for any animal or creature\n2. \"P\" for for any plant or tree\n3. \"O\" for for everything else\n---\n\n**Instructions:**\n- You must respond in one word. \n- You must respond only with the label ID (e.g., \"1\", \"2\", ...) that best fits the input.\n"), Dict("role" => "user", "content" => "User Input: pelican\n\nLabel:\n")] end + +@testset "aiextract-OpenAI" begin + # mock return type + struct RandomType1235 + x::Int + end + return_type = RandomType1235 + + mock_choice = Dict(:message => Dict(:content => "Hello!", + :tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(Dict(:x => 1)))), + ]), + :logprobs => Dict(:content => [Dict(:logprob => -0.5), Dict(:logprob => -0.4)]), + :finish_reason => "stop") + ## Test with a single sample + response = Dict(:choices => [mock_choice], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema1 = TestEchoOpenAISchema(; response, status = 200) + msg = aiextract(schema1, "Extract number 1"; return_type, + model = "gpt4", + api_kwargs = (; temperature = 0, n = 2)) + @test msg.content == RandomType1235(1) + @test msg.log_prob ≈ -0.9 + + ## Test multiple samples -- mock_choice is less probable + mock_choice2 = Dict(:message => Dict(:content => "Hello!", + :tool_calls => [ + Dict(:function => Dict(:arguments => JSON3.write(Dict(:x => 1)))), + ]), + :logprobs => Dict(:content => [Dict(:logprob => -1.2), Dict(:logprob => -0.4)]), + :finish_reason => "stop") + + response = Dict(:choices => [mock_choice, mock_choice2], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema2 = TestEchoOpenAISchema(; response, status = 200) + conv = aiextract(schema2, "Extract number 1"; return_type, + model = "gpt4", + api_kwargs = (; temperature = 0, n = 2)) + @test conv[1].content == RandomType1235(1) + @test conv[1].log_prob ≈ -1.6 # sorted first, despite sent later + @test conv[2].content == RandomType1235(1) + @test conv[2].log_prob ≈ -0.9 + + ## Wrong return_type so it returns a Dict + struct RandomType1236 + x::Int + y::Int + end + return_type = RandomType1236 + conv = aiextract(schema2, "Extract number 1"; return_type, + model = "gpt4", + api_kwargs = (; temperature = 0, n = 2)) + conv[1].content isa AbstractDict + conv[2].content isa AbstractDict +end + +@testset "aiscan-OpenAI" begin + ## Test with single sample and log_probs samples + response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello1!"), + :finish_reason => "stop", + :logprobs => Dict(:content => [ + Dict(:logprob => -0.1), + Dict(:logprob => -0.2), + ])), + ], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema1 = TestEchoOpenAISchema(; response, status = 200) + msg = aiscan(schema1, "Describe the image"; + image_url = "https://example.com/image.png", + model = "gpt4", http_kwargs = (; verbose = 3), + api_kwargs = (; temperature = 0)) + @test msg.content == "Hello1!" + @test msg.log_prob ≈ -0.3 + + ## Test multiple samples + response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello1!"), + :finish_reason => "stop"), + Dict(:message => Dict(:content => "Hello2!"), + :finish_reason => "stop"), + ], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema1 = TestEchoOpenAISchema(; response, status = 200) + conv = aiscan(schema1, "Describe the image"; + image_url = "https://example.com/image.png", + model = "gpt4", http_kwargs = (; verbose = 3), + api_kwargs = (; temperature = 0, n = 2)) + @test conv[end - 1].content == "Hello1!" + @test conv[end].content == "Hello2!" +end \ No newline at end of file diff --git a/test/llm_shared.jl b/test/llm_shared.jl index f449d9e5a..9c9c2b3a9 100644 --- a/test/llm_shared.jl +++ b/test/llm_shared.jl @@ -267,4 +267,32 @@ end conversation, return_all = true) @test output == expected_output + + ## With multiple samples + conversation = [ + SystemMessage("System message 1"), + UserMessage("User message {{name}}"), + AIMessage("AI message"), + ] + messages = [ + UserMessage("User message {{name}}"), + AIMessage("AI message 2"), + ] + msg = AIMessage("AI message 3") + expected_output = [ + SystemMessage("System message 1"), + UserMessage("User message {{name}}"), + AIMessage("AI message"), + UserMessage("User message John", [:name], :usermessage), + AIMessage("AI message 2"), + msg, + msg, + ] + output = finalize_outputs(messages, + [], + [msg, msg]; + name = "John", + conversation, + return_all = true) + @test output == expected_output end diff --git a/test/messages.jl b/test/messages.jl index 95cc5e15a..e90dcf467 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -1,7 +1,7 @@ using PromptingTools: AIMessage, SystemMessage, MetadataMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage using PromptingTools: _encode_local_image, attach_images_to_user_message -using PromptingTools: isusermessage, issystemmessage, isdatamessage +using PromptingTools: isusermessage, issystemmessage, isdatamessage, isaimessage @testset "Message constructors" begin # Creates an instance of MSG with the given content string. @@ -29,8 +29,8 @@ using PromptingTools: isusermessage, issystemmessage, isdatamessage @test UserMessage(content) |> isusermessage @test SystemMessage(content) |> issystemmessage @test DataMessage(; content) |> isdatamessage + @test AIMessage(; content) |> isaimessage end - @testset "UserMessageWithImages" begin content = "Hello, world!" image_path = joinpath(@__DIR__, "data", "julia.png") diff --git a/test/utils.jl b/test/utils.jl index a55310b84..5d6961cee 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -129,20 +129,23 @@ end end @testset "call_cost" begin + @test cost = call_cost(1000, 100, "unknown_model"; + cost_of_token_prompt = 1, + cost_of_token_generation = 1) ≈ 1100 msg = AIMessage(; content = "", tokens = (1000, 2000)) cost = call_cost(msg, "unknown_model") @test cost == 0.0 @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 0.5e-6 + 1.5e-6 * 2000 + # Test vector - same message, count once + @test call_cost([msg, msg], "gpt-3.5-turbo") ≈ (1000 * 0.5e-6 + 1.5e-6 * 2000) + msg2 = AIMessage(; content = "", tokens = (1000, 2000)) + @test call_cost([msg, msg2], "gpt-3.5-turbo") ≈ (1000 * 0.5e-6 + 1.5e-6 * 2000) * 2 + msg = DataMessage(; content = nothing, tokens = (1000, 1000)) cost = call_cost(msg, "unknown_model") @test cost == 0.0 @test call_cost(msg, "gpt-3.5-turbo") ≈ 1000 * 0.5e-6 + 1.5e-6 * 1000 - - @test call_cost(msg, - "gpt-3.5-turbo"; - cost_of_token_prompt = 1, - cost_of_token_generation = 1) ≈ 1000 + 1000 end @testset "report_stats" begin From c81140743df165406fc0d43805b696aa1184f2a2 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Fri, 23 Feb 2024 10:38:01 +0000 Subject: [PATCH 122/251] Add more API Providers (#80) --- CHANGELOG.md | 1 + docs/src/examples/working_with_custom_apis.md | 64 +++++++++++++++- docs/src/frequently_asked_questions.md | 2 +- src/llm_interface.jl | 30 ++++++++ src/llm_openai.jl | 73 ++++++++++++++++--- src/user_preferences.jl | 46 ++++++++++-- 6 files changed, 197 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cab889975..faba492f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `run_id` - ID of the AI API call - `sample_id` - ID of the sample in the batch if you requested multiple completions, otherwise `sample_id==nothing` (they will have the same `run_id`) - `finish_reason` - the reason why the AI stopped generating the sequence (eg, "stop", "length") to provide more visibility for the user +- Support for Fireworks.ai and Together.ai providers for fast and easy access to open-source models. Requires environment variables `FIREWORKS_API_KEY` and `TOGETHER_API_KEY` to be set, respectively. See the `?FireworksOpenAISchema` and `?TogetherOpenAISchema` for more information. ### Fixed diff --git a/docs/src/examples/working_with_custom_apis.md b/docs/src/examples/working_with_custom_apis.md index 2a083d778..d4f09fd1a 100644 --- a/docs/src/examples/working_with_custom_apis.md +++ b/docs/src/examples/working_with_custom_apis.md @@ -97,4 +97,66 @@ ai"Say hi to the llama!"dllama You can use `aiembed` as well. -Find more information [here](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html). \ No newline at end of file +Find more information [here](https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html). + +## Using Together.ai + +You can also use the Together.ai API with PromptingTools.jl. +It requires you to set ENV variable `TOGETHER_API_KEY`. + +The corresponding schema is `TogetherOpenAISchema`, but we have registered one model for you, so you can use it as usual. +Alias "tmixtral" (T for Together.ai and mixtral for the model name) is already set for you. + +```julia +msg = aigenerate("Say hi"; model="tmixtral") +## [ Info: Tokens: 87 @ Cost: \$0.0001 in 5.1 seconds +## AIMessage("Hello! I'm here to help you. Is there something specific you'd like to know or discuss? I can provide information on a wide range of topics, assist with tasks, and even engage in a friendly conversation. Let me know how I can best assist you today.") +``` + +For embedding a text, use `aiembed`: + +```julia +aiembed(PT.TogetherOpenAISchema(), "embed me"; model="BAAI/bge-large-en-v1.5") +``` +Note: You can register the model with `PT.register_model!` and use it as usual. + +## Using Fireworks.ai + +You can also use the Fireworks.ai API with PromptingTools.jl. +It requires you to set ENV variable `FIREWORKS_API_KEY`. + +The corresponding schema is `FireworksOpenAISchema`, but we have registered one model for you, so you can use it as usual. +Alias "fmixtral" (F for Fireworks.ai and mixtral for the model name) is already set for you. + +```julia +msg = aigenerate("Say hi"; model="fmixtral") +## [ Info: Tokens: 78 @ Cost: \$0.0001 in 0.9 seconds +## AIMessage("Hello! I'm glad you're here. I'm here to help answer any questions you have to the best of my ability. Is there something specific you'd like to know or discuss? I can assist with a wide range of topics, so feel free to ask me anything!") +``` + +In addition, at the time of writing (23rd Feb 2024), Fireworks is providing access to their new _function calling_ model (fine-tuned Mixtral) **for free**. + +Try it with `aiextract` for structured extraction (model is aliased as `firefunction`): + +```julia +""" +Extract the food from the sentence. Extract any provided adjectives for the food as well. + +Example: "I am eating a crunchy bread." -> Food("bread", ["crunchy"]) +""" +struct Food + name::String + adjectives::Union{Nothing,Vector{String}} +end +prompt = "I just ate a delicious and juicy apple." +msg = aiextract(prompt; return_type=Food, model="firefunction") +msg.content +# Output: Food("apple", ["delicious", "juicy"]) +``` + +For embedding a text, use `aiembed`: + +```julia +aiembed(PT.FireworksOpenAISchema(), "embed me"; model="nomic-ai/nomic-embed-text-v1.5") +``` +Note: You can register the model with `PT.register_model!` and use it as usual. diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md index 228d6a4c2..5f34172c3 100644 --- a/docs/src/frequently_asked_questions.md +++ b/docs/src/frequently_asked_questions.md @@ -166,7 +166,7 @@ There are three ways how you can customize your workflows (especially when you u 2) Register your model and its associated schema (`PT.register_model!(; name="123", schema=PT.OllamaSchema())`). You won't have to specify the schema anymore only the model name. See [Working with Ollama](#working-with-ollama) for more information. 3) Override your default model (`PT.MODEL_CHAT`) and schema (`PT.PROMPT_SCHEMA`). It can be done persistently with Preferences, eg, `PT.set_preferences!("PROMPT_SCHEMA" => "OllamaSchema", "MODEL_CHAT"=>"llama2")`. -## How to have a Multi-turn Conversations? +## How to have Multi-turn Conversations? Let's say you would like to respond back to a model's response. How to do it? diff --git a/src/llm_interface.jl b/src/llm_interface.jl index 639c188b4..5fd913389 100644 --- a/src/llm_interface.jl +++ b/src/llm_interface.jl @@ -146,6 +146,36 @@ Requires two environment variables to be set: """ struct DatabricksOpenAISchema <: AbstractOpenAISchema end +""" + FireworksOpenAISchema + +Schema to call the [Fireworks.ai](https://fireworks.ai/) API. + +Links: +- [Get your API key](https://fireworks.ai/api-keys) +- [API Reference](https://readme.fireworks.ai/reference/createchatcompletion) +- [Available models](https://fireworks.ai/models) + +Requires one environment variables to be set: +- `FIREWORKS_API_KEY`: Your API key +""" +struct FireworksOpenAISchema <: AbstractOpenAISchema end + +""" + TogetherOpenAISchema + +Schema to call the [Together.ai](https://www.together.ai/) API. + +Links: +- [Get your API key](https://api.together.xyz/settings/api-keys) +- [API Reference](https://docs.together.ai/docs/openai-api-compatibility) +- [Available models](https://docs.together.ai/docs/inference-models) + +Requires one environment variables to be set: +- `TOGETHER_API_KEY`: Your API key +""" +struct TogetherOpenAISchema <: AbstractOpenAISchema end + abstract type AbstractOllamaSchema <: AbstractPromptSchema end """ diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 3b35bf48a..764babe0f 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -70,7 +70,8 @@ function OpenAI.build_url(provider::AbstractCustomProvider, api::AbstractString) string(provider.base_url, "/", api) end function OpenAI.auth_header(provider::AbstractCustomProvider, api_key::AbstractString) - OpenAI.auth_header(OpenAI.OpenAIProvider(provider.api_key, + OpenAI.auth_header( + OpenAI.OpenAIProvider(provider.api_key, provider.base_url, provider.api_version), api_key) @@ -165,6 +166,32 @@ function OpenAI.create_chat(schema::MistralOpenAISchema, base_url = url) OpenAI.create_chat(provider, model, conversation; kwargs...) end +function OpenAI.create_chat(schema::FireworksOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String = "https://api.fireworks.ai/inference/v1", + kwargs...) + # Build the corresponding provider object + # try to override provided api_key because the default is OpenAI key + provider = CustomProvider(; + api_key = isempty(FIREWORKS_API_KEY) ? api_key : FIREWORKS_API_KEY, + base_url = url) + OpenAI.create_chat(provider, model, conversation; kwargs...) +end +function OpenAI.create_chat(schema::TogetherOpenAISchema, + api_key::AbstractString, + model::AbstractString, + conversation; + url::String = "https://api.together.xyz/v1", + kwargs...) + # Build the corresponding provider object + # try to override provided api_key because the default is OpenAI key + provider = CustomProvider(; + api_key = isempty(TOGETHER_API_KEY) ? api_key : TOGETHER_API_KEY, + base_url = url) + OpenAI.create_chat(provider, model, conversation; kwargs...) +end function OpenAI.create_chat(schema::DatabricksOpenAISchema, api_key::AbstractString, model::AbstractString, @@ -257,6 +284,28 @@ function OpenAI.create_embeddings(schema::DatabricksOpenAISchema, input = docs, kwargs...) end +function OpenAI.create_embeddings(schema::TogetherOpenAISchema, + api_key::AbstractString, + docs, + model::AbstractString; + url::String = "https://api.together.xyz/v1", + kwargs...) + provider = CustomProvider(; + api_key = isempty(TOGETHER_API_KEY) ? api_key : TOGETHER_API_KEY, + base_url = url) + OpenAI.create_embeddings(provider, docs, model; kwargs...) +end +function OpenAI.create_embeddings(schema::FireworksOpenAISchema, + api_key::AbstractString, + docs, + model::AbstractString; + url::String = "https://api.fireworks.ai/inference/v1", + kwargs...) + provider = CustomProvider(; + api_key = isempty(FIREWORKS_API_KEY) ? api_key : FIREWORKS_API_KEY, + base_url = url) + OpenAI.create_embeddings(provider, docs, model; kwargs...) +end ## Temporary fix -- it will be moved upstream function OpenAI.create_embeddings(provider::AbstractCustomProvider, @@ -316,8 +365,8 @@ function response_to_message(schema::AbstractOpenAISchema, nothing end ## calculate cost - tokens_prompt = resp.response[:usage][:prompt_tokens] - tokens_completion = resp.response[:usage][:completion_tokens] + tokens_prompt = get(resp.response, :usage, Dict(:prompt_tokens => 0))[:prompt_tokens] + tokens_completion = get(resp.response, :usage, Dict(:completion_tokens => 0))[:completion_tokens] cost = call_cost(tokens_prompt, tokens_completion, model_id) ## build AIMessage object msg = MSG(; @@ -434,7 +483,7 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ run_id = Int(rand(Int32)) # remember one run ID ## extract all message msgs = [response_to_message(prompt_schema, AIMessage, choice, r; - time, model_id, run_id, sample_id = i) + time, model_id, run_id, sample_id = i) for (i, choice) in enumerate(r.response[:choices])] ## Order by log probability if available ## bigger is better, keep it last @@ -537,11 +586,12 @@ function aiembed(prompt_schema::AbstractOpenAISchema, model_id; http_kwargs, api_kwargs...) + tokens_prompt = get(r.response, :usage, Dict(:prompt_tokens => 0))[:prompt_tokens] msg = DataMessage(; content = mapreduce(x -> postprocess(x[:embedding]), hcat, r.response[:data]), status = Int(r.status), - cost = call_cost(r.response[:usage][:prompt_tokens], 0, model_id), - tokens = (r.response[:usage][:prompt_tokens], 0), + cost = call_cost(tokens_prompt, 0, model_id), + tokens = (tokens_prompt, 0), elapsed = time) ## Reporting verbose && @info _report_stats(msg, model_id) @@ -773,7 +823,8 @@ aiclassify(:JudgeIsItTrue; function aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; choices::AbstractVector{T} = ["true", "false", "unknown"], api_kwargs::NamedTuple = NamedTuple(), - kwargs...) where {T <: Union{AbstractString, Tuple{<:AbstractString, <:AbstractString}}} + kwargs...) where {T <: + Union{AbstractString, Tuple{<:AbstractString, <:AbstractString}}} ## Encode the choices and the corresponding prompt ## TODO: maybe check the model provided as well? choices_prompt, logit_bias, decode_ids = encode_choices(prompt_schema, choices) @@ -808,8 +859,8 @@ function response_to_message(schema::AbstractOpenAISchema, nothing end ## calculate cost - tokens_prompt = resp.response[:usage][:prompt_tokens] - tokens_completion = resp.response[:usage][:completion_tokens] + tokens_prompt = get(resp.response, :usage, Dict(:prompt_tokens => 0))[:prompt_tokens] + tokens_completion = get(resp.response, :usage, Dict(:completion_tokens => 0))[:completion_tokens] cost = call_cost(tokens_prompt, tokens_completion, model_id) # "Safe" parsing of the response - it still fails if JSON is invalid content = try @@ -987,7 +1038,7 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T run_id = Int(rand(Int32)) # remember one run ID ## extract all message msgs = [response_to_message(prompt_schema, DataMessage, choice, r; - return_type, time, model_id, run_id, sample_id = i) + return_type, time, model_id, run_id, sample_id = i) for (i, choice) in enumerate(r.response[:choices])] ## Order by log probability if available ## bigger is better, keep it last @@ -1144,7 +1195,7 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE run_id = Int(rand(Int32)) # remember one run ID ## extract all message msgs = [response_to_message(prompt_schema, AIMessage, choice, r; - time, model_id, run_id, sample_id = i) + time, model_id, run_id, sample_id = i) for (i, choice) in enumerate(r.response[:choices])] ## Order by log probability if available ## bigger is better, keep it last diff --git a/src/user_preferences.jl b/src/user_preferences.jl index c016b6392..2e104236d 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -149,6 +149,14 @@ _temp = get(ENV, "GOOGLE_API_KEY", "") const GOOGLE_API_KEY::String = @load_preference("GOOGLE_API_KEY", default=_temp); +_temp = get(ENV, "TOGETHER_API_KEY", "") +const TOGETHER_API_KEY::String = @load_preference("TOGETHER_API_KEY", + default=_temp); + +_temp = get(ENV, "FIREWORKS_API_KEY", "") +const FIREWORKS_API_KEY::String = @load_preference("FIREWORKS_API_KEY", + default=_temp); + _temp = get(ENV, "LOCAL_SERVER", "") ## Address of the local server const LOCAL_SERVER::String = @load_preference("LOCAL_SERVER", @@ -267,7 +275,8 @@ end ### Model Aliases # global reference MODEL_ALIASES is defined below -aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", +aliases = merge( + Dict("gpt3" => "gpt-3.5-turbo", "gpt4" => "gpt-4", "gpt4v" => "gpt-4-vision-preview", # 4v is for "4 vision" "gpt4t" => "gpt-4-turbo-preview", # 4t is for "4 turbo" @@ -279,11 +288,17 @@ aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", "oh25" => "openhermes2.5-mistral", "starling" => "starling-lm", "local" => "local-server", - "gemini" => "gemini-pro"), + "gemini" => "gemini-pro", + ## f-mixtral -> Fireworks.ai Mixtral + "fmixtral" => "accounts/fireworks/models/mixtral-8x7b-instruct", + "firefunction" => "accounts/fireworks/models/firefunction-v1", + ## t-mixtral -> Together.ai Mixtral + "tmixtral" => "mistralai/Mixtral-8x7B-Instruct-v0.1"), ## Load aliases from preferences as well @load_preference("MODEL_ALIASES", default=Dict{String, String}())) -registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", +registry = Dict{String, ModelSpec}( + "gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", OpenAISchema(), 0.5e-6, 1.5e-6, @@ -389,9 +404,10 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", "Mistral AI's hosted model for embeddings."), "echo" => ModelSpec("echo", TestEchoOpenAISchema(; - response = Dict(:choices => [ + response = Dict( + :choices => [ Dict(:message => Dict(:content => "Hello!"), - :finish_reason => "stop"), + :finish_reason => "stop") ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, @@ -408,7 +424,25 @@ registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", GoogleSchema(), 0.0, #unknown, expected 1.25e-7 0.0, #unknown, expected 3.75e-7 - "Gemini Pro is a LLM from Google. For more information, see [models](https://ai.google.dev/models/gemini).")) + "Gemini Pro is a LLM from Google. For more information, see [models](https://ai.google.dev/models/gemini)."), + "accounts/fireworks/models/mixtral-8x7b-instruct" => ModelSpec( + "accounts/fireworks/models/mixtral-8x7b-instruct", + FireworksOpenAISchema(), + 4e-7, #unknown, expected 1.25e-7 + 1.6e-6, #unknown, expected 3.75e-7 + "Mixtral (8x7b) from Mistral, hosted by Fireworks.ai. For more information, see [models](https://fireworks.ai/models/fireworks/mixtral-8x7b-instruct)."), + "accounts/fireworks/models/firefunction-v1" => ModelSpec( + "accounts/fireworks/models/firefunction-v1", + FireworksOpenAISchema(), + 0.0, #unknown, expected to be the same as Mixtral + 0.0, #unknown, expected to be the same as Mixtral + "Fireworks' open-source function calling model (fine-tuned Mixtral). Useful for `aiextract` calls. For more information, see [models](https://fireworks.ai/models/fireworks/firefunction-v1)."), + "mistralai/Mixtral-8x7B-Instruct-v0.1" => ModelSpec( + "mistralai/Mixtral-8x7B-Instruct-v0.1", + TogetherOpenAISchema(), + 6e-7, + 6e-7, + "Mixtral (8x7b) from Mistral, hosted by Together.ai. For more information, see [models](https://docs.together.ai/docs/inference-models).")) ### Model Registry Structure @kwdef mutable struct ModelRegistry From f835a177dda0b5e44d01a603446ce762f57a7e05 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 26 Feb 2024 20:37:06 +0000 Subject: [PATCH 123/251] Add airetry! (#82) --- CHANGELOG.md | 11 +- Project.toml | 10 +- README.md | 85 +++ docs/src/examples/readme_examples.md | 83 +++ src/Experimental/AgentTools/AgentTools.jl | 12 + src/Experimental/AgentTools/lazy_types.jl | 213 +++++-- src/Experimental/AgentTools/mcts.jl | 239 ++++++++ src/Experimental/AgentTools/retry.jl | 533 ++++++++++++++++++ src/Experimental/AgentTools/utils.jl | 95 ++++ src/Experimental/RAGTools/preparation.jl | 10 +- src/Experimental/RAGTools/types.jl | 3 + src/llm_openai.jl | 66 ++- src/llm_shared.jl | 2 +- src/user_preferences.jl | 61 +- .../feedback/FeedbackFromEvaluator.json | 1 + test/Experimental/AgentTools/lazy_types.jl | 31 + test/Experimental/AgentTools/mcts.jl | 198 +++++++ test/Experimental/AgentTools/retry.jl | 184 ++++++ test/Experimental/AgentTools/runtests.jl | 3 + test/Experimental/AgentTools/utils.jl | 182 ++++++ test/llm_openai.jl | 2 +- test/runtests.jl | 1 + 22 files changed, 1929 insertions(+), 96 deletions(-) create mode 100644 src/Experimental/AgentTools/mcts.jl create mode 100644 src/Experimental/AgentTools/retry.jl create mode 100644 templates/agents/feedback/FeedbackFromEvaluator.json create mode 100644 test/Experimental/AgentTools/mcts.jl create mode 100644 test/Experimental/AgentTools/retry.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index faba492f5..d2d831fc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.13.0] + ### Added - Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). - Added a utility to compare any two string sequences (and other iterators)`length_longest_common_subsequence`. It can be used to fuzzy match strings (eg, detecting context/sources in an AI-generated response or fuzzy matching AI response to some preset categories). See the docstring for more information `?length_longest_common_subsequence`. @@ -18,8 +24,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `sample_id` - ID of the sample in the batch if you requested multiple completions, otherwise `sample_id==nothing` (they will have the same `run_id`) - `finish_reason` - the reason why the AI stopped generating the sequence (eg, "stop", "length") to provide more visibility for the user - Support for Fireworks.ai and Together.ai providers for fast and easy access to open-source models. Requires environment variables `FIREWORKS_API_KEY` and `TOGETHER_API_KEY` to be set, respectively. See the `?FireworksOpenAISchema` and `?TogetherOpenAISchema` for more information. +- Added an `extra` field to `ChunkIndex` object for RAG workloads to allow additional flexibility with metadata for each document chunk (assumed to be a vector of the same length as the document chunks). +- Added `airetry` function to `PromptingTools.Experimental.AgentTools` to allow "guided" automatic retries of the AI calls (eg, `AIGenerate` which is the "lazy" counterpart of `aigenerate`) if a given condition fails. It's useful for robustness and reliability in agentic workflows. You can provide conditions as functions and the same holds for feedback to the model as well. See a guessing game example in `?airetry`. -### Fixed +## Updated +- Updated names of endpoints and prices of Mistral.ai models as per the [latest announcement](https://mistral.ai/technology/#models) and [pricing](https://docs.mistral.ai/platform/pricing/). Eg, `mistral-small` -> `mistral-small-latest`. In addition, the latest Mistral model has been added `mistral-large-latest` (aliased as `mistral-large` and `mistrall`, same for the others). `mistral-small-latest` and `mistral-large-latest` now support function calling, which means they will work with `aiextract` (You need to explicitly provide `tool_choice`, see the docs `?aiextract`). ## [0.12.0] diff --git a/Project.toml b/Project.toml index 4d0b7ee39..d774f22bc 100644 --- a/Project.toml +++ b/Project.toml @@ -1,9 +1,10 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.13.0-DEV" +version = "0.13.0" [deps] +AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" @@ -12,6 +13,7 @@ OpenAI = "e9f21f70-7185-4079-aca2-91159181367c" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [weakdeps] @@ -26,6 +28,7 @@ MarkdownPromptingToolsExt = ["Markdown"] RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] [compat] +AbstractTrees = "0.4" Aqua = "0.7" Base64 = "<0.0.1, 1" GoogleGenAI = "0.1.0" @@ -38,7 +41,9 @@ OpenAI = "0.9" Pkg = "<0.0.1, 1" PrecompileTools = "1" Preferences = "1" +Random = "<0.0.1, 1" SparseArrays = "<0.0.1, 1" +Statistics = "<0.0.1, 1" Test = "<0.0.1, 1" julia = "1.9,1.10" @@ -46,6 +51,7 @@ julia = "1.9,1.10" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [targets] -test = ["Aqua", "SparseArrays", "LinearAlgebra", "Markdown"] +test = ["Aqua", "SparseArrays", "Statistics", "LinearAlgebra", "Markdown"] diff --git a/README.md b/README.md index 41ed9bbf8..0b0577cd0 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,10 @@ For more practical examples, see the `examples/` folder and the [Advanced Exampl - [Model Aliases](#model-aliases) - [Embeddings](#embeddings) - [Classification](#classification) + - [Routing to Defined Categories](#routing-to-defined-categories) - [Data Extraction](#data-extraction) - [OCR and Image Comprehension](#ocr-and-image-comprehension) + - [Experimental Agent Workflows / Output Validation with `airetry!`](#experimental-agent-workflows--output-validation-with-airetry) - [Using Ollama models](#using-ollama-models) - [Using MistralAI API and other OpenAI-compatible APIs](#using-mistralai-api-and-other-openai-compatible-apis) - [More Examples](#more-examples) @@ -295,6 +297,26 @@ In the above example, we used a prompt template `:JudgeIsItTrue`, which automati For more information on templates, see the [Templated Prompts](#templated-prompts) section. +### Routing to Defined Categories + +`aiclassify` can be also used for classification into a set of defined categories (maximum 20), so we can use it for routing. + +In addition, if you provide the choices as tuples (`(label, description)`), the model will use the descriptions to decide, but it will return the labels. + +Example: +```julia +choices = [("A", "any animal or creature"), ("P", "for any plant or tree"), ("O", "for everything else")] + +input = "spider" +aiclassify(:InputClassifier; choices, input) # -> returns "A" for any animal or creature + +# Try also with: +input = "daphodil" # -> returns "P" for any plant or tree +input = "castle" # -> returns "O" for everything else +``` + +Under the hood, we use the "logit bias" trick to force only 1 generated token - that means it's very cheap and very fast! + ### Data Extraction Are you tired of extracting data with regex? You can use LLMs to extract structured data from text! @@ -374,6 +396,69 @@ using Markdown msg.content |> Markdown.parse ``` +## Experimental Agent Workflows / Output Validation with `airetry!` + +This is an experimental feature, so you have to import it explicitly: +```julia +using PromptingTools.Experimental.AgentTools +``` + +This module offers "lazy" counterparts to the `ai...` functions, so you can use them in a more controlled way, eg, `aigenerate` -> `AIGenerate` (notice the CamelCase), which has exactly the same arguments except it generates only when `run!` is called. + +For example: +```julia +out = AIGenerate("Say hi!"; model="gpt4t") +run!(out) +``` + +How is it useful? We can use the same "inputs" for repeated calls, eg, when we want to validate +or regenerate some outputs. We have a function `airetry` to help us with that. + +The signature of `airetry` is `airetry(condition_function, aicall::AICall, feedback_function)`. +It evaluates the condition `condition_function` on the `aicall` object (eg, we evaluate `f_cond(aicall) -> Bool`). If it fails, we call `feedback_function` on the `aicall` object to provide feedback for the AI model (eg, `f_feedback(aicall) -> String`) and repeat the process until it passes or until `max_retries` value is exceeded. + +We can catch API failures (no feedback needed, so none is provided) +```julia +# API failure because of a non-existent model +# RetryConfig allows us to change the "retry" behaviour of any lazy call +out = AIGenerate("say hi!"; config = RetryConfig(; catch_errors = true), + model = "NOTEXIST") +run!(out) # fails + +# we ask to wait 2s between retries and retry 2 times (can be set in `config` in aicall as well) +airetry!(isvalid, out; retry_delay = 2, max_retries = 2) +``` + +Or we can validate some outputs (eg, its format, its content, etc.) + +We'll play a color guessing game (I'm thinking "yellow"): + +```julia +# Notice that we ask for two samples (`n_samples=2`) at each attempt (to improve our chances). +# Both guesses are scored at each time step, and the best one is chosen for the next step. +# And with OpenAI, we can set `api_kwargs = (;n=2)` to get both samples simultaneously (cheaper and faster)! +out = AIGenerate( + "Guess what color I'm thinking. It could be: blue, red, black, white, yellow. Answer with 1 word only"; + verbose = false, + config = RetryConfig(; n_samples = 2), api_kwargs = (; n = 2)) +run!(out) + +## Check that the output is 1 word only, third argument is the feedback that will be provided if the condition fails +## Notice: functions operate on `aicall` as the only argument. We can use utilities like `last_output` and `last_message` to access the last message and output in the conversation. +airetry!(x -> length(split(last_output(x), r" |\\.")) == 1, out, + "You must answer with 1 word only.") + +# Note: you could also use the do-syntax, eg, +airetry!(out, "You must answer with 1 word only.") do aicall + length(split(last_output(aicall), r" |\\.")) == 1 +end +``` + +You can place multiple `airetry!` calls in a sequence. They will keep retrying until they run out of maximum AI calls allowed (`max_calls`) or maximum retries (`max_retries`). + +See the docs for more complex examples and usage tips (`?airetry`). +We leverage Monte Carlo Tree Search (MCTS) to optimize the sequence of retries, so it's a very powerful tool for building robust AI workflows (inspired by [Language Agent Tree Search paper](https://arxiv.org/abs/2310.04406) and by [DSPy Assertions paper](https://arxiv.org/abs/2312.13382)). + ### Using Ollama models [Ollama.ai](https://ollama.ai/) is an amazingly simple tool that allows you to run several Large Language Models (LLM) on your computer. It's especially suitable when you're working with some sensitive data that should not be sent anywhere. diff --git a/docs/src/examples/readme_examples.md b/docs/src/examples/readme_examples.md index 5ec371b73..08925e101 100644 --- a/docs/src/examples/readme_examples.md +++ b/docs/src/examples/readme_examples.md @@ -180,6 +180,26 @@ In the above example, we used a prompt template `:JudgeIsItTrue`, which automati For more information on templates, see the [Templated Prompts](#templated-prompts) section. +### Routing to Defined Categories + +`aiclassify` can be also used for classification into a set of defined categories (maximum 20), so we can use it for routing. + +In addition, if you provide the choices as tuples (`(label, description)`), the model will use the descriptions to decide, but it will return the labels. + +Example: +```julia +choices = [("A", "any animal or creature"), ("P", "for any plant or tree"), ("O", "for everything else")] + +input = "spider" +aiclassify(:InputClassifier; choices, input) # -> returns "A" for any animal or creature + +# Try also with: +input = "daphodil" # -> returns "P" for any plant or tree +input = "castle" # -> returns "O" for everything else +``` + +Under the hood, we use the "logit bias" trick to force only 1 generated token - that means it's very cheap and very fast! + ## Data Extraction Are you tired of extracting data with regex? You can use LLMs to extract structured data from text! @@ -259,6 +279,69 @@ using Markdown msg.content |> Markdown.parse ``` +## Experimental Agent Workflows / Output Validation with `airetry!` + +This is an experimental feature, so you have to import it explicitly: +```julia +using PromptingTools.Experimental.AgentTools +``` + +This module offers "lazy" counterparts to the `ai...` functions, so you can use them in a more controlled way, eg, `aigenerate` -> `AIGenerate` (notice the CamelCase), which has exactly the same arguments except it generates only when `run!` is called. + +For example: +```julia +out = AIGenerate("Say hi!"; model="gpt4t") +run!(out) +``` + +How is it useful? We can use the same "inputs" for repeated calls, eg, when we want to validate +or regenerate some outputs. We have a function `airetry` to help us with that. + +The signature of `airetry` is `airetry(condition_function, aicall::AICall, feedback_function)`. +It evaluates the condition `condition_function` on the `aicall` object (eg, we evaluate `f_cond(aicall) -> Bool`). If it fails, we call `feedback_function` on the `aicall` object to provide feedback for the AI model (eg, `f_feedback(aicall) -> String`) and repeat the process until it passes or until `max_retries` value is exceeded. + +We can catch API failures (no feedback needed, so none is provided) +```julia +# API failure because of a non-existent model +# RetryConfig allows us to change the "retry" behaviour of any lazy call +out = AIGenerate("say hi!"; config = RetryConfig(; catch_errors = true), + model = "NOTEXIST") +run!(out) # fails + +# we ask to wait 2s between retries and retry 2 times (can be set in `config` in aicall as well) +airetry!(isvalid, out; retry_delay = 2, max_retries = 2) +``` + +Or we can validate some outputs (eg, its format, its content, etc.) + +We'll play a color guessing game (I'm thinking "yellow"): + +```julia +# Notice that we ask for two samples (`n_samples=2`) at each attempt (to improve our chances). +# Both guesses are scored at each time step, and the best one is chosen for the next step. +# And with OpenAI, we can set `api_kwargs = (;n=2)` to get both samples simultaneously (cheaper and faster)! +out = AIGenerate( + "Guess what color I'm thinking. It could be: blue, red, black, white, yellow. Answer with 1 word only"; + verbose = false, + config = RetryConfig(; n_samples = 2), api_kwargs = (; n = 2)) +run!(out) + +## Check that the output is 1 word only, third argument is the feedback that will be provided if the condition fails +## Notice: functions operate on `aicall` as the only argument. We can use utilities like `last_output` and `last_message` to access the last message and output in the conversation. +airetry!(x -> length(split(last_output(x), r" |\\.")) == 1, out, + "You must answer with 1 word only.") + +# Note: you could also use the do-syntax, eg, +airetry!(out, "You must answer with 1 word only.") do aicall + length(split(last_output(aicall), r" |\\.")) == 1 +end +``` + +You can place multiple `airetry!` calls in a sequence. They will keep retrying until they run out of maximum AI calls allowed (`max_calls`) or maximum retries (`max_retries`). + +See the docs for more complex examples and usage tips (`?airetry`). +We leverage Monte Carlo Tree Search (MCTS) to optimize the sequence of retries, so it's a very powerful tool for building robust AI workflows (inspired by [Language Agent Tree Search paper](https://arxiv.org/abs/2310.04406) and by [DSPy Assertions paper](https://arxiv.org/abs/2312.13382)). + ## Using Ollama models [Ollama.ai](https://ollama.ai/) is an amazingly simple tool that allows you to run several Large Language Models (LLM) on your computer. It's especially suitable when you're working with some sensitive data that should not be sent anywhere. diff --git a/src/Experimental/AgentTools/AgentTools.jl b/src/Experimental/AgentTools/AgentTools.jl index f6c847d0b..72d717111 100644 --- a/src/Experimental/AgentTools/AgentTools.jl +++ b/src/Experimental/AgentTools/AgentTools.jl @@ -9,15 +9,27 @@ module AgentTools using PromptingTools const PT = PromptingTools +using AbstractTrees +using AbstractTrees: print_tree, PreOrderDFS, PostOrderDFS +using Random using Test +export print_tree, PreOrderDFS, PostOrderDFS include("utils.jl") +export print_samples, find_node +include("mcts.jl") + export aicodefixer_feedback, error_feedback, score_feedback include("code_feedback.jl") export AICall, AIGenerate, AIExtract, AIEmbed, AIClassify, AIScan +export RetryConfig, last_output, last_message export AICodeFixer, run! include("lazy_types.jl") +export airetry! +# export add_feedback!, evaluate_condition! +include("retry.jl") + end diff --git a/src/Experimental/AgentTools/lazy_types.jl b/src/Experimental/AgentTools/lazy_types.jl index 6201080e4..d7aac7136 100644 --- a/src/Experimental/AgentTools/lazy_types.jl +++ b/src/Experimental/AgentTools/lazy_types.jl @@ -1,5 +1,52 @@ +# The following implements lazy types for all ai* functions (eg, aigenerate -> AIGenerate) and AICodeFixer + +abstract type AbstractAIPrompter end abstract type AICallBlock end +""" + RetryConfig + +Configuration for self-fixing the AI calls. It includes the following fields: + +# Fields +- `retries::Int`: The number of retries ("fixing rounds") that have been attempted so far. +- `calls::Int`: The total number of SUCCESSFULLY generated ai* function calls made so far (across all samples/retry rounds). + Ie, if a call fails, because of an API error, it's not counted, because it didn't reach the LLM. +- `max_retries::Int`: The maximum number of retries ("fixing rounds") allowed for the AI call. Defaults to 10. +- `max_calls::Int`: The maximum number of ai* function calls allowed for the AI call. Defaults to 99. +- `retry_delay::Int`: The delay (in seconds) between retry rounds. Defaults to 0s. +- `n_samples::Int`: The number of samples to generate in each ai* call round (to increase changes of successful pass). Defaults to 1. +- `scoring::AbstractScoringMethod`: The scoring method to use for generating multiple samples. Defaults to `UCT(sqrt(2))`. +- `ordering::Symbol`: The ordering to use for select the best samples. With `:PostOrderDFS` we prioritize leaves, with `:PreOrderDFS` we prioritize the root. Defaults to `:PostOrderDFS`. +- `feedback_inplace::Bool`: Whether to provide feedback in previous UserMessage (and remove the past AIMessage) or to create a new UserMessage. Defaults to `false`. +- `feedback_template::Symbol`: Template to use for feedback in place. Defaults to `:FeedbackFromEvaluator`. +- `temperature::Float64`: The temperature to use for sampling. Relevant only if not defined in `api_kwargs` provided. Defaults to 0.7. +- `catch_errors::Bool`: Whether to catch errors during `run!` of AICall. Saves them in `aicall.error`. Defaults to `false`. +""" +@kwdef mutable struct RetryConfig + retries::Int = 0 + calls::Int = 0 + max_retries::Int = 10 + max_calls::Int = 99 + retry_delay::Int = 0 + n_samples::Int = 1 + scoring::AbstractScoringMethod = UCT() + ordering::Symbol = :PostOrderDFS + feedback_inplace::Bool = false + feedback_template::Symbol = :FeedbackFromEvaluator # Template to use for feedback + temperature::Float64 = 0.7 + catch_errors::Bool = false +end +function Base.show(io::IO, config::RetryConfig) + dump(IOContext(io, :limit => true), config, maxdepth = 1) +end +function Base.copy(config::RetryConfig) + return deepcopy(config) +end +function Base.var"=="(c1::RetryConfig, c2::RetryConfig) + all(f -> getfield(c1, f) == getfield(c2, f), fieldnames(typeof(c1))) +end + """ AICall(func::F, args...; kwargs...) where {F<:Function} @@ -59,34 +106,20 @@ This can be used to "reply" to previous message / continue the stored conversati schema::Union{Nothing, PT.AbstractPromptSchema} = nothing conversation::Vector{<:PT.AbstractMessage} = Vector{PT.AbstractMessage}() kwargs::NamedTuple = NamedTuple() - success::Union{Nothing, Bool} = nothing + success::Union{Nothing, Bool} = nothing # success of the last call - in different airetry checks etc error::Union{Nothing, Exception} = nothing + ## by default, we use samples to hold the conversation attempts across different fixing rounds + samples::SampleNode = SampleNode(; data = Vector{PT.AbstractMessage}()) + active_sample_id::Int = -1 + memory::Dict{Symbol, Any} = Dict{Symbol, Any}() + config::RetryConfig = RetryConfig() # Configuration for retries + prompter::Union{Nothing, AbstractAIPrompter} = nothing end -## main sample -## samples function AICall(func::F, args...; kwargs...) where {F <: Function} - @assert length(args)<=2 "AICall takes at most 2 positional arguments (provided: $(length(args)))" - schema = nothing - conversation = Vector{PT.AbstractMessage}() - for arg in args - if isa(arg, PT.AbstractPromptSchema) - schema = arg - elseif isa(arg, Vector{<:PT.AbstractMessage}) - conversation = arg - elseif isa(arg, AbstractString) && isempty(conversation) - ## User Prompt -- create a UserMessage - push!(conversation, PT.UserMessage(arg)) - elseif isa(arg, Symbol) && isempty(conversation) - conversation = PT.render(schema, AITemplate(arg)) - elseif isa(arg, AITemplate) && isempty(conversation) - conversation = PT.render(schema, arg) - else - error("Invalid argument type: $(typeof(arg))") - end - end - - return AICall{F}(; func, schema, conversation, kwargs = NamedTuple(kwargs)) + schema, conversation = unwrap_aicall_args(args) + kwargs, config = extract_config(kwargs, RetryConfig()) + return AICall{F}(; func, schema, conversation, config, kwargs) end """ @@ -159,10 +192,12 @@ end Executes the AI call wrapped by an `AICallBlock` instance. This method triggers the actual communication with the AI model and processes the response based on the provided conversation context and parameters. +Note: Currently `return_all` must always be set to true. + # Arguments - `aicall::AICallBlock`: An instance of `AICallBlock` which encapsulates the AI function call along with its context and parameters (eg, `AICall`, `AIGenerate`) -- `verbose::Int=1`: A verbosity level for logging. A higher value indicates more detailed logging. -- `catch_errors::Bool=false`: If set to `true`, the method will catch and handle errors internally. Otherwise, errors are propagated. +- `verbose::Integer=1`: A verbosity level for logging. A higher value indicates more detailed logging. +- `catch_errors::Union{Nothing, Bool}=nothing`: A flag to indicate whether errors should be caught and saved to `aicall.error`. If `nothing`, it defaults to `aicall.config.catch_errors`. - `return_all::Bool=true`: A flag to indicate whether the whole conversation from the AI call should be returned. It should always be true. - `kwargs...`: Additional keyword arguments that are passed to the AI function. @@ -187,27 +222,84 @@ aicall("Say hi!") - This method is essential for scenarios where AI interactions are based on dynamic or evolving contexts, as it allows for real-time updates and responses based on the latest information. """ function run!(aicall::AICallBlock; - verbose::Int = 1, - catch_errors::Bool = false, + verbose::Integer = 1, + catch_errors::Union{Nothing, Bool} = nothing, return_all::Bool = true, kwargs...) @assert return_all "`return_all` must be true (provided: $return_all)" - (; schema, conversation) = aicall - try - result = if isnothing(schema) - aicall.func(conversation; aicall.kwargs..., kwargs..., return_all) - else - aicall.func(schema, conversation; aicall.kwargs..., kwargs..., return_all) + (; schema, conversation, config) = aicall + (; max_calls, n_samples) = aicall.config + + catch_errors = isnothing(catch_errors) ? config.catch_errors : catch_errors + verbose = min(verbose, get(aicall.kwargs, :verbose, 99)) + + ## Locate the parent node in samples node, if it's the first call, we'll fall back to `aicall.samples` itself + parent_node = find_node(aicall.samples, aicall.active_sample_id) |> + x -> isnothing(x) ? aicall.samples : x + ## Obtain the new API kwargs (if we need to tweak parameters) + new_api_kwargs = merge(get(aicall.kwargs, :api_kwargs, NamedTuple()), + get(kwargs, :api_kwargs, NamedTuple()), + (; temperature = aicall.config.temperature)) + ## Collect n_samples in a loop + ## if API supports it, you can speed it up via `api_kwargs=(; n= n_samples)` to generate them at once + samples_collected = 0 + for i in 1:n_samples + ## Check if we don't need to collect more samples + samples_collected >= n_samples && break + ## Check if we have budget left + if aicall.config.calls >= max_calls + verbose > 0 && + @info "Max calls limit reached (calls: $(aicall.config.calls)). Generation interrupted." + break end + ## We need to set explicit temperature to ensure our calls are not cached + ## (small perturbations in temperature of each request, unless user requested temp=0) + if !iszero(new_api_kwargs.temperature) + new_api_kwargs = merge(new_api_kwargs, + (; temperature = new_api_kwargs.temperature + 1e-3)) + end + ## Call the API with try-catch (eg, catch API errors, bad user inputs, etc.) + try + ## Note: always return all conversation (including prompt) + result = if isnothing(schema) + aicall.func(conversation; aicall.kwargs..., kwargs..., + new_api_kwargs..., return_all = true) + else + aicall.func(schema, conversation; aicall.kwargs..., kwargs..., + new_api_kwargs..., return_all = true) + end + # unpack multiple samples (if present; if not, it will be a single sample in a vector) + conv_list = split_multi_samples(result) + for conv in conv_list + ## save the sample into our sample tree + node = expand!(parent_node, conv; success = true) + aicall.active_sample_id = node.id + aicall.config.calls += 1 + samples_collected += 1 + end + aicall.success = true + catch e + verbose > 0 && @info "Error detected and caught in AICall" + aicall.success = false + aicall.error = e + !catch_errors && rethrow(aicall.error) + end + ## Break the loop - no point in sampling if we get errors + aicall.success == false && break + end + ## Finalize the generaion + if aicall.success == true + ## overwrite the active conversation + current_node = find_node(aicall.samples, aicall.active_sample_id) + aicall.conversation = current_node.data # Remove used kwargs (for placeholders) - aicall.kwargs = remove_used_kwargs(aicall.kwargs, conversation) - aicall.conversation = result - aicall.success = true - catch e - verbose > 0 && @info "Error detected and caught in AICall" - aicall.success = false - aicall.error = e - !catch_errors && rethrow(aicall.error) + aicall.kwargs = remove_used_kwargs(aicall.kwargs, aicall.conversation) + ## If first sample (parent == root node), + ## make sure that root node sample has a conversation to retry from + if current_node.parent.id == aicall.samples.id && isempty(aicall.samples.data) + aicall.samples.data = copy(aicall.conversation) + pop!(aicall.samples.data) # remove the last AI message + end end return aicall end @@ -231,6 +323,38 @@ function (aicall::AICall)(msg::PT.UserMessage; kwargs...) return run!(aicall; kwargs...) end +"Helpful accessor for AICall blocks. Returns the last message in the conversation." +function last_message(aicall::AICallBlock) + length(aicall.conversation) == 0 ? nothing : aicall.conversation[end] +end + +"Helpful accessor for AICall blocks. Returns the last output in the conversation (eg, the string/data in the last message)." +function last_output(aicall::AICallBlock) + msg = last_message(aicall) + return isnothing(msg) ? nothing : msg.content +end + +function Base.isvalid(aicall::AICallBlock) + aicall.success == true +end + +function Base.copy(aicall::AICallBlock) + return AICall{typeof(aicall.func)}(aicall.func, + aicall.schema, + copy(aicall.conversation), + aicall.kwargs, + aicall.success, + aicall.error, + copy(aicall.samples), + aicall.active_sample_id, + copy(aicall.memory), + copy(aicall.config), + aicall.prompter) +end +function Base.var"=="(c1::AICallBlock, c2::AICallBlock) + all(f -> getfield(c1, f) == getfield(c2, f), fieldnames(typeof(c1))) +end + """ AICodeFixer(aicall::AICall, templates::Vector{<:PT.UserMessage}; num_rounds::Int = 3, feedback_func::Function = aicodefixer_feedback; kwargs...) AICodeFixer(aicall::AICall, template::Union{AITemplate, Symbol} = :CodeFixerRCI; kwargs...) @@ -407,3 +531,8 @@ function run!(codefixer::AICodeFixer; return codefixer end + +### Prompt Generators +# Placeholder for future +@kwdef mutable struct AIPrompter +end diff --git a/src/Experimental/AgentTools/mcts.jl b/src/Experimental/AgentTools/mcts.jl new file mode 100644 index 000000000..b582fe83c --- /dev/null +++ b/src/Experimental/AgentTools/mcts.jl @@ -0,0 +1,239 @@ +### Monte Carlo Tree Search +# Lightweight implementation of the Monte Carlo Tree Search algorithm. +# Source: [Wikipedia: Monte Carlo Tree Search](https://en.wikipedia.org/wiki/Monte_Carlo_tree_search) +# Source: [Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models](https://arxiv.org/abs/2310.04406) +# +# Key types: +# - `SampleNode` +# - `UCT` + +abstract type AbstractScoringMethod end + +""" + ThompsonSampling <: AbstractScoringMethod + +Implements scoring and selection for Thompson Sampling method. See https://en.wikipedia.org/wiki/Thompson_sampling for more details. +""" +@kwdef struct ThompsonSampling <: AbstractScoringMethod + alpha::Float64 = 1.0 # Alpha parameter for the Beta distribution + beta::Float64 = 1.0 # Beta parameter for the Beta distribution +end + +""" + UCT <: AbstractScoringMethod + +Implements scoring and selection for UCT (Upper Confidence Bound for Trees) sampling method. See https://en.wikipedia.org/wiki/Monte_Carlo_tree_search#Exploration_and_exploitation for more details. +""" +@kwdef struct UCT <: AbstractScoringMethod + exploration::Float64 = sqrt(2.0) # Exploration parameter, higher values encourage more exploration +end + +""" + SampleNode{T} + +A node in the Monte Carlo Tree Search tree. + +It's used to hold the `data` we're trying to optimize/discover (eg, a conversation), the scores from evaluation (`wins`, `visits`) and the results of the evaluations upon failure (`feedback`). + +# Fields +- `id::UInt16`: Unique identifier for the node +- `parent::Union{SampleNode, Nothing}`: Parent node that current node was built on +- `children::Vector{SampleNode}`: Children nodes +- `wins::Int`: Number of successful outcomes +- `visits::Int`: Number of condition checks done (eg, losses are `checks - wins`) +- `data::T`: eg, the conversation or some parameter to be optimized +- `feedback::String`: Feedback from the evaluation, always a string! Defaults to empty string. +- `success::Union{Nothing, Bool}`: Success of the generation and subsequent evaluations, proxy for whether it should be further evaluated. Defaults to nothing. +""" +@kwdef mutable struct SampleNode{T} + id::UInt16 = rand(UInt16) + parent::Union{SampleNode, Nothing} = nothing + children::Vector{SampleNode} = SampleNode[] + wins::Int = 0 # Number of successful outcomes + visits::Int = 0 # Number of condition checks done (eg, losses are `checks - wins`) + data::T # eg, the conversation or some parameter to be optimized + feedback::String = "" + success::Union{Nothing, Bool} = nothing # succes of the generation/tests, proxy for whether it should be further evaluated +end + +Base.IteratorEltype(::Type{<:TreeIterator{SampleNode}}) = Base.HasEltype() +Base.eltype(::Type{<:TreeIterator{SampleNode{T}}}) where {T} = SampleNode{T} +AbstractTrees.childtype(::Type{SampleNode{T}}) where {T} = SampleNode{T} + +function AbstractTrees.children(node::SampleNode) + return node.children +end +AbstractTrees.parent(n::SampleNode) = n.parent +## AbstractTrees.nodevalue(n::SampleNode) = n.data +function Base.show(io::IO, node::SampleNode; + scoring::Union{Nothing, AbstractScoringMethod} = nothing) + score_str = isnothing(scoring) ? "" : ", score: $(round(score(node, scoring),digits=2))" + length_str = node.data isa AbstractVector ? ", length: $(length(node.data))" : "" + print(io, + "SampleNode(id: $(node.id), stats: $(node.wins)/$(node.visits)$(score_str)$(length_str))") +end +function Base.getindex(node::SampleNode, id::Integer) + find_node(node, id) +end +function Base.var"=="(n1::SampleNode, n2::SampleNode) + all(fieldnames(typeof(n1))) do f + if f == :parent + ## both must have a parent or both must not have a parent + ## if they don't have a parent, they are equal + ## if they have a parent, the parent id must be the same + isnothing(n1.parent) == isnothing(n2.parent) && + (isnothing(n1.parent) || (n1.parent.id == n2.parent.id)) + elseif f == :children + all(x -> x[1].id == x[2].id, zip(n1.children, n2.children)) + else + getfield(n1, f) == getfield(n2, f) + end + end +end +function Base.copy(n::SampleNode) + return deepcopy(n) +end + +"Expands the tree with a new node from `parent` using the given `data` and `success`." +function expand!(parent::SampleNode, data; + success::Union{Nothing, Bool} = true, feedback::String = "") + child = SampleNode(; data, parent, success, feedback) + push!(AbstractTrees.children(parent), child) + return child +end + +"Provides scores for a given node (and all its ancestors) based on the evaluation (`wins`, `visits`)." +function backpropagate!(node::SampleNode; wins::Integer, visits::Int = 1) + # Update current node and all ancestors + while node !== nothing + node.wins += wins + node.visits += visits + # Backprop to parent + node = AbstractTrees.parent(node) + end +end + +function score(node::SampleNode, scoring::AbstractScoringMethod) + throw(ArgumentError("Scoring method not implemented for `score` with $(typeof(scoring))")) +end + +"Scores a node using the UCT (Upper Confidence Bound for Trees) method." +function score(node::SampleNode, scoring::UCT) + parent_node = AbstractTrees.parent(node) + parent_node_score = isnothing(parent_node) || iszero(parent_node.visits) ? 0.0 : + scoring.exploration * sqrt(log(parent_node.visits) / node.visits) + s = iszero(node.visits) ? 0.0 : node.wins / node.visits + parent_node_score +end + +"Scores a node using the ThomsonSampling method, similar to Bandit algorithms." +function score(node::SampleNode, scoring::ThompsonSampling) + (; alpha, beta) = scoring + s = beta_sample(alpha + node.wins, beta + node.visits - node.wins) +end + +""" + select_best(node::SampleNode, scoring::AbstractScoringMethod = UCT(); + ordering::Symbol = :PostOrderDFS) + +Selects the best node from the tree using the given `scoring` (`UCT` or `ThompsonSampling`). Defaults to UCT. +Thompson Sampling is more random with small samples, while UCT stabilizes much quicker thanks to looking at parent nodes as well. + +Ordering can be either `:PreOrderDFS` or `:PostOrderDFS`. Defaults to `:PostOrderDFS`, which favors the leaves (end points of the tree). + +# Example +Compare the different scoring methods: +```julia +# Set up mock samples and scores +data = PT.AbstractMessage[] +root = SampleNode(; data) +child1 = expand!(root, data) +backpropagate!(child1; wins = 1, visits = 1) +child2 = expand!(root, data) +backpropagate!(child2; wins = 0, visits = 1) +child11 = expand!(child1, data) +backpropagate!(child11; wins = 1, visits = 1) + +# Select with UCT +n = select_best(root, UCT()) +SampleNode(id: 29826, stats: 1/1, length: 0) + +# Show the tree: +print_samples(root; scoring = UCT()) +## SampleNode(id: 13184, stats: 2/3, score: 0.67, length: 0) +## ├─ SampleNode(id: 26078, stats: 2/2, score: 2.05, length: 0) +## │ └─ SampleNode(id: 29826, stats: 1/1, score: 2.18, length: 0) +## └─ SampleNode(id: 39931, stats: 0/1, score: 1.48, length: 0) + +# Select with ThompsonSampling - much more random with small samples +n = select_best(root, ThompsonSampling()) +SampleNode(id: 26078, stats: 2/2, length: 0) + +# Show the tree (run it a few times and see how the scores jump around): +print_samples(root; scoring = ThompsonSampling()) +## SampleNode(id: 13184, stats: 2/3, score: 0.6, length: 0) +## ├─ SampleNode(id: 26078, stats: 2/2, score: 0.93, length: 0) +## │ └─ SampleNode(id: 29826, stats: 1/1, score: 0.22, length: 0) +## └─ SampleNode(id: 39931, stats: 0/1, score: 0.84, length: 0) + +``` +""" +function select_best(node::SampleNode, scoring::AbstractScoringMethod = UCT(); + ordering::Symbol = :PostOrderDFS) + @assert ordering in (:PreOrderDFS, :PostOrderDFS) "Only PreOrderDFS and PostOrderDFS are supported for `ordering` (provided: $ordering)." + best_val = -Inf + best_node = nothing + if ordering == :PreOrderDFS + for n in AbstractTrees.PreOrderDFS(node) + val = score(n, scoring) + if val > best_val + best_val = val + best_node = n + end + end + elseif ordering == :PostOrderDFS + for n in AbstractTrees.PostOrderDFS(node) + val = score(n, scoring) + if val > best_val + best_val = val + best_node = n + end + end + end + return best_node +end + +"Finds a node with a given `id` in the tree starting from `node`." +function find_node(node::SampleNode, id::Integer) + for n in AbstractTrees.PreOrderDFS(node) + if n.id == id + return n + end + end + return nothing +end + +"Pretty prints the samples tree starting from `node`. Usually, `node` is the root of the tree. Example: `print_samples(aicall.samples)`." +function print_samples(node::SampleNode; scoring::AbstractScoringMethod = UCT()) + print_samples(stdout, node; scoring) +end +function print_samples(io::IO, node::SampleNode; scoring::AbstractScoringMethod = UCT()) + print_tree(show, io, node; printnode_kw = (; scoring)) +end + +"Sets the `success` field of all nodes in the tree to `success` value." +function reset_success!(node::SampleNode, success::Bool = true) + for n in AbstractTrees.PreOrderDFS(node) + n.success = success + end + return nothing +end + +"Collects all feedback from the node and its ancestors (parents). Returns a string separated by `separator`." +function collect_all_feedback(node::SampleNode; separator::String = "\n$("-"^10)\n") + feedback = String[] + while node !== nothing + !isempty(node.feedback) && push!(feedback, node.feedback) + node = AbstractTrees.parent(node) + end + return join(reverse(feedback), separator) +end diff --git a/src/Experimental/AgentTools/retry.jl b/src/Experimental/AgentTools/retry.jl new file mode 100644 index 000000000..7e2099258 --- /dev/null +++ b/src/Experimental/AgentTools/retry.jl @@ -0,0 +1,533 @@ +""" + airetry!( + f_cond::Function, aicall::AICallBlock, feedback::Union{AbstractString, Function} = ""; + verbose::Bool = true, throw::Bool = false, evaluate_all::Bool = true, feedback_expensive::Bool = false, + max_retries::Union{Nothing, Int} = nothing, retry_delay::Union{Nothing, Int} = nothing) + +Evaluates the condition `f_cond` on the `aicall` object (eg, we evaluate `f_cond(aicall) -> Bool`). +If the condition is not met, it will return the best sample to retry from and provide `feedback` to `aicall`. That's why it's mutating. +It will retry running the `aicall` `max_retries` times. +If `throw` is `true`, it will throw an error if the function does not return `true` after `max_retries` retries. + +If feedback is provided (not empty), it will be append it to the conversation before the retry. +If a function is provided, it must accept the `aicall` object as the only argument and return a string. + +Function `f_cond` is expected to accept the `aicall` object as the only argument. +It must return a boolean value, which indicates whether the condition is met. +You can leverage the `last_message`, `last_output`, and `AICode` functions to access the last message, last output and code blocks in the conversation, respectively. + +# Good Use Cases +- Retry with API failures/drops (add `retry_delay=2` to wait 2s between retries) +- Check the output format / type / length / etc +- Check the output with `aiclassify` call (LLM Judge) to catch unsafe/NSFW/out-of-scope content +- Provide hints to the model to guide it to the correct answer + +# Gotchas +- If controlling keyword arguments are set to nothing, they will fall back to the default values in `aicall.config`. You can override them by passing the keyword arguments explicitly. +- If there multiple `airetry!` checks, they are evaluted sequentially. As long as `throw==false`, they will be all evaluated even if they failed previous checks. +- Only samples which passed previous evaluations are evaluated (`sample.success` is `true`). If there are no successful samples, the function will evaluate only the active sample (`aicall.active_sample_id`) and nothing else. +- Feedback from all "ancestor" evaluations is added upon retry, not feedback from the "sibblings" or other branches. To have only ONE long BRANCH (no sibblings), make sure to keep `RetryConfig(; n_samples=1)`. + That way the model will always see ALL previous feedback. +- We implement a version of Monte Carlo Tree Search (MCTS) to always pick the most promising sample to restart from (you can tweak the options in `RetryConfig` to change the behaviour). +- For large number of parallel branches (ie, "shallow and wide trees"), you might benefit from switching scoring to `scoring=ThompsonSampling()` (similar to how Bandit algorithms work). +- Open-source/local models can struggle with too long conversation, you might want to experiment with `in-place feedback` (set `RetryConfig(; feedback_inplace=true)`). + + +# Arguments +- `f_cond::Function`: A function that accepts the `aicall` object and returns a boolean value. Retry will be attempted if the condition is not met (`f_cond -> false`). +- `aicall::AICallBlock`: The `aicall` object to evaluate the condition on. +- `feedback::Union{AbstractString, Function}`: Feedback to provide if the condition is not met. If a function is provided, it must accept the `aicall` object as the only argument and return a string. +- `verbose::Integer=1`: A verbosity level for logging the retry attempts and warnings. A higher value indicates more detailed logging. +- `throw::Bool=false`: If true, it will throw an error if the function `f_cond` does not return `true` after `max_retries` retries. +- `evaluate_all::Bool=false`: If true, it will evaluate all the "successful" samples in the `aicall` object. Otherwise, it will only evaluate the active sample. +- `feedback_expensive::Bool=false`: If false, it will provide feedback to all samples that fail the condition. + If `feedback` function is expensive to call (eg, another ai* function), set this to `true` and feedback will be provided only to the sample we will retry from. +- `max_retries::Union{Nothing, Int}=nothing`: Maximum number of retries. If not provided, it will fall back to the `max_retries` in `aicall.config`. +- `retry_delay::Union{Nothing, Int}=nothing`: Delay between retries in seconds. If not provided, it will fall back to the `retry_delay` in `aicall.config`. + +# Returns +- The `aicall` object with the updated `conversation`, and `samples` (saves the evaluations and their scores/feedback). + +# Example + +You can use `airetry!` to catch API errors in `run!` and auto-retry the call. +`RetryConfig` is how you influence all the subsequent retry behaviours - see `?RetryConfig` for more details. +```julia +# API failure because of a non-existent model +out = AIGenerate("say hi!"; config = RetryConfig(; catch_errors = true), + model = "NOTEXIST") +run!(out) # fails + +# we ask to wait 2s between retries and retry 2 times (can be set in `config` in aicall as well) +airetry!(isvalid, out; retry_delay = 2, max_retries = 2) +``` + +If you provide arguments to the aicall, we try to honor them as much as possible in the following calls, +eg, set low verbosity +```julia +out = AIGenerate("say hi!"; config = RetryConfig(; catch_errors = true), +model = "NOTEXIST", verbose=false) +run!(out) +# No info message, you just see `success = false` in the properties of the AICall +``` + +Let's show a toy example to demonstrate the runtime checks / guardrails for the model output. +We'll play a color guessing game (I'm thinking "yellow"): + +```julia +# Notice that we ask for two samples (`n_samples=2`) at each attempt (to improve our chances). +# Both guesses are scored at each time step, and the best one is chosen for the next step. +# And with OpenAI, we can set `api_kwargs = (;n=2)` to get both samples simultaneously (cheaper and faster)! +out = AIGenerate( + "Guess what color I'm thinking. It could be: blue, red, black, white, yellow. Answer with 1 word only"; + verbose = false, + config = RetryConfig(; n_samples = 2), api_kwargs = (; n = 2)) +run!(out) + +## Check that the output is 1 word only, third argument is the feedback that will be provided if the condition fails +## Notice: functions operate on `aicall` as the only argument. We can use utilities like `last_output` and `last_message` to access the last message and output in the conversation. +airetry!(x -> length(split(last_output(x), r" |\\.")) == 1, out, + "You must answer with 1 word only.") + +## Let's ensure that the output is in lowercase - simple and short +airetry!(x -> all(islowercase, last_output(x)), out, "You must answer in lowercase.") +# [ Info: Condition not met. Retrying... + +## Let's add final hint - it took us 2 retries +airetry!(x -> startswith(last_output(x), "y"), out, "It starts with \"y\"") +# [ Info: Condition not met. Retrying... +# [ Info: Condition not met. Retrying... + +## We end up with the correct answer +last_output(out) +# Output: "yellow" +``` + +Let's explore how we got here. +We save the various attempts in a "tree" (SampleNode object) +You can access it in `out.samples`, which is the ROOT of the tree (top level). +Currently "active" sample ID is `out.active_sample_id` -> that's the same as `conversation` field in your AICall. + +```julia +# Root node: +out.samples +# Output: SampleNode(id: 46839, stats: 6/12, length: 2) + +# Active sample (our correct answer): +out.active_sample_id +# Output: 50086 + +# Let's obtain the active sample node with this ID - use getindex notation or function find_node +out.samples[out.active_sample_id] +# Output: SampleNode(id: 50086, stats: 1/1, length: 7) + +# The SampleNode has two key fields: data and feedback. Data is where the conversation is stored: +active_sample = out.samples[out.active_sample_id] +active_sample.data == out.conversation # Output: true -> This is the winning guess! +``` + +We also get a clear view of the tree structure of all samples with `print_samples`: +```julia +julia> print_samples(out.samples) +SampleNode(id: 46839, stats: 6/12, score: 0.5, length: 2) +├─ SampleNode(id: 12940, stats: 5/8, score: 1.41, length: 4) +│ ├─ SampleNode(id: 34315, stats: 3/4, score: 1.77, length: 6) +│ │ ├─ SampleNode(id: 20493, stats: 1/1, score: 2.67, length: 7) +│ │ └─ SampleNode(id: 50086, stats: 1/1, score: 2.67, length: 7) +│ └─ SampleNode(id: 2733, stats: 1/2, score: 1.94, length: 5) +└─ SampleNode(id: 48343, stats: 1/4, score: 1.36, length: 4) + ├─ SampleNode(id: 30088, stats: 0/1, score: 1.67, length: 5) + └─ SampleNode(id: 44816, stats: 0/1, score: 1.67, length: 5) +``` + +You can use the `id` to grab and inspect any of these nodes, eg, +```julia +out.samples[2733] +# Output: SampleNode(id: 2733, stats: 1/2, length: 5) +``` + +We can also iterate through all samples and extract whatever information we want with `PostOrderDFS` or `PreOrderDFS` (exported from AbstractTrees.jl) +```julia +for sample in PostOrderDFS(out.samples) + # Data is the universal field for samples, we put `conversation` in there + # Last item in data is the last message in coversation + msg = sample.data[end] + if msg isa PT.AIMessage # skip feedback + # get only the message content, ie, the guess + println("ID: \$(sample.id), Answer: \$(msg.content)") + end +end + +# ID: 20493, Answer: yellow +# ID: 50086, Answer: yellow +# ID: 2733, Answer: red +# ID: 30088, Answer: blue +# ID: 44816, Answer: blue +``` + +Note: `airetry!` will attempt to fix the model `max_retries` times. +If you set `throw=true`, it will throw an ErrorException if the condition is not met after `max_retries` retries. + + +```julia +# Let's define a mini program to guess the number +\"\"\" + llm_guesser() + +Mini program to guess the number provided by the user (betwee 1-100). +\"\"\" +function llm_guesser(user_number::Int) + @assert 1 <= user_number <= 100 + prompt = \"\"\" +I'm thinking a number between 1-100. Guess which one it is. +You must respond only with digits and nothing else. +Your guess:\"\"\" + ## 2 samples at a time, max 5 fixing rounds + out = AIGenerate(prompt; config = RetryConfig(; n_samples = 2, max_retries = 5), + api_kwargs = (; n = 2)) |> run! + ## Check the proper output format - must parse to Int, use do-syntax + ## We can provide feedback via a function! + function feedback_f(aicall) + "Output: \$(last_output(aicall))\nFeedback: You must respond only with digits!!" + end + airetry!(out, feedback_f) do aicall + !isnothing(tryparse(Int, last_output(aicall))) + end + ## Give a hint on bounds + lower_bound = (user_number ÷ 10) * 10 + upper_bound = lower_bound + 10 + airetry!( + out, "The number is between or equal to \$lower_bound to \$upper_bound.") do aicall + guess = tryparse(Int, last_output(aicall)) + lower_bound <= guess <= upper_bound + end + ## You can make at most 3x guess now -- if there is max_retries in `config.max_retries` left + max_retries = out.config.retries + 3 + function feedback_f2(aicall) + guess = tryparse(Int, last_output(aicall)) + "Your guess of \$(guess) is wrong, it's \$(abs(guess-user_number)) numbers away." + end + airetry!(out, feedback_f2; max_retries) do aicall + tryparse(Int, last_output(aicall)) == user_number + end + + ## Evaluate the best guess + @info "Results: Guess: \$(last_output(out)) vs User: \$user_number (Number of calls made: \$(out.config.calls))" + return out +end + +# Let's play the game +out = llm_guesser(33) +[ Info: Condition not met. Retrying... +[ Info: Condition not met. Retrying... +[ Info: Condition not met. Retrying... +[ Info: Condition not met. Retrying... +[ Info: Results: Guess: 33 vs User: 33 (Number of calls made: 10) +``` +Yay! We got it :) + +Now, we could explore different samples (eg, `print_samples(out.samples)`) or see what the model guessed at each step: +```julia +print_samples(out.samples) +## SampleNode(id: 57694, stats: 6/14, score: 0.43, length: 2) +## ├─ SampleNode(id: 35603, stats: 5/10, score: 1.23, length: 4) +## │ ├─ SampleNode(id: 55394, stats: 1/4, score: 1.32, length: 6) +## │ │ ├─ SampleNode(id: 20737, stats: 0/1, score: 1.67, length: 7) +## │ │ └─ SampleNode(id: 52910, stats: 0/1, score: 1.67, length: 7) +## │ └─ SampleNode(id: 43094, stats: 3/4, score: 1.82, length: 6) +## │ ├─ SampleNode(id: 14966, stats: 1/1, score: 2.67, length: 7) +## │ └─ SampleNode(id: 32991, stats: 1/1, score: 2.67, length: 7) +## └─ SampleNode(id: 20506, stats: 1/4, score: 1.4, length: 4) +## ├─ SampleNode(id: 37581, stats: 0/1, score: 1.67, length: 5) +## └─ SampleNode(id: 46632, stats: 0/1, score: 1.67, length: 5) + +# Lastly, let's check all the guesses AI made across all samples. +# Our winning guess was ID 32991 (`out.active_sample_id`) + +for sample in PostOrderDFS(out.samples) + [println("ID: \$(sample.id), Guess: \$(msg.content)") + for msg in sample.data if msg isa PT.AIMessage] +end +## ID: 20737, Guess: 50 +## ID: 20737, Guess: 35 +## ID: 20737, Guess: 37 +## ID: 52910, Guess: 50 +## ID: 52910, Guess: 35 +## ID: 52910, Guess: 32 +## ID: 14966, Guess: 50 +## ID: 14966, Guess: 35 +## ID: 14966, Guess: 33 +## ID: 32991, Guess: 50 +## ID: 32991, Guess: 35 +## ID: 32991, Guess: 33 +## etc... +``` + +Note that if there are multiple "branches" the model will see only the feedback of its own and its ancestors not the other "branches". +If you want to show all object, set `n_samples=1`, so all fixing happens sequantially and model sees all feedback (less powerful if model falls into a bad state). +Alternatively, you can tweak the feedback function. + +# See Also + +References: `airetry` is inspired by the [Language Agent Tree Search paper](https://arxiv.org/abs/2310.04406) and by [DSPy Assertions paper](https://arxiv.org/abs/2312.13382). +""" +function airetry!(f_cond::Function, aicall::AICallBlock, + feedback::Union{AbstractString, Function} = ""; + verbose::Integer = 1, throw::Bool = false, evaluate_all::Bool = true, + feedback_expensive::Bool = false, + max_retries::Union{Nothing, Int} = nothing, retry_delay::Union{Nothing, Int} = nothing) + (; config) = aicall + (; max_calls, feedback_inplace, feedback_template) = aicall.config + + max_retries = max_retries isa Nothing ? config.max_retries : max_retries + retry_delay = retry_delay isa Nothing ? config.retry_delay : retry_delay + verbose = min(verbose, get(aicall.kwargs, :verbose, 99)) + + ## Enter the retry loop + condition_passed = false + while !condition_passed + + ## Evaluation + feedback (sample is either the "successful" node or the best node to retry from) + condition_passed, sample = evaluate_condition!(f_cond, aicall, feedback; + evaluate_all, feedback_expensive) + + ## Update the aicall + aicall.conversation = sample.data + aicall.active_sample_id = sample.id + aicall.success = condition_passed + + if condition_passed + ## If condition is met, break the loop + break + elseif (config.calls >= max_calls) || (config.retries >= max_retries) + ## condition not met, but no budget + balance_str = "(Retries: $(config.retries)/$(max_retries), Calls: $(config.calls)/$(max_calls))." + if throw + throw(ErrorException("Maximum retry budget reached $balance_str")) + else + verbose > 0 && + @info "Condition not met, but maximum retry budget was reached $balance_str" + break + end + end + + ## If the condition is not met and we have budget, retry the aicall + verbose > 0 && @info "Condition not met. Retrying..." + ## Note: we already sampled the best node to expand from in aicall in evaluate_condition + + ## Append feedback if provided + if sample.feedback != "" + aicall.conversation = add_feedback!(aicall.conversation, sample; + feedback_inplace, feedback_template) + end + sleep(retry_delay) + aicall.config.retries += 1 + run!(aicall; verbose) + end + + return aicall +end + +""" + evaluate_condition!(f_cond::Function, aicall::AICallBlock, + feedback::Union{AbstractString, Function} = ""; + evaluate_all::Bool = true, feedback_expensive::Bool = false) + +Evalutes the condition `f_cond` (must return Bool) on the `aicall` object. +If the condition is not met, it will return the best sample to retry from and provide `feedback`. + +Mutating as the results are saved in `aicall.samples` + +If `evaluate_all` is `true`, it will evaluate all the "successful" samples in the `aicall` object. Otherwise, it will only evaluate the active sample.. + +For `f_cond` and `feedback` functions, you can use the `last_message` and `last_output` utilities to access the last message and last output in the conversation, respectively. + +# Arguments +- `f_cond::Function`: A function that accepts the `aicall` object and returns a boolean value. Retry will be attempted if the condition is not met (`f_cond -> false`). +- `aicall::AICallBlock`: The `aicall` object to evaluate the condition on. +- `feedback::Union{AbstractString, Function}`: Feedback to provide if the condition is not met. If a function is provided, it must accept the `aicall` object as the only argument and return a string. +- `evaluate_all::Bool=false`: If true, it will evaluate all the "successful" samples in the `aicall` object. Otherwise, it will only evaluate the active sample. +- `feedback_expensive::Bool=false`: If false, it will provide feedback to all samples that fail the condition. + If `feedback` function is expensive to call (eg, another ai* function), set this to `true` and feedback will be provided only to the sample we will retry from. + +# Returns +- a tuple `(condition_passed, sample)`, where `condition_passed` is a boolean indicating whether the condition was met, and `sample` is the best sample to retry from. + +# Example +```julia +# Mimic AIGenerate run! +aicall = AIGenerate("Say hi!"; config = RetryConfig(; n_samples = 2)) +sample = expand!(aicall.samples, aicall.conversation; success = true) +aicall.active_sample_id = sample.id + +# Return whether it passed and node to take the next action from +cond, node = AT.evaluate_condition!(x -> occursin("hi", last_output(x)), aicall) + +# Checks: +cond == true +node == sample +node.wins == 1 +``` + +With feedback: +```julia +# Mimic AIGenerate run with feedback +aicall = AIGenerate( + :BlankSystemUser; system = "a", user = "b") +sample = expand!(aicall.samples, aicall.conversation; success = true) +aicall.active_sample_id = sample.id + +# Evaluate +cond, node = AT.evaluate_condition!( + x -> occursin("NOTFOUND", last_output(x)), aicall, "Feedback X") +cond == false # fail +sample == node # same node (no other choice) +node.wins == 0 +node.feedback == "\nFeedback X" +""" +function evaluate_condition!(f_cond::Function, aicall::AICallBlock, + feedback::Union{AbstractString, Function} = ""; + evaluate_all::Bool = true, feedback_expensive::Bool = false) + (; scoring, ordering) = aicall.config + + ## Memorize the current conversation + conversation_ = aicall.conversation + active_id_ = aicall.active_sample_id + + ## Init + condition_passed = false + successful_id = nothing + + for sample in AbstractTrees.PreOrderDFS(aicall.samples) + ## If we want to evaluate only the active sample, skip the rest + if !evaluate_all && sample.id != active_id_ + continue + end + ## Eveluate only if the sample was successful so far, or if it's the active one + if sample.success == true || sample.id == active_id_ + ## Set the conversation for eval + aicall.conversation = sample.data + aicall.active_sample_id = sample.id + result = f_cond(aicall) + if result + ## If successful evaluation + condition_passed = true + successful_id = sample.id + else + ## Evaluation failed + sample.success = false + if !feedback_expensive && feedback != "" + ## If feedback is not expensive, get it + if feedback isa Function + sample.feedback *= "\n" * feedback(aicall) + else + sample.feedback *= "\n" * feedback + end + end + end + ## Backprop the results + backpropagate!(sample; wins = result, visits = 1) + end + end + + ## Finalize + sample = if condition_passed + ## Grab a successful sample + find_node(aicall.samples, successful_id) + else + ## We were unsuccessful, pick the best node to retry from + select_best(aicall.samples, scoring; ordering) + end + if !condition_passed && feedback != "" && + feedback_expensive + ## We were unsuccessful and haven't given any feedback yet + sample.feedback = if feedback isa Function + "\n" * feedback(aicall) + else + "\n" * feedback + end + end + + ## return to original state + aicall.conversation = conversation_ + aicall.active_sample_id = active_id_ + + return condition_passed, sample +end + +""" + add_feedback!( + conversation::AbstractVector{<:PT.AbstractMessage}, sample::SampleNode; feedback_inplace::Bool = false, + feedback_template::Symbol = :FeedbackFromEvaluator) + +Adds formatted feedback to the `conversation` based on the `sample` node feedback (and its ancestors). + +# Arguments +- `conversation::AbstractVector{<:PT.AbstractMessage}`: The conversation to add the feedback to. +- `sample::SampleNode`: The sample node to extract the feedback from. +- `feedback_inplace::Bool=false`: If true, it will add the feedback to the last user message inplace (and pop the last AIMessage). Otherwise, it will append the feedback as a new message. +- `feedback_template::Symbol=:FeedbackFromEvaluator`: The template to use for the feedback message. It must be a valid `AITemplate` name. + +# Example + +```julia +sample = SampleNode(; data = nothing, feedback = "Feedback X") +conversation = [PT.UserMessage("I say hi!"), PT.AIMessage(; content = "I say hi!")] +conversation = AT.add_feedback!(conversation, sample) +conversation[end].content == "### Feedback from Evaluator\nFeedback X\n" + +Inplace feedback: +```julia +conversation = [PT.UserMessage("I say hi!"), PT.AIMessage(; content = "I say hi!")] +conversation = AT.add_feedback!(conversation, sample; feedback_inplace = true) +conversation[end].content == "I say hi!\n\n### Feedback from Evaluator\nFeedback X\n" +``` + +Sample with ancestors with feedback: +```julia +sample_p = SampleNode(; data = nothing, feedback = "\nFeedback X") +sample = expand!(sample_p, nothing) +sample.feedback = "\nFeedback Y" +conversation = [PT.UserMessage("I say hi!"), PT.AIMessage(; content = "I say hi!")] +conversation = AT.add_feedback!(conversation, sample) + +conversation[end].content == +"### Feedback from Evaluator\n\nFeedback X\n----------\n\nFeedback Y\n" +``` +""" +function add_feedback!(conversation::AbstractVector{<:PT.AbstractMessage}, + sample::SampleNode; feedback_inplace::Bool = false, + feedback_template::Symbol = :FeedbackFromEvaluator) + ## + all_feedback = collect_all_feedback(sample) + ## short circuit if no feedback + if strip(all_feedback) == "" + return conversation + end + ## Prepare feedback as a UserMessage + feedback_message = let schema = PT.NoSchema() + # feedback from all ancestors, newline separated + template = PT.AITemplate(feedback_template) + output = PT.render(schema, template) # render the feedback template + output = PT.render(schema, output; feedback = all_feedback) # replace the placeholder + output[end] # UserMessage with the feedback + end + if feedback_inplace + ## Remove AI Message and extract he user message + user_msg = pop!(conversation) ## pop the last AI message + while !PT.isusermessage(user_msg) + length(conversation) == 0 && + throw("Something went wrong, no user messages detected to add feedback into.") + ## keep popping until we find the user message + user_msg = pop!(conversation) + end + ## Concatenate the feedback message with the user message + user_msg = PT.UserMessage(; + content = user_msg.content * "\n\n" * feedback_message.content) + push!(conversation, user_msg) + else + ## append the feedback message to the conversation + push!(conversation, feedback_message) + end + return conversation +end diff --git a/src/Experimental/AgentTools/utils.jl b/src/Experimental/AgentTools/utils.jl index babdcbafd..b552fed61 100644 --- a/src/Experimental/AgentTools/utils.jl +++ b/src/Experimental/AgentTools/utils.jl @@ -10,6 +10,64 @@ function remove_used_kwargs(kwargs::NamedTuple, return filter(pair -> !(pair.first in used_kwargs), pairs(kwargs)) |> NamedTuple end +"Unwraps the arguments for AICall and returns the schema and conversation (if provided). Expands any provided AITemplate." +function unwrap_aicall_args(args) + @assert length(args)<=2 "AICall takes at most 2 positional arguments (provided: $(length(args)))" + schema = nothing + conversation = Vector{PT.AbstractMessage}() + for arg in args + if isa(arg, PT.AbstractPromptSchema) + schema = arg + elseif isa(arg, Vector{<:PT.AbstractMessage}) + conversation = arg + elseif isa(arg, AbstractString) && isempty(conversation) + ## User Prompt -- create a UserMessage + push!(conversation, PT.UserMessage(arg)) + elseif isa(arg, Symbol) && isempty(conversation) + conversation = PT.render(schema, AITemplate(arg)) + elseif isa(arg, AITemplate) && isempty(conversation) + conversation = PT.render(schema, arg) + else + error("Invalid argument type: $(typeof(arg))") + end + end + return schema, conversation +end + +"Extracts `config::RetryConfig` from kwargs and returns the rest of the kwargs." +function extract_config(kwargs, default_config::T) where {T} + new_kwargs = [] + config = default_config + for (key, val) in Base.pairs(kwargs) + if key == :config && val isa T + config = val + else + push!(new_kwargs, (key => val)) + end + end + return NamedTuple(new_kwargs), config +end + +"If the conversation has multiple AIMessage samples, split them into separate conversations with the common past." +function split_multi_samples(conv) + ## shortcircuit if the conversation is too short, has no AIMessage or has no integer sample_id + if length(conv) <= 1 || !(last(conv) isa PT.AIMessage) || + isnothing(last(conv).sample_id) + return [conv] + end + + split_convos = typeof(conv)[] + run_id = last(conv).run_id + ## Extract the common history for all new samples + past_conv = filter(x -> !PT.isaimessage(x) || x.run_id != run_id, conv) + for i in eachindex(conv) + if PT.isaimessage(conv[i]) && conv[i].run_id == run_id + push!(split_convos, vcat(copy(past_conv)..., conv[i])) + end + end + return split_convos +end + """ truncate_conversation(conversation::AbstractVector{<:PT.AbstractMessage}; max_conversation_length::Int = 32000) @@ -48,3 +106,40 @@ function truncate_conversation(conversation::AbstractVector{<:PT.AbstractMessage end return new_conversation end + +""" + gamma_sample(α::Real, θ::Real) + +Approximates a sample from the Gamma distribution using the Marsaglia and Tsang method. +""" +function gamma_sample(α::Real, θ::Real) + if α < 1 + return gamma_sample(1.0 + α, θ) * (rand()^(1 / α)) + end + d = α - 1.0 / 3 + c = 1.0 / sqrt(9d) + while true + x = randn() + v = 1.0 + c * x + while v <= 0 + x = randn() + v = 1.0 + c * x + end + v = v^3 + u = rand() + if u < 1 - 0.0331 * (x^4) || log(u) < 0.5 * x^2 + d * (1 - v + log(v)) + return d * v * θ + end + end +end + +""" + beta_sample(α::Real, β::Real) + +Approximates a sample from the Beta distribution by generating two independent Gamma distributed samples and using their ratio. +""" +function beta_sample(α::Real, β::Real) + x = gamma_sample(α, 1.0) + y = gamma_sample(β, 1.0) + return x / (x + y) +end diff --git a/src/Experimental/RAGTools/preparation.jl b/src/Experimental/RAGTools/preparation.jl index cf7304e5f..a61b23bfb 100644 --- a/src/Experimental/RAGTools/preparation.jl +++ b/src/Experimental/RAGTools/preparation.jl @@ -203,8 +203,9 @@ end """ build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, - separators = ["\n\n", ". ", "\n"], max_length::Int = 256, + separators = ["\\n\\n", ". ", "\\n"], max_length::Int = 256, sources::Vector{<:AbstractString} = files_or_docs, + extras::Union{Nothing, AbstractVector} = nothing, extract_metadata::Bool = false, verbose::Integer = 1, index_id = gensym("ChunkIndex"), metadata_template::Symbol = :RAGExtractMetadataShort, @@ -222,9 +223,10 @@ optionally extracts metadata, and then compiles this information into a retrieva # Arguments - `files_or_docs`: A vector of valid file paths OR string documents to be indexed (chunked and embedded). - `reader`: A symbol indicating the type of input, can be either `:files` or `:docs`. Default is `:files`. -- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\n\n", ". ", "\n"]`. +- `separators`: A list of strings used as separators for splitting the text in each file into chunks. Default is `[\\n\\n, ". ", "\\n"]`. - `max_length`: The maximum length of each chunk (if possible with provided separators). Default is 256. - `sources`: A vector of strings indicating the source of each chunk. Default is equal to `files_or_docs` (for `reader=:files`) +- `extras`: An optional vector of extra information to be stored with each chunk. Default is `nothing`. - `extract_metadata`: A boolean flag indicating whether to extract metadata from each chunk (to build filter `tags` in the index). Default is `false`. Metadata extraction incurs additional cost and requires `model_metadata` and `metadata_template` to be provided. - `verbose`: An Integer specifying the verbosity of the logs. Default is `1` (high-level logging). `0` is disabled. @@ -262,6 +264,7 @@ index = build_index(["file1.txt", "file2.txt"]; function build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = :files, separators = ["\n\n", ". ", "\n"], max_length::Int = 256, sources::Vector{<:AbstractString} = files_or_docs, + extras::Union{Nothing, AbstractVector} = nothing, extract_metadata::Bool = false, verbose::Integer = 1, index_id = gensym("ChunkIndex"), metadata_template::Symbol = :RAGExtractMetadataShort, @@ -304,6 +307,7 @@ function build_index(files_or_docs::Vector{<:AbstractString}; reader::Symbol = : embeddings, tags, tags_vocab, chunks = output_chunks, - sources = output_sources) + sources = output_sources, + extras) return index end diff --git a/src/Experimental/RAGTools/types.jl b/src/Experimental/RAGTools/types.jl index 863a25abb..35afe6f1e 100644 --- a/src/Experimental/RAGTools/types.jl +++ b/src/Experimental/RAGTools/types.jl @@ -20,11 +20,13 @@ Main struct for storing document chunks and their embeddings. It also stores tag - `tags::Union{Nothing, AbstractMatrix{<:Bool}}`: for exact search, filtering, etc. This is often a sparse matrix indicating which chunks have the given `tag` (see `tag_vocab` for the position lookup) - `tags_vocab::Union{Nothing, Vector{<:AbstractString}}`: vocabulary for the `tags` matrix (each column in `tags` is one item in `tags_vocab` and rows are the chunks) - `sources::Vector{<:AbstractString}`: sources of the chunks +- `extras::Union{Nothing, AbstractVector}`: additional data, eg, metadata, source code, etc. """ @kwdef struct ChunkIndex{ T1 <: AbstractString, T2 <: Union{Nothing, Matrix{<:Real}}, T3 <: Union{Nothing, AbstractMatrix{<:Bool}}, + T4 <: Union{Nothing, AbstractVector}, } <: AbstractChunkIndex id::Symbol = gensym("ChunkIndex") # underlying document chunks / snippets @@ -37,6 +39,7 @@ Main struct for storing document chunks and their embeddings. It also stores tag tags::T3 = nothing tags_vocab::Union{Nothing, Vector{<:AbstractString}} = nothing sources::Vector{<:AbstractString} + extras::T4 = nothing end embeddings(index::ChunkIndex) = index.embeddings chunks(index::ChunkIndex) = index.chunks diff --git a/src/llm_openai.jl b/src/llm_openai.jl index 764babe0f..cdbd3738b 100644 --- a/src/llm_openai.jl +++ b/src/llm_openai.jl @@ -70,8 +70,7 @@ function OpenAI.build_url(provider::AbstractCustomProvider, api::AbstractString) string(provider.base_url, "/", api) end function OpenAI.auth_header(provider::AbstractCustomProvider, api_key::AbstractString) - OpenAI.auth_header( - OpenAI.OpenAIProvider(provider.api_key, + OpenAI.auth_header(OpenAI.OpenAIProvider(provider.api_key, provider.base_url, provider.api_version), api_key) @@ -483,7 +482,7 @@ function aigenerate(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ run_id = Int(rand(Int32)) # remember one run ID ## extract all message msgs = [response_to_message(prompt_schema, AIMessage, choice, r; - time, model_id, run_id, sample_id = i) + time, model_id, run_id, sample_id = i) for (i, choice) in enumerate(r.response[:choices])] ## Order by log probability if available ## bigger is better, keep it last @@ -788,9 +787,11 @@ aiclassify(:InputClassifier; choices, input) Choices with descriptions provided as tuples: ```julia choices = [("A", "any animal or creature"), ("P", "for any plant or tree"), ("O", "for everything else")] -input = "spider" -input = "daphodil" -input = "castle" + +# try the below inputs: +input = "spider" # -> returns "A" for any animal or creature +input = "daphodil" # -> returns "P" for any plant or tree +input = "castle" # -> returns "O" for everything else aiclassify(:InputClassifier; choices, input) ``` @@ -824,7 +825,7 @@ function aiclassify(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_ choices::AbstractVector{T} = ["true", "false", "unknown"], api_kwargs::NamedTuple = NamedTuple(), kwargs...) where {T <: - Union{AbstractString, Tuple{<:AbstractString, <:AbstractString}}} + Union{AbstractString, Tuple{<:AbstractString, <:AbstractString}}} ## Encode the choices and the corresponding prompt ## TODO: maybe check the model provided as well? choices_prompt, logit_bias, decode_ids = encode_choices(prompt_schema, choices) @@ -886,16 +887,17 @@ function response_to_message(schema::AbstractOpenAISchema, end """ - aiextract([prompt_schema::AbstractOpenAISchema,] prompt::ALLOWED_PROMPT_TYPE; - return_type::Type, - verbose::Bool = true, + aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; + return_type::Type, + verbose::Bool = true, + api_key::String = OPENAI_API_KEY, model::String = MODEL_CHAT, - return_all::Bool = false, dry_run::Bool = false, + return_all::Bool = false, dry_run::Bool = false, conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], - http_kwargs::NamedTuple = (; - retry_non_idempotent = true, + http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, - readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + readtimeout = 120), api_kwargs::NamedTuple = (; + tool_choice = "exact"), kwargs...) Extract required information (defined by a struct **`return_type`**) from the provided prompt by leveraging OpenAI function calling mode. @@ -917,7 +919,10 @@ It's effectively a light wrapper around `aigenerate` call, which requires additi - `dry_run::Bool=false`: If `true`, skips sending the messages to the model (for debugging, often used with `return_all=true`). - `conversation`: An optional vector of `AbstractMessage` objects representing the conversation history. If not provided, it is initialized as an empty vector. - `http_kwargs`: A named tuple of HTTP keyword arguments. -- `api_kwargs`: A named tuple of API keyword arguments. +- `api_kwargs`: A named tuple of API keyword arguments. + - `tool_choice`: A string representing the tool choice to use for the API call. Usually, one of "auto","any","exact". + Defaults to `"exact"`, which is a made-up value to enforce the OpenAI requirements if we want one exact function. + Providers like Mistral, Together, etc. use `"any"` instead. - `kwargs`: Prompt variables to be used to fill the prompt/template # Returns @@ -943,7 +948,6 @@ struct MyMeasurement weight::Union{Nothing,Float64} # optional end msg = aiextract("James is 30, weighs 80kg. He's 180cm tall."; return_type=MyMeasurement) -# [ Info: Tokens: 129 @ Cost: \$0.0002 in 1.0 seconds # PromptingTools.DataMessage(MyMeasurement) msg.content # MyMeasurement(30, 180, 80.0) @@ -1002,6 +1006,17 @@ That way, you can handle the error gracefully and get a reason why extraction fa Note that the error message refers to a giraffe not being a human, because in our `MyMeasurement` docstring, we said that it's for people! + +Some non-OpenAI providers require a different specification of the "tool choice" than OpenAI. +For example, to use Mistral models ("mistrall" for mistral large), do: +```julia +"Some fruit" +struct Fruit + name::String +end +aiextract("I ate an apple",return_type=Fruit,api_kwargs=(;tool_choice="any"),model="mistrall") +# Notice two differences: 1) struct MUST have a docstring, 2) tool_choice is set explicitly set to "any" +``` """ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE; return_type::Type, @@ -1012,15 +1027,24 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T conversation::AbstractVector{<:AbstractMessage} = AbstractMessage[], http_kwargs::NamedTuple = (retry_non_idempotent = true, retries = 5, - readtimeout = 120), api_kwargs::NamedTuple = NamedTuple(), + readtimeout = 120), api_kwargs::NamedTuple = (; + tool_choice = "exact"), kwargs...) ## global MODEL_ALIASES ## Function calling specifics tools = [Dict(:type => "function", :function => function_call_signature(return_type))] ## force our function to be used - tool_choice = Dict(:type => "function", - :function => Dict(:name => only(tools)[:function]["name"])) + tool_choice_ = get(api_kwargs, :tool_choice, "exact") + tool_choice = if tool_choice_ == "exact" + ## Standard for OpenAI API + Dict(:type => "function", + :function => Dict(:name => only(tools)[:function]["name"])) + else + # User provided value, eg, "auto", "any" for various providers like Mistral, Together, etc. + tool_choice_ + end + ## Add the function call signature to the api_kwargs api_kwargs = merge(api_kwargs, (; tools, tool_choice)) ## Find the unique ID for the model alias provided @@ -1038,7 +1062,7 @@ function aiextract(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_T run_id = Int(rand(Int32)) # remember one run ID ## extract all message msgs = [response_to_message(prompt_schema, DataMessage, choice, r; - return_type, time, model_id, run_id, sample_id = i) + return_type, time, model_id, run_id, sample_id = i) for (i, choice) in enumerate(r.response[:choices])] ## Order by log probability if available ## bigger is better, keep it last @@ -1195,7 +1219,7 @@ function aiscan(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYPE run_id = Int(rand(Int32)) # remember one run ID ## extract all message msgs = [response_to_message(prompt_schema, AIMessage, choice, r; - time, model_id, run_id, sample_id = i) + time, model_id, run_id, sample_id = i) for (i, choice) in enumerate(r.response[:choices])] ## Order by log probability if available ## bigger is better, keep it last diff --git a/src/llm_shared.jl b/src/llm_shared.jl index a611c5929..b84d73371 100644 --- a/src/llm_shared.jl +++ b/src/llm_shared.jl @@ -118,4 +118,4 @@ function decode_choices(schema::AbstractPromptSchema, choices, conv; kwargs...) end function decode_choices(schema::AbstractPromptSchema, choices, conv::Nothing; kwargs...) nothing -end \ No newline at end of file +end diff --git a/src/user_preferences.jl b/src/user_preferences.jl index 2e104236d..ea734deba 100644 --- a/src/user_preferences.jl +++ b/src/user_preferences.jl @@ -275,8 +275,7 @@ end ### Model Aliases # global reference MODEL_ALIASES is defined below -aliases = merge( - Dict("gpt3" => "gpt-3.5-turbo", +aliases = merge(Dict("gpt3" => "gpt-3.5-turbo", "gpt4" => "gpt-4", "gpt4v" => "gpt-4-vision-preview", # 4v is for "4 vision" "gpt4t" => "gpt-4-turbo-preview", # 4t is for "4 turbo" @@ -293,12 +292,18 @@ aliases = merge( "fmixtral" => "accounts/fireworks/models/mixtral-8x7b-instruct", "firefunction" => "accounts/fireworks/models/firefunction-v1", ## t-mixtral -> Together.ai Mixtral - "tmixtral" => "mistralai/Mixtral-8x7B-Instruct-v0.1"), + "tmixtral" => "mistralai/Mixtral-8x7B-Instruct-v0.1", + ## Mistral AI + "mistral-small" => "mistral-small-latest", + "mistral-medium" => "mistral-medium-latest", + "mistral-large" => "mistral-large-latest", + "mistrals" => "mistral-small-latest", + "mistralm" => "mistral-medium-latest", + "mistrall" => "mistral-large-latest"), ## Load aliases from preferences as well @load_preference("MODEL_ALIASES", default=Dict{String, String}())) -registry = Dict{String, ModelSpec}( - "gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", +registry = Dict{String, ModelSpec}("gpt-3.5-turbo" => ModelSpec("gpt-3.5-turbo", OpenAISchema(), 0.5e-6, 1.5e-6, @@ -382,32 +387,41 @@ registry = Dict{String, ModelSpec}( OllamaSchema(), 0.0, 0.0, "BakLLaVA is a multimodal model consisting of the Mistral 7B base model augmented with the LLaVA architecture."), - "mistral-tiny" => ModelSpec("mistral-tiny", + "open-mistral-7b" => ModelSpec("open-mistral-7b", MistralOpenAISchema(), - 1.4e-7, - 4.53e-7, - "Mistral AI's hosted version of Mistral-7B-v0.2. Great for simple tasks."), - "mistral-small" => ModelSpec("mistral-small", + 2.5e-7, + 2.5e-7, + "Mistral AI's hosted version of openly available Mistral-7B-v0.2. Great for simple tasks."), + "open-mixtral-8x7b" => ModelSpec("open-mixtral-8x7b", MistralOpenAISchema(), - 6.47e-7, - 1.94e-6, - "Mistral AI's hosted version of Mixtral-8x7B-v0.1. Good for more complicated tasks."), - "mistral-medium" => ModelSpec("mistral-medium", + 7e-7, + 7e-7, + "Mistral AI's hosted version of openly available Mixtral-8x7B-v0.1. Good for more complicated tasks."), + "mistral-small-latest" => ModelSpec("mistral-small-latest", + MistralOpenAISchema(), + 2e-6, + 6e-6, + "Mistral AI's own finetune (historically similar to Mixtral-8x7B)."), + "mistral-medium-latest" => ModelSpec("mistral-medium-latest", MistralOpenAISchema(), 2.7e-6, - 8.09e-6, + 8.1e-6, + "Mistral AI's own model. Details unknown."), + "mistral-large-latest" => ModelSpec("mistral-large-latest", + MistralOpenAISchema(), + 8e-6, + 2.4e-5, "Mistral AI's hosted version of their best model available. Details unknown."), "mistral-embed" => ModelSpec("mistral-embed", MistralOpenAISchema(), - 1.08e-7, + 1e-7, 0.0, "Mistral AI's hosted model for embeddings."), "echo" => ModelSpec("echo", TestEchoOpenAISchema(; - response = Dict( - :choices => [ + response = Dict(:choices => [ Dict(:message => Dict(:content => "Hello!"), - :finish_reason => "stop") + :finish_reason => "stop"), ], :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, @@ -425,20 +439,17 @@ registry = Dict{String, ModelSpec}( 0.0, #unknown, expected 1.25e-7 0.0, #unknown, expected 3.75e-7 "Gemini Pro is a LLM from Google. For more information, see [models](https://ai.google.dev/models/gemini)."), - "accounts/fireworks/models/mixtral-8x7b-instruct" => ModelSpec( - "accounts/fireworks/models/mixtral-8x7b-instruct", + "accounts/fireworks/models/mixtral-8x7b-instruct" => ModelSpec("accounts/fireworks/models/mixtral-8x7b-instruct", FireworksOpenAISchema(), 4e-7, #unknown, expected 1.25e-7 1.6e-6, #unknown, expected 3.75e-7 "Mixtral (8x7b) from Mistral, hosted by Fireworks.ai. For more information, see [models](https://fireworks.ai/models/fireworks/mixtral-8x7b-instruct)."), - "accounts/fireworks/models/firefunction-v1" => ModelSpec( - "accounts/fireworks/models/firefunction-v1", + "accounts/fireworks/models/firefunction-v1" => ModelSpec("accounts/fireworks/models/firefunction-v1", FireworksOpenAISchema(), 0.0, #unknown, expected to be the same as Mixtral 0.0, #unknown, expected to be the same as Mixtral "Fireworks' open-source function calling model (fine-tuned Mixtral). Useful for `aiextract` calls. For more information, see [models](https://fireworks.ai/models/fireworks/firefunction-v1)."), - "mistralai/Mixtral-8x7B-Instruct-v0.1" => ModelSpec( - "mistralai/Mixtral-8x7B-Instruct-v0.1", + "mistralai/Mixtral-8x7B-Instruct-v0.1" => ModelSpec("mistralai/Mixtral-8x7B-Instruct-v0.1", TogetherOpenAISchema(), 6e-7, 6e-7, diff --git a/templates/agents/feedback/FeedbackFromEvaluator.json b/templates/agents/feedback/FeedbackFromEvaluator.json new file mode 100644 index 000000000..d9e393b8f --- /dev/null +++ b/templates/agents/feedback/FeedbackFromEvaluator.json @@ -0,0 +1 @@ +[{"content":"Template Metadata","description":"Simple user message with \"Feedback from Evaluator\". Placeholders: `feedback`","version":"1.0","source":"","_type":"metadatamessage"},{"content":"### Feedback from Evaluator\n{{feedback}}\n","variables":["feedback"],"_type":"usermessage"}] \ No newline at end of file diff --git a/test/Experimental/AgentTools/lazy_types.jl b/test/Experimental/AgentTools/lazy_types.jl index fe1dc19a2..7d5a24fab 100644 --- a/test/Experimental/AgentTools/lazy_types.jl +++ b/test/Experimental/AgentTools/lazy_types.jl @@ -1,3 +1,21 @@ +@testset "RetryConfig" begin + config = RetryConfig() + config.retries = 1 + config.calls = 1 + + # Show method + io = IOBuffer() + show(io, config) + str = String(take!(io)) + @test str == + "RetryConfig\n retries: Int64 1\n calls: Int64 1\n max_retries: Int64 10\n max_calls: Int64 99\n retry_delay: Int64 0\n n_samples: Int64 1\n scoring: PromptingTools.Experimental.AgentTools.UCT\n ordering: Symbol PostOrderDFS\n feedback_inplace: Bool false\n feedback_template: Symbol FeedbackFromEvaluator\n temperature: Float64 0.7\n catch_errors: Bool false\n" + + ## copy + config2 = copy(config) + @test config2 == config + @test config2 !== config +end + @testset "AICall" begin # Create AICall with default parameters default_call = AICall(identity) @@ -81,6 +99,19 @@ output = String(take!(io)) @test output == "AICall{typeof(identity)}(Messages: 2, Success: true)\n- Preview of the Latest AIMessage (see property `:conversation`):\n Test message" + + ## last_message, last_output + @test last_output(aicall) == aicall.conversation[end].content + @test last_message(aicall) == aicall.conversation[end] + + ## isvalid + @test isvalid(aicall) == aicall.success + + ## copy + aicall = AICall(identity) + aicall2 = copy(aicall) + @test aicall == aicall2 + @test aicall !== aicall2 end @testset "AICodeFixer" begin diff --git a/test/Experimental/AgentTools/mcts.jl b/test/Experimental/AgentTools/mcts.jl new file mode 100644 index 000000000..18e8c529b --- /dev/null +++ b/test/Experimental/AgentTools/mcts.jl @@ -0,0 +1,198 @@ +using PromptingTools.Experimental.AgentTools: expand!, find_node, backpropagate!, SampleNode +using PromptingTools.Experimental.AgentTools: print_tree, + print_samples, reset_success!, + collect_all_feedback +using PromptingTools.Experimental.AgentTools: score, + UCT, ThompsonSampling, + AbstractScoringMethod, select_best + +@testset "SampleNode,expand!,find_node,reset_success!,print_samples" begin + data = PT.AbstractMessage[] + root = SampleNode(; data) + child1 = expand!(root, data) + child2 = expand!(root, data) + child11 = expand!(child1, data; success = true) + + ## parent, children + @test AbstractTrees.parent(root) == nothing + @test AbstractTrees.children(root) == [child1, child2] + @test AbstractTrees.children(child1) == [child11] + + ## Getindex, find_node + @test root[child11.id] == child11 + @test root[-1] == nothing + + ## Show method + io = IOBuffer() + show(io, child1) + @test String(take!(io)) == "SampleNode(id: $(child1.id), stats: 0/0, length: 0)" + + ## print_tree + io = IOBuffer() + print_tree(io, root) + s = String(take!(io)) + @test occursin("id: $(root.id)", s) + @test occursin("id: $(child1.id)", s) + @test occursin("id: $(child2.id)", s) + @test occursin("id: $(child11.id)", s) + + ## print_samples + io = IOBuffer() + print_samples(io, root) + s = String(take!(io)) + @test occursin("id: $(root.id)", s) + @test occursin("id: $(child1.id)", s) + @test occursin("id: $(child2.id)", s) + @test occursin("id: $(child11.id)", s) + @test occursin("score: ", s) + + ## expand! kwargs + @test root.success == nothing + @test child11.success == true + + ## reset_success! + reset_success!(root, true) + @test root.success == true + @test child1.success == true + @test child2.success == true + @test child11.success == true + + ## copy + root_copy = copy(root) + @test root_copy.id == root.id + @test root_copy.data == root.data + @test root_copy.success == root.success + @test root_copy.feedback == root.feedback + @test root_copy !== root +end + +@testset "collect_all_feedback" begin + data = PT.AbstractMessage[] + root = SampleNode(; data, feedback = "Feedback 3") + child1 = expand!(root, data, feedback = "Feedback 2") + child2 = expand!(root, data, feedback = "") + child11 = expand!(child1, data; feedback = "Feedback 0") + + # Test for collecting feedback of the toplevel node + @test collect_all_feedback(root) == "Feedback 3" + + @test collect_all_feedback(child11) == + "Feedback 3\n----------\nFeedback 2\n----------\nFeedback 0" + + # Test for correct handling of custom separator + alternative_separator = " | " + @test collect_all_feedback(child11, separator = alternative_separator) == + "Feedback 3 | Feedback 2 | Feedback 0" + + # Test to ensure function works with empty feedback strings + node_without_feedback = SampleNode(; data, feedback = "") + @test collect_all_feedback(node_without_feedback) == "" + + # Test functionality when nodes have a mix of empty and non-empty feedback + @test collect_all_feedback(child2) == "Feedback 3" +end + +@testset "score" begin + data = PT.AbstractMessage[] + node_with_parent = SampleNode(; data, wins = 10, visits = 20) # Provide necessary fields + parent_node = SampleNode(; data, wins = 15, visits = 30) + node_with_parent.parent = parent_node + + node_without_parent = SampleNode(; data, wins = 5, visits = 10) + ## UCT + uct_scoring = UCT(exploration = 2) + @test score(node_with_parent, uct_scoring) ≈ 0.5 + sqrt(log(30) / 20) * 2 + @test score(node_without_parent, uct_scoring) == 0.5 + + ## Thompson Sampling -- beta_sample is a random function, so we can only test for the range + ts_scoring = ThompsonSampling(alpha = 1, beta = 1) + @test 0 <= score(node_with_parent, ts_scoring) <= 1 + @test 0 <= score(node_without_parent, ts_scoring) <= 1 + ## high alpha means closer to 1 + ts_scoring_high = ThompsonSampling(alpha = 1001, beta = 1) + @test score(node_with_parent, ts_scoring_high) >= 1001 / 1002 - 0.2 # tolerance + ts_scoring_low = ThompsonSampling(alpha = 1, beta = 1001) + @test score(node_with_parent, ts_scoring_low) <= 1 / 1002 + 0.2 # tolerance + + ## unsupported scoring method + struct MyRand125Scoring <: AbstractScoringMethod end + @test_throws ArgumentError score(node_with_parent, MyRand125Scoring()) # should throw an error +end + +@testset "backpropagate!" begin + data = PT.AbstractMessage[] + root = SampleNode(; data) + child1 = expand!(root, data) + backpropagate!(child1; wins = 1, visits = 1) + child2 = expand!(root, data) + backpropagate!(child2; wins = 0, visits = 1) + child11 = expand!(child1, data) + backpropagate!(child11; wins = 1, visits = 1) + + @test root.wins == 2 + @test root.visits == 3 + @test child1.wins == 2 + @test child1.visits == 2 + @test child2.wins == 0 + @test child2.visits == 1 + @test child11.wins == 1 + @test child11.visits == 1 + + # Scenario: applying backpropagate! to the root node (only affects the root) + backpropagate!(root; wins = 2, visits = 2) + @test root.wins == 4 + @test root.visits == 5 + + # Scenario: applying backpropagate! to a child node (affects the child and the root) + backpropagate!(child11; wins = 2, visits = 2) + @test child11.wins == 3 + @test child11.visits == 3 + @test root.wins == 6 + @test root.visits == 7 + # no change + @test child2.wins == 0 + @test child2.visits == 1 +end + +@testset "select_best" begin + data = PT.AbstractMessage[] + root = SampleNode(; data) + child1 = expand!(root, data) + backpropagate!(child1; wins = 1, visits = 1) + child2 = expand!(root, data) + backpropagate!(child2; wins = 0, visits = 1) + child11 = expand!(child1, data) + backpropagate!(child11; wins = 1, visits = 1) + + for scoring in [UCT(), ThompsonSampling()] + for ordering in [:PreOrderDFS, :PostOrderDFS] + s = select_best(root, scoring, ordering = ordering) + @test s isa SampleNode + end + end + + # Ensure that an assertion error is raised for invalid `ordering` values + @test_throws AssertionError select_best(root, UCT(), ordering = :InvalidOrder) + + ## UCT is quite stable + best_uct1 = select_best(root, UCT(); ordering = :PreOrderDFS) + best_uct2 = select_best(root, UCT(); ordering = :PostOrderDFS) + @test child11 == best_uct1 == best_uct2 + + best_ts = select_best(root, ThompsonSampling()) + @test child2 != best_ts # Unlikely to be the losing node + + ## if no scores, Pre/Post with UCT determines which node is selected + data = PT.AbstractMessage[] + root = SampleNode(; data) + child1 = expand!(root, data) + child2 = expand!(root, data) + child11 = expand!(child1, data) + + # PreOrder picks the root node + best_uct1 = select_best(root, UCT(); ordering = :PreOrderDFS) + @test root == best_uct1 + # PostOrder picks the leaf node + best_uct2 = select_best(root, UCT(); ordering = :PostOrderDFS) + @test child11 == best_uct2 +end diff --git a/test/Experimental/AgentTools/retry.jl b/test/Experimental/AgentTools/retry.jl new file mode 100644 index 000000000..788ce166c --- /dev/null +++ b/test/Experimental/AgentTools/retry.jl @@ -0,0 +1,184 @@ +using PromptingTools.Experimental.AgentTools: add_feedback!, + evaluate_condition!, + SampleNode, expand!, AICallBlock + +@testset "add_feedback!" begin + # Test for adding feedback as a new message to the conversation + sample = SampleNode(; data = nothing, feedback = "Test Feedback") + conversation = [ + PT.UserMessage("User says hello"), PT.AIMessage(; content = "AI responds")] + updated_conversation = add_feedback!(conversation, sample) + + @test length(updated_conversation) == 3 + @test updated_conversation[end].content == + "### Feedback from Evaluator\nTest Feedback\n" + + # Test for adding feedback inplace to the last user message + sample = SampleNode(; data = nothing, feedback = "Inplace Feedback") + conversation = [ + PT.UserMessage("Initial message"), PT.AIMessage(; content = "AI message")] + updated_conversation = add_feedback!(conversation, sample; feedback_inplace = true) + + # remove AI message, so only 1 is left + @test length(updated_conversation) == 1 + @test occursin("Inplace Feedback", updated_conversation[end].content) + + # Test with empty feedback should not alter conversation + sample = SampleNode(; data = nothing, feedback = "") + conversation = [PT.UserMessage("Empty feedback scenario"), + PT.AIMessage(; content = "No feedback here")] + updated_conversation = add_feedback!(conversation, sample) + @test length(updated_conversation) == 2 + + # Test with empty feedback should not alter anything + sample = SampleNode(; data = nothing, feedback = "") + conversation = [PT.UserMessage("Empty feedback scenario"), + PT.AIMessage(; content = "No feedback here")] + updated_conversation = add_feedback!(conversation, sample; feedback_inplace = true) + @test length(updated_conversation) == 2 + + # Test for adding feedback with multiple ancestors' feedback collected + sample = SampleNode(; data = nothing, feedback = "Test Feedback") + child = expand!(sample, nothing; feedback = "Extra test") + conversation = [ + PT.UserMessage("User says hello"), PT.AIMessage(; content = "AI responds")] + updated_conversation = add_feedback!(conversation, child) + @test length(updated_conversation) == 3 + @test updated_conversation[end].content == + "### Feedback from Evaluator\nTest Feedback\n----------\nExtra test\n" + + # Test for attempting to add feedback inplace with no prior user message + sample = SampleNode(; data = nothing, feedback = "Orphan Feedback") + conversation = [AIMessage(; content = "Lonely AI message")] + @test_throws Exception add_feedback!(conversation, sample; feedback_inplace = false) +end + +@testset "evaluate_condition!" begin + function mock_f_cond_positive(aicall::AICallBlock) + return true + end + function mock_f_cond_negative(aicall::AICallBlock) + return false + end + feedback_str = "Test Feedback" + feedback_fun(aicall::AICallBlock) = "Function Feedback" + + # Test condition met, evaluate_all default (true) + aicall = AIGenerate("Say hi!"; config = RetryConfig(; n_samples = 1)) + aicall.active_sample_id = aicall.samples.id # mimick what happens in run! + condition_passed, suggested_sample = evaluate_condition!(mock_f_cond_positive, aicall) + @test condition_passed == true + @test suggested_sample === aicall.samples + + # Test condition not met, with string feedback, evaluate_all true + aicall = AIGenerate("Say hi!"; config = RetryConfig(; n_samples = 1)) + aicall.samples.success = true + aicall.active_sample_id = aicall.samples.id # mimick what happens in run! + node_success = expand!(aicall.samples, PT.AbstractMessage[]; success = true) + condition_passed, suggested_sample = evaluate_condition!(mock_f_cond_negative, aicall, + feedback_str; evaluate_all = true) + @test condition_passed == false + ## all nodes were evaluated and set to false + @test suggested_sample.feedback == "\n" * feedback_str + @test suggested_sample == node_success + @test suggested_sample.success == false + @test aicall.samples.feedback == "\n" * feedback_str + @test aicall.samples.success == false + + # Test condition not met, with function feedback, evaluate_all true + aicall = AIGenerate("Say hi!"; config = RetryConfig(; n_samples = 1)) + aicall.samples.success = true + aicall.active_sample_id = aicall.samples.id # mimick what happens in run! + condition_passed, suggested_sample = evaluate_condition!(mock_f_cond_negative, aicall, + feedback_fun, evaluate_all = true) + @test condition_passed == false + @test suggested_sample.feedback == "\n" * feedback_fun(aicall) + + # Test condition not met, feedback is expensive + aicall = AIGenerate("Say hi!"; config = RetryConfig(; n_samples = 1)) + aicall.samples.success = true + aicall.active_sample_id = aicall.samples.id # mimick what happens in run! + node_success = expand!(aicall.samples, PT.AbstractMessage[]; success = true) + condition_passed, suggested_sample = evaluate_condition!(mock_f_cond_negative, aicall, + feedback_str, feedback_expensive = true) + @test condition_passed == false + @test suggested_sample.feedback == "\n" * feedback_str + @test aicall.samples.feedback == "" # Not provided because marked as expensive! + + # Test condition not met, feedback is expensive -- with function feedback + aicall = AIGenerate("Say hi!"; config = RetryConfig(; n_samples = 1)) + aicall.samples.success = true + aicall.active_sample_id = aicall.samples.id # mimick what happens in run! + node_success = expand!(aicall.samples, PT.AbstractMessage[]; success = true) + condition_passed, suggested_sample = evaluate_condition!(mock_f_cond_negative, aicall, + feedback_fun, feedback_expensive = true) + @test condition_passed == false + @test suggested_sample.feedback == "\n" * feedback_fun(aicall) + @test aicall.samples.feedback == "" # Not provided because marked as expensive! + + # Test condition evaluated only on active sample, condition fails + aicall = AIGenerate("Say hi!"; config = RetryConfig(; n_samples = 1)) + aicall.samples.success = true + aicall.active_sample_id = aicall.samples.id # mimick what happens in run! + node_success = expand!(aicall.samples, PT.AbstractMessage[]; success = true) + condition_passed, suggested_sample = evaluate_condition!(mock_f_cond_negative, aicall, + "", evaluate_all = false) + @test condition_passed == false + @test aicall.samples.success == false + @test aicall.samples.children[1].success == true ## not actually checked! +end + +@testset "airetry!" begin + response = Dict(:choices => [ + Dict(:message => Dict(:content => "Hello!"), + :finish_reason => "stop"), + ], + :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1)) + schema = PT.TestEchoOpenAISchema(; response, status = 200) + + # Check condition passing without retries + aicall = AIGenerate(schema, "Say hi!"; + config = RetryConfig(max_retries = 0, retries = 0, calls = 0)) + run!(aicall) + # This condition should immediately pass + condition_func = _ -> true + airetry!(condition_func, aicall) + @test aicall.success == true + @test aicall.samples[aicall.active_sample_id].success == true + @test length(aicall.conversation) == 3 # No retries, only initial the basic messages + 1 response + @test aicall.config.retries == 0 # No retries performed + + # Fail condition and check retries + aicall = AIGenerate(schema, "Say hi!"; + config = RetryConfig(max_retries = 2, retries = 0, calls = 0)) + run!(aicall) + condition_not_met = _ -> false + airetry!(condition_not_met, aicall) + @test aicall.samples[aicall.active_sample_id].success == false + @test length(aicall.conversation) == 5 # Retries, no feedback, but 3 AI calls + @test count(PT.isaimessage, aicall.conversation) == 3 + @test count(PT.isusermessage, aicall.conversation) == 1 + @test aicall.config.retries == 2 + + # Fail condition and throw error + aicall = AIGenerate(schema, "Say hi!"; + config = RetryConfig(max_retries = 2, retries = 0, calls = 0)) + run!(aicall) + condition_not_met = _ -> false + @test_throws Exception airetry!(condition_not_met, aicall, throw = true) + @test aicall.config.retries == 2 + + # Fail condition and check retries + aicall = AIGenerate(schema, "Say hi!"; + config = RetryConfig(max_retries = 2, retries = 0, calls = 0)) + run!(aicall) + condition_not_met = _ -> false + airetry!(condition_not_met, aicall, "Retry feedback") + @test aicall.samples[aicall.active_sample_id].success == false + @test length(aicall.conversation) == 7 # Retries, no feedback, but 3 AI calls, 2 feedback msg + @test count(PT.isaimessage, aicall.conversation) == 3 + @test count(PT.isusermessage, aicall.conversation) == 3 # 1 initial, 2 feedbacks + @test occursin("Retry feedback", aicall.conversation[end - 3].content) + @test occursin("Retry feedback", aicall.conversation[end - 1].content) + @test aicall.config.retries == 2 +end diff --git a/test/Experimental/AgentTools/runtests.jl b/test/Experimental/AgentTools/runtests.jl index 55eef4bdc..b0c13d116 100644 --- a/test/Experimental/AgentTools/runtests.jl +++ b/test/Experimental/AgentTools/runtests.jl @@ -1,10 +1,13 @@ using Test using PromptingTools using PromptingTools.Experimental.AgentTools +using AbstractTrees const PT = PromptingTools @testset "AgentTools" begin include("utils.jl") include("code_feedback.jl") include("lazy_types.jl") + include("mcts.jl") + include("retry.jl") end diff --git a/test/Experimental/AgentTools/utils.jl b/test/Experimental/AgentTools/utils.jl index a52bc0320..63e9ea983 100644 --- a/test/Experimental/AgentTools/utils.jl +++ b/test/Experimental/AgentTools/utils.jl @@ -1,4 +1,7 @@ using PromptingTools.Experimental.AgentTools: remove_used_kwargs, truncate_conversation +using PromptingTools.Experimental.AgentTools: beta_sample, + gamma_sample, extract_config, + unwrap_aicall_args, split_multi_samples @testset "remove_used_kwargs" begin # Test 1: No overlapping keys @@ -20,6 +23,105 @@ using PromptingTools.Experimental.AgentTools: remove_used_kwargs, truncate_conve @test remove_used_kwargs(NamedTuple(), [PT.UserMessage("{{c}} {{d}}")]) == NamedTuple() end +@testset "unwrap_aicall_args" begin + + # Test with too many arguments + @test_throws AssertionError unwrap_aicall_args((1, 2, 3, 4)) + @test_throws AssertionError unwrap_aicall_args([ + "Hello", :ExampleTemplate, AITemplate(:AnotherExample)]) + + # Test with one valid String argument (UserMessage) + schema, conversation = unwrap_aicall_args(["Hello"]) + @test schema === nothing + @test length(conversation) == 1 + @test isa(conversation[1], PT.UserMessage) + @test conversation[1].content == "Hello" + + # Test with one valid Symbol argument (template-based conversion) + schema, conversation = unwrap_aicall_args([PT.OpenAISchema(), :BlankSystemUser]) + @test schema == PT.OpenAISchema() + @test length(conversation) == 2 + @test isa(conversation[1], PT.SystemMessage) + @test isa(conversation[2], PT.UserMessage) + schema, conversation = unwrap_aicall_args([ + PT.OpenAISchema(), AITemplate(:BlankSystemUser)]) + @test schema == PT.OpenAISchema() + @test length(conversation) == 2 + + # Test with an invalid argument type + @test_throws ErrorException unwrap_aicall_args([123]) + + # Test with two valid arguments in accepted combination (String and AITemplate) + schema, conversation = unwrap_aicall_args([PT.OpenAISchema(), "text"]) + @test schema == PT.OpenAISchema() + @test length(conversation) == 1 + @test isa(conversation[1], PT.UserMessage) + @test conversation[1].content == "text" +end + +@testset "extract_config" begin + # With config in kwargs + kwargs = (a = 1, b = 2, config = RetryConfig(; max_calls = 50)) + new_kwargs, config = extract_config(kwargs, RetryConfig()) + + @test config.max_calls == 50 + @test !haskey(new_kwargs, :config) + @test new_kwargs == (; a = 1, b = 2) + + # No config in kwargs + kwargs = (d = 4, e = 5) + new_kwargs, config = extract_config(kwargs, RetryConfig()) + @test config == RetryConfig() + @test new_kwargs == kwargs + + # Empty kwargs + kwargs = NamedTuple() + new_kwargs, config = extract_config(kwargs, RetryConfig()) + @test config == RetryConfig() + @test new_kwargs == NamedTuple() +end + +@testset "split_multi_samples" begin + + # Test for handling a conversation with no AIMessages + userMsg = PT.UserMessage("This is a user message.") + @test split_multi_samples([userMsg]) == [[userMsg]] + + # Test for handling a single AIMessage with no sample ID + conv = [userMsg, + AIMessage(; + content = "AI message with no sample ID", run_id = 1, sample_id = nothing)] + @test split_multi_samples(conv) == [conv] + + # Splitting conversation + conv = [PT.SystemMessage("Say hi!"), PT.SystemMessage("Hello!"), + PT.AIMessage(; content = "hi1", run_id = 1, sample_id = 1), + PT.AIMessage(; content = "hi2", run_id = 1, sample_id = 2), + ] + @test split_multi_samples(conv) == [conv[1:3], conv[[1, 2, 4]]] + + # Test for handling a conversation with only a single AIMessage sample + conv = [userMsg, + AIMessage(; content = "AI message with no sample ID", run_id = 1, sample_id = 1)] + @test split_multi_samples(conv) == [conv] + + # No AI Message + conv = [PT.SystemMessage("Say hi!"), PT.SystemMessage("Hello!"), + PT.SystemMessage("Hello!"), PT.SystemMessage("Hello!")] + @test split_multi_samples(conv) == [conv] + + # Do not change if AIMessage is not the last one + conv = [PT.SystemMessage("Say hi!"), PT.SystemMessage("Hello!"), + PT.AIMessage(; content = "hi1", run_id = 1, sample_id = 1), + PT.AIMessage(; content = "hi2", run_id = 1, sample_id = 2), + PT.SystemMessage("Hello"), + ] + @test split_multi_samples(conv) == [conv] + + # Test for handling an empty conversation + @test split_multi_samples([]) == [[]] +end + @testset "truncate_conversation" begin conversation = [ PT.SystemMessage("Hello"), @@ -56,3 +158,83 @@ end truncated = truncate_conversation(conversation, max_conversation_length = 32000) @test isempty(truncated) end + +@testset "beta_sample,gamma_sample" begin + N = 1000 + tolerance_mean = 0.05 # Tolerance for mean comparison + tolerance_variance = 0.02 # A tighter tolerance for variance, adjust based on observed precision + + # Test 1: Alpha and Beta are integers > 1 + α, β = 2, 3 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + samples = [beta_sample(α, β) for _ in 1:N] + sample_mean = mean(samples) + sample_variance = var(samples, corrected = true) + @test abs(sample_mean - expected_mean) < tolerance_mean + @test abs(sample_variance - expected_variance) < tolerance_variance + + # Test 2: Alpha and Beta are large integers + α, β = 10, 10 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance + + # Test 3: Alpha and Beta are floats > 1 + α, β = 2.5, 3.5 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance + + # Test 4: Alpha < 1 and Beta > 1 + α, β = 0.5, 5 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance + + # Test 5: Alpha > 1 and Beta < 1 + α, β = 5, 0.5 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance + + # Test 6: Alpha and Beta are both < 1 + α, β = 0.5, 0.5 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance + + # Test 7: Alpha = 1 and Beta = 1 (Uniform distribution) + α, β = 1, 1 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance + + # Test 8: Very small Alpha and Beta + α, β = 0.1, 0.1 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance + + # Test 9: Very large Alpha and Beta + α, β = 100, 100 + expected_mean = α / (α + β) + expected_variance = (α * β) / ((α + β)^2 * (α + β + 1)) + sample_values = [beta_sample(α, β) for _ in 1:N] + @test abs(mean(sample_values) - expected_mean) < tolerance_mean + @test abs(var(sample_values, corrected = true) - expected_variance) < tolerance_variance +end diff --git a/test/llm_openai.jl b/test/llm_openai.jl index 71beb0ef2..1b440f55d 100644 --- a/test/llm_openai.jl +++ b/test/llm_openai.jl @@ -662,4 +662,4 @@ end api_kwargs = (; temperature = 0, n = 2)) @test conv[end - 1].content == "Hello1!" @test conv[end].content == "Hello2!" -end \ No newline at end of file +end diff --git a/test/runtests.jl b/test/runtests.jl index 15503a401..5016b47d9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,7 @@ using PromptingTools using OpenAI, HTTP, JSON3 using SparseArrays, LinearAlgebra, Markdown +using Statistics using Test, Pkg const PT = PromptingTools using Aqua From 8b640e1349e43b5aed7fae339767519932e66450 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 26 Feb 2024 20:47:17 +0000 Subject: [PATCH 124/251] update deps --- CHANGELOG.md | 5 ++++- Project.toml | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2d831fc6..48d9e4c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.13.0] ### Added -- Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). +- Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). It must be imported explicitly because it's not registered yet. - Added a utility to compare any two string sequences (and other iterators)`length_longest_common_subsequence`. It can be used to fuzzy match strings (eg, detecting context/sources in an AI-generated response or fuzzy matching AI response to some preset categories). See the docstring for more information `?length_longest_common_subsequence`. - Rewrite of `aiclassify` to classify into an arbitrary list of categories (including with descriptions). It's a quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick and outputs only 1 token. Currently, only `OpenAISchema` is supported. See `?aiclassify` for more information. - Initial support for multiple completions in one request for OpenAI-compatible API servers. Set via API kwarg `n=5` and it will request 5 completions in one request, saving the network communication time and paying the prompt tokens only once. It's useful for majority voting, diversity, or challenging agentic workflows. @@ -30,6 +30,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Updated - Updated names of endpoints and prices of Mistral.ai models as per the [latest announcement](https://mistral.ai/technology/#models) and [pricing](https://docs.mistral.ai/platform/pricing/). Eg, `mistral-small` -> `mistral-small-latest`. In addition, the latest Mistral model has been added `mistral-large-latest` (aliased as `mistral-large` and `mistrall`, same for the others). `mistral-small-latest` and `mistral-large-latest` now support function calling, which means they will work with `aiextract` (You need to explicitly provide `tool_choice`, see the docs `?aiextract`). +## Removed +- Removed package extension for GoogleGenAI.jl, as it's not yet registered. Users must load the code manually for now. + ## [0.12.0] ### Added diff --git a/Project.toml b/Project.toml index d774f22bc..95dc8ce2d 100644 --- a/Project.toml +++ b/Project.toml @@ -17,13 +17,11 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [weakdeps] -GoogleGenAI = "903d41d1-eaca-47dd-943b-fee3930375ab" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] -GoogleGenAIPromptingToolsExt = ["GoogleGenAI"] MarkdownPromptingToolsExt = ["Markdown"] RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] @@ -31,7 +29,6 @@ RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] AbstractTrees = "0.4" Aqua = "0.7" Base64 = "<0.0.1, 1" -GoogleGenAI = "0.1.0" HTTP = "1" JSON3 = "1" LinearAlgebra = "<0.0.1, 1" From a6af64fe6b2d84795cc6c06ee721caccfb1e2c06 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 26 Feb 2024 20:47:34 +0000 Subject: [PATCH 125/251] Revert "update deps" This reverts commit 8b640e1349e43b5aed7fae339767519932e66450. --- CHANGELOG.md | 5 +---- Project.toml | 3 +++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48d9e4c66..d2d831fc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.13.0] ### Added -- Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). It must be imported explicitly because it's not registered yet. +- Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). - Added a utility to compare any two string sequences (and other iterators)`length_longest_common_subsequence`. It can be used to fuzzy match strings (eg, detecting context/sources in an AI-generated response or fuzzy matching AI response to some preset categories). See the docstring for more information `?length_longest_common_subsequence`. - Rewrite of `aiclassify` to classify into an arbitrary list of categories (including with descriptions). It's a quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick and outputs only 1 token. Currently, only `OpenAISchema` is supported. See `?aiclassify` for more information. - Initial support for multiple completions in one request for OpenAI-compatible API servers. Set via API kwarg `n=5` and it will request 5 completions in one request, saving the network communication time and paying the prompt tokens only once. It's useful for majority voting, diversity, or challenging agentic workflows. @@ -30,9 +30,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Updated - Updated names of endpoints and prices of Mistral.ai models as per the [latest announcement](https://mistral.ai/technology/#models) and [pricing](https://docs.mistral.ai/platform/pricing/). Eg, `mistral-small` -> `mistral-small-latest`. In addition, the latest Mistral model has been added `mistral-large-latest` (aliased as `mistral-large` and `mistrall`, same for the others). `mistral-small-latest` and `mistral-large-latest` now support function calling, which means they will work with `aiextract` (You need to explicitly provide `tool_choice`, see the docs `?aiextract`). -## Removed -- Removed package extension for GoogleGenAI.jl, as it's not yet registered. Users must load the code manually for now. - ## [0.12.0] ### Added diff --git a/Project.toml b/Project.toml index 95dc8ce2d..d774f22bc 100644 --- a/Project.toml +++ b/Project.toml @@ -17,11 +17,13 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [weakdeps] +GoogleGenAI = "903d41d1-eaca-47dd-943b-fee3930375ab" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] +GoogleGenAIPromptingToolsExt = ["GoogleGenAI"] MarkdownPromptingToolsExt = ["Markdown"] RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] @@ -29,6 +31,7 @@ RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] AbstractTrees = "0.4" Aqua = "0.7" Base64 = "<0.0.1, 1" +GoogleGenAI = "0.1.0" HTTP = "1" JSON3 = "1" LinearAlgebra = "<0.0.1, 1" From 843ab957afbccac019752452d451fd8948d45d44 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Mon, 26 Feb 2024 20:49:36 +0000 Subject: [PATCH 126/251] remove GoogleGenAI (#83) --- CHANGELOG.md | 5 ++++- Project.toml | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2d831fc6..c841ce728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.13.0] ### Added -- Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). +- Added initial support for Google Gemini models for `aigenerate` (requires environment variable `GOOGLE_API_KEY` and package [GoogleGenAI.jl](https://github.com/tylerjthomas9/GoogleGenAI.jl) to be loaded). It must be added explicitly as it is not yet registered. - Added a utility to compare any two string sequences (and other iterators)`length_longest_common_subsequence`. It can be used to fuzzy match strings (eg, detecting context/sources in an AI-generated response or fuzzy matching AI response to some preset categories). See the docstring for more information `?length_longest_common_subsequence`. - Rewrite of `aiclassify` to classify into an arbitrary list of categories (including with descriptions). It's a quick and easy option for "routing" and similar use cases, as it exploits the logit bias trick and outputs only 1 token. Currently, only `OpenAISchema` is supported. See `?aiclassify` for more information. - Initial support for multiple completions in one request for OpenAI-compatible API servers. Set via API kwarg `n=5` and it will request 5 completions in one request, saving the network communication time and paying the prompt tokens only once. It's useful for majority voting, diversity, or challenging agentic workflows. @@ -30,6 +30,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Updated - Updated names of endpoints and prices of Mistral.ai models as per the [latest announcement](https://mistral.ai/technology/#models) and [pricing](https://docs.mistral.ai/platform/pricing/). Eg, `mistral-small` -> `mistral-small-latest`. In addition, the latest Mistral model has been added `mistral-large-latest` (aliased as `mistral-large` and `mistrall`, same for the others). `mistral-small-latest` and `mistral-large-latest` now support function calling, which means they will work with `aiextract` (You need to explicitly provide `tool_choice`, see the docs `?aiextract`). +## Removed +- Removed package extension for GoogleGenAI.jl, as it's not yet registered. Users must load the code manually for now. + ## [0.12.0] ### Added diff --git a/Project.toml b/Project.toml index d774f22bc..95dc8ce2d 100644 --- a/Project.toml +++ b/Project.toml @@ -17,13 +17,11 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [weakdeps] -GoogleGenAI = "903d41d1-eaca-47dd-943b-fee3930375ab" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] -GoogleGenAIPromptingToolsExt = ["GoogleGenAI"] MarkdownPromptingToolsExt = ["Markdown"] RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] @@ -31,7 +29,6 @@ RAGToolsExperimentalExt = ["SparseArrays", "LinearAlgebra"] AbstractTrees = "0.4" Aqua = "0.7" Base64 = "<0.0.1, 1" -GoogleGenAI = "0.1.0" HTTP = "1" JSON3 = "1" LinearAlgebra = "<0.0.1, 1" From a41a90811f9989cc63a5153fba2b51208cd0b434 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Tue, 27 Feb 2024 09:28:01 +0000 Subject: [PATCH 127/251] fix docs --- src/Experimental/AgentTools/retry.jl | 39 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/Experimental/AgentTools/retry.jl b/src/Experimental/AgentTools/retry.jl index 7e2099258..27dac99ce 100644 --- a/src/Experimental/AgentTools/retry.jl +++ b/src/Experimental/AgentTools/retry.jl @@ -4,17 +4,16 @@ verbose::Bool = true, throw::Bool = false, evaluate_all::Bool = true, feedback_expensive::Bool = false, max_retries::Union{Nothing, Int} = nothing, retry_delay::Union{Nothing, Int} = nothing) -Evaluates the condition `f_cond` on the `aicall` object (eg, we evaluate `f_cond(aicall) -> Bool`). -If the condition is not met, it will return the best sample to retry from and provide `feedback` to `aicall`. That's why it's mutating. -It will retry running the `aicall` `max_retries` times. -If `throw` is `true`, it will throw an error if the function does not return `true` after `max_retries` retries. +Evaluates the condition `f_cond` on the `aicall` object. +If the condition is not met, it will return the best sample to retry from and provide `feedback` (string or function) to `aicall`. That's why it's mutating. +It will retry maximum `max_retries` times, with `throw=true`, an error will be thrown if the condition is not met after `max_retries` retries. -If feedback is provided (not empty), it will be append it to the conversation before the retry. -If a function is provided, it must accept the `aicall` object as the only argument and return a string. +Function signatures +- `f_cond(aicall::AICallBlock) -> Bool`, ie, it must accept the aicall object and return a boolean value. +- `feedback` can be a string or `feedback(aicall::AICallBlock) -> String`, ie, it must accept the aicall object and return a string. -Function `f_cond` is expected to accept the `aicall` object as the only argument. -It must return a boolean value, which indicates whether the condition is met. -You can leverage the `last_message`, `last_output`, and `AICode` functions to access the last message, last output and code blocks in the conversation, respectively. +You can leverage the `last_message`, `last_output`, and `AICode` functions to access the last message, last output and execute code blocks in the conversation, respectively. +See examples below. # Good Use Cases - Retry with API failures/drops (add `retry_delay=2` to wait 2s between retries) @@ -62,6 +61,7 @@ run!(out) # fails airetry!(isvalid, out; retry_delay = 2, max_retries = 2) ``` + If you provide arguments to the aicall, we try to honor them as much as possible in the following calls, eg, set low verbosity ```julia @@ -71,6 +71,7 @@ run!(out) # No info message, you just see `success = false` in the properties of the AICall ``` + Let's show a toy example to demonstrate the runtime checks / guardrails for the model output. We'll play a color guessing game (I'm thinking "yellow"): @@ -84,25 +85,30 @@ out = AIGenerate( config = RetryConfig(; n_samples = 2), api_kwargs = (; n = 2)) run!(out) + ## Check that the output is 1 word only, third argument is the feedback that will be provided if the condition fails ## Notice: functions operate on `aicall` as the only argument. We can use utilities like `last_output` and `last_message` to access the last message and output in the conversation. airetry!(x -> length(split(last_output(x), r" |\\.")) == 1, out, "You must answer with 1 word only.") + ## Let's ensure that the output is in lowercase - simple and short airetry!(x -> all(islowercase, last_output(x)), out, "You must answer in lowercase.") # [ Info: Condition not met. Retrying... + ## Let's add final hint - it took us 2 retries airetry!(x -> startswith(last_output(x), "y"), out, "It starts with \"y\"") # [ Info: Condition not met. Retrying... # [ Info: Condition not met. Retrying... + ## We end up with the correct answer last_output(out) # Output: "yellow" ``` + Let's explore how we got here. We save the various attempts in a "tree" (SampleNode object) You can access it in `out.samples`, which is the ROOT of the tree (top level). @@ -169,8 +175,9 @@ Note: `airetry!` will attempt to fix the model `max_retries` times. If you set `throw=true`, it will throw an ErrorException if the condition is not met after `max_retries` retries. + +Let's define a mini program to guess the number and use `airetry!` to guide the model to the correct answer: ```julia -# Let's define a mini program to guess the number \"\"\" llm_guesser() @@ -264,7 +271,7 @@ end ``` Note that if there are multiple "branches" the model will see only the feedback of its own and its ancestors not the other "branches". -If you want to show all object, set `n_samples=1`, so all fixing happens sequantially and model sees all feedback (less powerful if model falls into a bad state). +If you wanted to provide ALL feedback, set `RetryConfig(; n_samples=1)` to remove any "branching". It fixing will be done sequentially in one conversation and the model will see all feedback (less powerful if the model falls into a bad state). Alternatively, you can tweak the feedback function. # See Also @@ -474,25 +481,25 @@ Adds formatted feedback to the `conversation` based on the `sample` node feedbac sample = SampleNode(; data = nothing, feedback = "Feedback X") conversation = [PT.UserMessage("I say hi!"), PT.AIMessage(; content = "I say hi!")] conversation = AT.add_feedback!(conversation, sample) -conversation[end].content == "### Feedback from Evaluator\nFeedback X\n" +conversation[end].content == "### Feedback from Evaluator\\nFeedback X\\n" Inplace feedback: ```julia conversation = [PT.UserMessage("I say hi!"), PT.AIMessage(; content = "I say hi!")] conversation = AT.add_feedback!(conversation, sample; feedback_inplace = true) -conversation[end].content == "I say hi!\n\n### Feedback from Evaluator\nFeedback X\n" +conversation[end].content == "I say hi!\\n\\n### Feedback from Evaluator\\nFeedback X\\n" ``` Sample with ancestors with feedback: ```julia -sample_p = SampleNode(; data = nothing, feedback = "\nFeedback X") +sample_p = SampleNode(; data = nothing, feedback = "\\nFeedback X") sample = expand!(sample_p, nothing) -sample.feedback = "\nFeedback Y" +sample.feedback = "\\nFeedback Y" conversation = [PT.UserMessage("I say hi!"), PT.AIMessage(; content = "I say hi!")] conversation = AT.add_feedback!(conversation, sample) conversation[end].content == -"### Feedback from Evaluator\n\nFeedback X\n----------\n\nFeedback Y\n" +"### Feedback from Evaluator\\n\\nFeedback X\\n----------\\n\\nFeedback Y\\n" ``` """ function add_feedback!(conversation::AbstractVector{<:PT.AbstractMessage}, From 41535183ed8506a05d23eab2ed7b4114199e825b Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Wed, 28 Feb 2024 21:20:05 +0000 Subject: [PATCH 128/251] Templating utilities (#84) --- CHANGELOG.md | 3 + Project.toml | 2 +- docs/make.jl | 9 +- docs/src/frequently_asked_questions.md | 172 +++++------- docs/src/getting_started.md | 4 + docs/src/how_it_works.md | 367 +++++++++++++++++++++++++ src/PromptingTools.jl | 3 +- src/templates.jl | 239 ++++++++++++---- test/templates.jl | 40 ++- test/utils.jl | 6 +- 10 files changed, 668 insertions(+), 177 deletions(-) create mode 100644 docs/src/how_it_works.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c841ce728..a592a8a17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added a new documentation section "How it works" to explain the inner workings of the package. It's a work in progress, but it should give you a good idea of what's happening under the hood. +- Improved template loading, so if you load your custom templates once with `load_templates!("my/template/folder)`, it will remember your folder for all future re-loads. +- Added convenience function `create_template` to create templates on the fly without having to deal with `PT.UserMessage` etc. See `?create_template` for more information. ### Fixed diff --git a/Project.toml b/Project.toml index 95dc8ce2d..da0c0e797 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.13.0" +version = "0.14.0-DEV" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" diff --git a/docs/make.jl b/docs/make.jl index e5ca9f168..eeb1cc6d4 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -15,7 +15,7 @@ makedocs(; modules = [ PromptingTools, PromptingTools.Experimental.RAGTools, - PromptingTools.Experimental.AgentTools, + PromptingTools.Experimental.AgentTools ], authors = "J S <49557684+svilupp@users.noreply.github.com> and contributors", repo = "https://github.com/svilupp/PromptingTools.jl/blob/{commit}{path}#{line}", @@ -30,13 +30,14 @@ makedocs(; pages = [ "Home" => "index.md", "Getting Started" => "getting_started.md", + "How It Works" => "how_it_works.md", "Examples" => [ "Various examples" => "examples/readme_examples.md", "Using AITemplates" => "examples/working_with_aitemplates.md", "Local models with Ollama.ai" => "examples/working_with_ollama.md", "Google AIStudio" => "examples/working_with_google_ai_studio.md", "Custom APIs (Mistral, Llama.cpp)" => "examples/working_with_custom_apis.md", - "Building RAG Application" => "examples/building_RAG.md", + "Building RAG Application" => "examples/building_RAG.md" ], "F.A.Q." => "frequently_asked_questions.md", "Reference" => [ @@ -44,8 +45,8 @@ makedocs(; "Experimental Modules" => "reference_experimental.md", "RAGTools" => "reference_ragtools.md", "AgentTools" => "reference_agenttools.md", - "APITools" => "reference_apitools.md", - ], + "APITools" => "reference_apitools.md" + ] ]) deploydocs(; diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md index 5f34172c3..923b3e915 100644 --- a/docs/src/frequently_asked_questions.md +++ b/docs/src/frequently_asked_questions.md @@ -201,134 +201,90 @@ conversation = aigenerate("What's my name?"; return_all=true, conversation) ``` Notice that the last message is the response to the second request, but with `return_all=true` we can see the whole conversation from the beginning. -## Explain What Happens Under the Hood +## How to have typed responses? -4 Key Concepts/Objects: -- Schemas -> object of type `AbstractPromptSchema` that determines which methods are called and, hence, what providers/APIs are used -- Prompts -> the information you want to convey to the AI model -- Messages -> the basic unit of communication between the user and the AI model (eg, `UserMessage` vs `AIMessage`) -- Prompt Templates -> re-usable "prompts" with placeholders that you can replace with your inputs at the time of making the request +Our responses are always in `AbstractMessage` types to ensure we can also handle downstream processing, error handling, and self-healing code (see `airetry!`). -When you call `aigenerate`, roughly the following happens: `render` -> `UserMessage`(s) -> `render` -> `OpenAI.create_chat` -> ... -> `AIMessage`. - -We'll deep dive into an example in the end. - -### Schemas - -For your "message" to reach an AI model, it needs to be formatted and sent to the right place. - -We leverage the multiple dispatch around the "schemas" to pick the right logic. -All schemas are subtypes of `AbstractPromptSchema` and there are many subtypes, eg, `OpenAISchema <: AbstractOpenAISchema <:AbstractPromptSchema`. - -For example, if you provide `schema = OpenAISchema()`, the system knows that: -- it will have to format any user inputs to OpenAI's "message specification" (a vector of dictionaries, see their API documentation). Function `render(OpenAISchema(),...)` will take care of the rendering. -- it will have to send the message to OpenAI's API. We will use the amazing `OpenAI.jl` package to handle the communication. - -### Prompts - -Prompt is loosely the information you want to convey to the AI model. It can be a question, a statement, or a command. It can have instructions or some context, eg, previous conversation. - -You need to remember that Large Language Models (LLMs) are **stateless**. They don't remember the previous conversation/request, so you need to provide the whole history/context every time (similar to how REST APIs work). - -Prompts that we send to the LLMs are effectively a sequence of messages (`<:AbstractMessage`). - -### Messages - -Messages are the basic unit of communication between the user and the AI model. - -There are 5 main types of messages (`<:AbstractMessage`): - -- `SystemMessage` - this contains information about the "system", eg, how it should behave, format its output, etc. (eg, `You're a world-class Julia programmer. You write brief and concise code.) -- `UserMessage` - the information "from the user", ie, your question/statement/task -- `UserMessageWithImages` - the same as `UserMessage`, but with images (URLs or Base64-encoded images) -- `AIMessage` - the response from the AI model, when the "output" is text -- `DataMessage` - the response from the AI model, when the "output" is data, eg, embeddings with `aiembed` or user-defined structs with `aiextract` - -### Prompt Templates - -We want to have re-usable "prompts", so we provide you with a system to retrieve pre-defined prompts with placeholders (eg, `{{name}}`) that you can replace with your inputs at the time of making the request. - -"AI Templates" as we call them (`AITemplate`) are usually a vector of `SystemMessage` and a `UserMessage` with specific purpose/task. - -For example, the template `:AssistantAsk` is defined loosely as: +A good use case for a typed response is when you have a complicated control flow and would like to group and handle certain outcomes differently. You can easily do it as an extra step after the response is received. +Trivially, we can use `aiclassifier` for Bool statements, eg, ```julia - template = [SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer."), - UserMessage("# Question\n\n{{ask}}")] -``` - -Notice that we have a placeholder `ask` (`{{ask}}`) that you can replace with your question without having to re-write the generic system instructions. - -When you provide a Symbol (eg, `:AssistantAsk`) to ai* functions, thanks to the multiple dispatch, it recognizes that it's an `AITemplate(:AssistantAsk)` and looks it up. +# We can do either +mybool = tryparse(Bool, aiclassify("Is two plus two four?")) isa Bool # true -You can discover all available templates with `aitemplates("some keyword")` or just see the details of some template `aitemplates(:AssistantAsk)`. - -### Walkthrough Example - -```julia -using PromptingTools -const PT = PromptingTools - -# Let's say this is our ask -msg = aigenerate(:AssistantAsk; ask="What is the capital of France?") - -# it is effectively the same as: -msg = aigenerate(PT.OpenAISchema(), PT.AITemplate(:AssistantAsk); ask="What is the capital of France?", model="gpt3t") +# or simply check equality +msg = aiclassify("Is two plus two four?") # true +mybool = msg.content == "true" ``` -There is no `model` provided, so we use the default `PT.MODEL_CHAT` (effectively GPT3.5-Turbo). Then we look it up in `PT.MDOEL_REGISTRY` and use the associated schema for it (`OpenAISchema` in this case). - -The next step is to render the template, replace the placeholders and render it for the OpenAI model. - +Now a more complicated example with multiple categories mapping to an enum: ```julia -# Let's remember out schema -schema = PT.OpenAISchema() -ask = "What is the capital of France?" -``` +choices = [("A", "any animal or creature"), ("P", "for any plant or tree"), ("O", "for everything else")] -First, we obtain the template (no placeholder replacement yet) and "expand it" -```julia -template_rendered = PT.render(schema, AITemplate(:AssistantAsk); ask) -``` +# Set up the return types we want +@enum Categories A P O +string_to_category = Dict("A" => A, "P" => P,"O" => O) -```plaintext -2-element Vector{PromptingTools.AbstractChatMessage}: - PromptingTools.SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") - PromptingTools.UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) -``` +# Run an example +input = "spider" +msg = aiclassify(:InputClassifier; choices, input) -Second, we replace the placeholders -```julia -rendered_for_api = PT.render(schema, template_rendered; ask) -``` - -```plaintext -2-element Vector{Dict{String, Any}}: - Dict("role" => "system", "content" => "You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") - Dict("role" => "user", "content" => "# Question\n\nWhat is the capital of France?") +mytype = string_to_category[msg.content] # A (for animal) ``` +How does it work? `aiclassify` guarantees to output one of our choices (and it handles some of the common quirks)! -Notice that the placeholders are only replaced in the second step. The final output here is a vector of messages with "role" and "content" keys, which is the format required by the OpenAI API. +How would we achieve the same with `aigenerate` and arbitrary struct? +We need to use the "lazy" `AIGenerate` struct and `airetry!` to ensure we get the response and then we can process it further. -As a side note, under the hood, the second step is done in two steps: +`AIGenerate` has two fields you should know about: +- `conversation` - eg, the vector of "messages" in the current conversation (same as what you get from `aigenerate` with `return_all=true`) +- `success` - a boolean flag if the request was successful AND if it passed any subsequent `airetry!` calls -- replace the placeholders `messages_rendered = PT.render(PT.NoSchema(), template_rendered; ask)` -> returns a vector of Messages! -- then, we convert the messages to the format required by the provider/schema `PT.render(schema, messages_rendered)` -> returns the OpenAI formatted messages - - -Next, we send the above `rendered_for_api` to the OpenAI API and get the response back. +Let's mimic a case where our "program" should return one of three types: `SmallInt`, `LargeInt`, `FailedResponse`. +We first need to define our custom types: ```julia -using OpenAI -OpenAI.create_chat(api_key, model, rendered_for_api) + +# not needed, just to show a fully typed example +abstract type MyAbstractResponse end +struct SmallInt <: MyAbstractResponse + number::Int +end +struct LargeInt <: MyAbstractResponse + number::Int +end +struct FailedResponse <: MyAbstractResponse + content::String +end ``` -The last step is to take the JSON response from the API and convert it to the `AIMessage` object. +Let's define our "program" as a function to be cleaner. Notice that we use `AIGenerate` and `airetry!` to ensure we get the response and then we can process it further. ```julia -# simplification for educational purposes -msg = AIMessage(; content = r.response[:choices][1][:message][:content]) +using PromptingTools.Experimental.AgentTools + +function give_me_number(prompt::String)::MyAbstractResponse + # Generate the response + response = AIGenerate(prompt; config=RetryConfig(;max_retries=2)) |> run! + + # Check if it's parseable as Int, if not, send back to be fixed + # syntax: airetry!(CONDITION-TO-CHECK, , FEEDBACK-TO-MODEL) + airetry!(x->tryparse(Int,last_output(x))|>!isnothing, response, "Wrong output format! Answer with digits and nothing else. The number is:") + + if response.success != true + ## we failed to generate a parseable integer + return FailedResponse("I failed to get the response. Last output: $(last_output(response))") + end + number = tryparse(Int,last_output(response)) + return number < 1000 ? SmallInt(number) : LargeInt(number) +end + +give_me_number("How many car seats are in Porsche 911T?") +## [ Info: Condition not met. Retrying... +## [ Info: Condition not met. Retrying... +## SmallInt(2) ``` -In practice, there are more fields we extract, so we define a utility for it: `PT.response_to_message`. Especially, since with parameter `n`, you can request multiple AI responses at once, so we want to re-use our response processing logic. -That's it! I hope you've learned something new about how PromptingTools.jl works under the hood. \ No newline at end of file +We ultimately received our custom type `SmallInt` with the number of car seats in the Porsche 911T (I hope it's correct!). + +If you want to access the full conversation history (all the attempts and feedback), simply output the `response` object and explore `response.conversation`. \ No newline at end of file diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index bb25e667c..c445ca7d1 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -1,3 +1,7 @@ +```@meta +CurrentModule = PromptingTools +``` + # Getting Started ## Prerequisites diff --git a/docs/src/how_it_works.md b/docs/src/how_it_works.md new file mode 100644 index 000000000..ad9a4dec0 --- /dev/null +++ b/docs/src/how_it_works.md @@ -0,0 +1,367 @@ +```@meta +CurrentModule = PromptingTools +``` + +# How It Works + +This is an advanced section that explains how PromptingTools.jl works under the hood. It is not necessary to understand this to use the package, but it can be helpful for debugging and understanding the limitations of the package. + +We'll start with the key concepts and then walk through an example of `aigenerate` to see how it all fits together. + +## Key Concepts + +5 Key Concepts (/Objects): +- API/Model Providers -> The method that gives you access to Large Language Models (LLM), it can be an API (eg, OpenAI) or a locally-hosted application (eg, Llama.cpp or Ollama) +- Schemas -> object of type `AbstractPromptSchema` that determines which methods are called and, hence, what providers/APIs are used +- Prompts -> the information you want to convey to the AI model +- Messages -> the basic unit of communication between the user and the AI model (eg, `UserMessage` vs `AIMessage`) +- Prompt Templates -> re-usable "prompts" with placeholders that you can replace with your inputs at the time of making the request + +When you call `aigenerate`, roughly the following happens: `render` -> `UserMessage`(s) -> `render` -> `OpenAI.create_chat` -> ... -> `AIMessage`. + +### API/Model Providers + +You can think of "API/Model Providers" as the method that gives you access to Large Language Models (LLM). It can be an API (eg, OpenAI) or a locally-hosted application (eg, Llama.cpp or Ollama). + +You interact with them via the `schema` object, which is a subtype of `AbstractPromptSchema`, +eg, there is an `OpenAISchema` for the provider "OpenAI" and its supertype `AbstractOpenAISchema` is for all other providers that mimic the OpenAI API. + +### Schemas + +For your "message" to reach an AI model, it needs to be formatted and sent to the right place (-> provider!). + +We leverage the multiple dispatch around the "schemas" to pick the right logic. +All schemas are subtypes of `AbstractPromptSchema` and there are many subtypes, eg, `OpenAISchema <: AbstractOpenAISchema <:AbstractPromptSchema`. + +For example, if you provide `schema = OpenAISchema()`, the system knows that: +- it will have to format any user inputs to OpenAI's "message specification" (a vector of dictionaries, see their API documentation). Function `render(OpenAISchema(),...)` will take care of the rendering. +- it will have to send the message to OpenAI's API. We will use the amazing `OpenAI.jl` package to handle the communication. + +### Prompts + +Prompt is loosely the information you want to convey to the AI model. It can be a question, a statement, or a command. It can have instructions or some context, eg, previous conversation. + +You need to remember that Large Language Models (LLMs) are **stateless**. They don't remember the previous conversation/request, so you need to provide the whole history/context every time (similar to how REST APIs work). + +Prompts that we send to the LLMs are effectively a sequence of messages (`<:AbstractMessage`). + +### Messages + +Messages are the basic unit of communication between the user and the AI model. + +There are 5 main types of messages (`<:AbstractMessage`): + +- `SystemMessage` - this contains information about the "system", eg, how it should behave, format its output, etc. (eg, `You're a world-class Julia programmer. You write brief and concise code.) +- `UserMessage` - the information "from the user", ie, your question/statement/task +- `UserMessageWithImages` - the same as `UserMessage`, but with images (URLs or Base64-encoded images) +- `AIMessage` - the response from the AI model, when the "output" is text +- `DataMessage` - the response from the AI model, when the "output" is data, eg, embeddings with `aiembed` or user-defined structs with `aiextract` + +### Prompt Templates + +We want to have re-usable "prompts", so we provide you with a system to retrieve pre-defined prompts with placeholders (eg, `{{name}}`) that you can replace with your inputs at the time of making the request. + +"AI Templates" as we call them (`AITemplate`) are usually a vector of `SystemMessage` and a `UserMessage` with specific purpose/task. + +For example, the template `:AssistantAsk` is defined loosely as: + +```julia + template = [SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer."), + UserMessage("# Question\n\n{{ask}}")] +``` + +Notice that we have a placeholder `ask` (`{{ask}}`) that you can replace with your question without having to re-write the generic system instructions. + +When you provide a Symbol (eg, `:AssistantAsk`) to ai* functions, thanks to the multiple dispatch, it recognizes that it's an `AITemplate(:AssistantAsk)` and looks it up. + +You can discover all available templates with `aitemplates("some keyword")` or just see the details of some template `aitemplates(:AssistantAsk)`. + +## ai* Functions + +The above steps are implemented in the `ai*` functions, eg, `aigenerate`, `aiembed`, `aiextract`, etc. They all have the same basic structure: + +`ai*(,; )`, + +but they differ in purpose: + +- `aigenerate` is the general-purpose function to generate any text response with LLMs, ie, it returns `AIMessage` with field `:content` containing the generated text (eg, `ans.content isa AbstractString`) +- `aiembed` is designed to extract embeddings from the AI model's response, ie, it returns `DataMessage` with field `:content` containing the embeddings (eg, `ans.content isa AbstractArray`) +- `aiextract` is designed to extract structured data from the AI model's response and return them as a Julia struct (eg, if we provide `return_type=Food`, we get `ans.content isa Food`). You need to define the return type first and then provide it as a keyword argument. +- `aiclassify` is designed to classify the input text into (or simply respond within) a set of discrete `choices` provided by the user. It can be very useful as an LLM Judge or a router for RAG systems, as it uses the "logit bias trick" and generates exactly 1 token. It returns `AIMessage` with field `:content`, but the `:content` can be only one of the provided `choices` (eg, `ans.content in choices`) +- `aiscan` is for working with images and vision-enabled models (as an input), but it returns `AIMessage` with field `:content` containing the generated text (eg, `ans.content isa AbstractString`) similar to `aigenerate`. +- `aitemplates` is a helper function to discover available templates and see their details (eg, `aitemplates("some keyword")` or `aitemplates(:AssistantAsk)`) + +In addition to the above list, you can also use the **"lazy" counterparts** of these functions from the experimental AgentTools module. +```julia +using PromptingTools.Experimental.AgentTools +``` + +For example, `AIGenerate()` will create a lazy instance of `aigenerate`. It is an instance of `AICall` with `aigenerate` as its ai function. +It uses exactly the same arguments and keyword arguments as `aigenerate` (see `?aigenerate` for details). + +"lazy" refers to the fact that it does NOT generate any output when instantiated (only when `run!` is called). + +Or said differently, the `AICall` struct and all its flavors (`AIGenerate`, ...) are designed to facilitate a deferred execution model (lazy evaluation) for AI functions that interact with a Language Learning Model (LLM). It stores the necessary information for an AI call and executes the underlying AI function only when supplied with a `UserMessage` or when the `run!` method is applied. + +This approach allows us to remember user inputs and trigger the LLM call repeatedly if needed, which enables automatic fixing (see `?airetry!`). + +Example: +```julia +result = AIGenerate(:JuliaExpertAsk; ask="xyz", model="abc", api_kwargs=(; temperature=0.1)) +result |> run! + +# Is equivalent to +result = aigenerate(:JuliaExpertAsk; ask="xyz", model="abc", api_kwargs=(; temperature=0.1), return_all=true) +# The only difference is that we default to `return_all=true` with lazy types because we have a dedicated `conversation` field, which makes it much easier +``` + +Lazy AI calls and self-healing mechanisms unlock much more robust and useful LLM workflows! + +## Walkthroughs + +### Walkthrough Example for `aigenerate` + +```julia +using PromptingTools +const PT = PromptingTools + +# Let's say this is our ask +msg = aigenerate(:AssistantAsk; ask="What is the capital of France?") + +# it is effectively the same as: +msg = aigenerate(PT.OpenAISchema(), PT.AITemplate(:AssistantAsk); ask="What is the capital of France?", model="gpt3t") +``` + +There is no `model` provided, so we use the default `PT.MODEL_CHAT` (effectively GPT3.5-Turbo). Then we look it up in `PT.MDOEL_REGISTRY` and use the associated schema for it (`OpenAISchema` in this case). + +The next step is to render the template, replace the placeholders and render it for the OpenAI model. + +```julia +# Let's remember out schema +schema = PT.OpenAISchema() +ask = "What is the capital of France?" +``` + +First, we obtain the template (no placeholder replacement yet) and "expand it" +```julia +template_rendered = PT.render(schema, AITemplate(:AssistantAsk); ask) +``` + +```plaintext +2-element Vector{PromptingTools.AbstractChatMessage}: + PromptingTools.SystemMessage("You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + PromptingTools.UserMessage{String}("# Question\n\n{{ask}}", [:ask], :usermessage) +``` + +Second, we replace the placeholders +```julia +rendered_for_api = PT.render(schema, template_rendered; ask) +``` + +```plaintext +2-element Vector{Dict{String, Any}}: + Dict("role" => "system", "content" => "You are a world-class AI assistant. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.") + Dict("role" => "user", "content" => "# Question\n\nWhat is the capital of France?") +``` + +Notice that the placeholders are only replaced in the second step. The final output here is a vector of messages with "role" and "content" keys, which is the format required by the OpenAI API. + +As a side note, under the hood, the second step is done in two sub-steps: + +- replace the placeholders `messages_rendered = PT.render(PT.NoSchema(), template_rendered; ask)` -> returns a vector of Messages! +- then, we convert the messages to the format required by the provider/schema `PT.render(schema, messages_rendered)` -> returns the OpenAI formatted messages + +Next, we send the above `rendered_for_api` to the OpenAI API and get the response back. + +```julia +using OpenAI +OpenAI.create_chat(api_key, model, rendered_for_api) +``` + +The last step is to take the JSON response from the API and convert it to the `AIMessage` object. + +```julia +# simplification for educational purposes +msg = AIMessage(; content = r.response[:choices][1][:message][:content]) +``` + +In practice, there are more fields we extract, so we define a utility for it: `PT.response_to_message`. Especially, since with parameter `n`, you can request multiple AI responses at once, so we want to re-use our response processing logic. + +That's it! I hope you've learned something new about how PromptingTools.jl works under the hood. + +## Walkthrough Example for `aiextract` + +Whereas `aigenerate` is a general-purpose function to generate any text response with LLMs, `aiextract` is designed to extract structured data from the AI model's response and return them as a Julia struct. + +It's a bit more complicated than `aigenerate` because it needs to handle the JSON schema of the return type (= our struct). + +Let's define a toy example of a struct and see how `aiextract` works under the hood. +```julia +using PromptingTools +const PT = PromptingTools + +""" +Extract the name of the food from the sentence. Extract any provided adjectives for the food as well. + +Example: "I am eating a crunchy bread." -> Food("bread", ["crunchy"]) +""" +struct Food + name::String # required field! + adjectives::Union{Nothing,Vector{String}} # not required because `Nothing` is allowed +end + +msg = aiextract("I just ate a delicious and juicy apple."; return_type=Food) +msg.content +# Food("apple", ["delicious", "juicy"]) +``` + +You can see that we sent a prompt to the AI model and it returned a `Food` object. +We provided some light guidance as a docstring of the return type, but the AI model did the heavy lifting. + +`aiextract` leverages native "function calling" (supported by OpenAI, Fireworks, Together, and many others). + +We encode the user-provided `return_type` into the corresponding JSON schema and create the payload as per the specifications of the provider. + +Let's how that's done: +```julia +sig = PT.function_call_signature(Food) +## Dict{String, Any} with 3 entries: +## "name" => "Food_extractor" +## "parameters" => Dict{String, Any}("properties"=>Dict{String, Any}("name"=>Dict("type"=>"string"), "adjectives"=>Dict{String, … +## "description" => "Extract the food from the sentence. Extract any provided adjectives for the food as well.\n\nExample: " +``` +You can see that we capture the field names and types in `parameters` and the description in `description` key. + +Furthermore, if we zoom in on the "parameter" field, you can see that we encode not only the names and types but also whether the fields are required (ie, do they allow `Nothing`) +You can see below that the field `adjectives` accepts `Nothing`, so it's not required. Only the `name` field is required. +```julia +sig["parameters"] +## Dict{String, Any} with 3 entries: +## "properties" => Dict{String, Any}("name"=>Dict("type"=>"string"), "adjectives"=>Dict{String, Any}("items"=>Dict("type"=>"strin… +## "required" => ["name"] +## "type" => "object" +``` + +For `aiextract`, the signature is provided to the API provider via `tools` parameter, eg, + +`api_kwargs = (; tools = [Dict(:type => "function", :function => sig)])` + +Optionally, we can provide also `tool_choice` parameter to specify which tool to use if we provided multiple (differs across providers). + +When the message is returned, we extract the JSON object in the response and decode it into Julia object via `JSON3.read(obj, Food)`. For example, +```julia +model_response = Dict(:tool_calls => [Dict(:function => Dict(:arguments => JSON3.write(Dict("name" => "apple", "adjectives" => ["delicious", "juicy"]))))]) +food = JSON3.read(model_response[:tool_calls][1][:function][:arguments], Food) +# Output: Food("apple", ["delicious", "juicy"]) +``` + +This is why you can sometimes have errors when you use abstract types in your `return_type` -> to enable that, you would need to set the right `StructTypes` behavior for your abstract type (see the JSON3.jl documentation for more details on how to do that). + +It works quite well for concrete types and "vanilla" structs, though. + +Unfortunately, function calling is generally NOT supported by locally-hosted / open-source models, +so let's try to build a workaround with `aigenerate` + +You need to pick a bigger / more powerful model, as it's NOT an easy task to output a correct JSON specification. +My laptop isn't too powerful and I don't like waiting, so I'm going to use Mixtral model hosted on Together.ai (you get \$25 credit when you join)! + +```julia +model = "tmixtral" # tmixtral is an alias for "mistralai/Mixtral-8x7B-Instruct-v0.1" on Together.ai and it automatically sets `schema = TogetherOpenAISchema()` +``` + +We'll add the signature to the prompt and we'll request the JSON output in two places - in the prompt and in the `api_kwargs` (to ensure that the model outputs the JSON via "grammar") +NOTE: You can write much better and more specific prompt if you have a specific task / return type in mind + you should make sure that the prompt + struct description make sense together! + +Let's define a prompt and `return_type`. Notice that we add several placeholders (eg, `{{description}}`) to fill with user inputs later. +```julia +prompt = """ +You're a world-class data extraction engine. + +Your task is to extract information formatted as per the user provided schema. +You MUST response in JSON format. + +**Example:** +--------- +Description: "Extract the Car from the sentence. Extract the corresponding brand and model as well." +Input: "I drive a black Porsche 911 Turbo." +Schema: "{\"properties\":{\"model\":{\"type\":\"string\"},\"brand\":{\"type\":\"string\"}},\"required\":[\"brand\",\"model\"],\"type\":\"object\"}" +Output: "{\"model\":\"911 Turbo\",\"brand\":\"Porsche\"}" +--------- + +**User Request:** +Description: {{description}} +Input: {{input}} +Schema: {{signature}} +Output: + +You MUST OUTPUT in JSON format. +""" +``` + +We need to extract the "signature of our `return_type` and put it in the right placeholders. +Let's generate now! +```julia +sig = PT.function_call_signature(Food) +result = aigenerate(prompt; input="I just ate a delicious and juicy apple.", + schema=JSON3.write(sig["parameters"]), description=sig["description"], + ## We provide the JSON output requirement as per API docs: https://docs.together.ai/docs/json-mode + model, api_kwargs=(; response_format=Dict("type" => "json_object"), temperature=0.2), return_all=true) +result[end].content +## "{\n \"adjectives\": [\"delicious\", \"juicy\"],\n \"food\": \"apple\"\n}" +``` + +We're using a smaller model, so the output is not perfect. +Let's try to load into our object: +```julia +obj = JSON3.read(result[end].content, Food) +# Output: ERROR: MethodError: Cannot `convert` an object of type Nothing to an object of type String +``` + +Unfortunately, we get an error because the model mixed up the key "name" for "food", so it cannot be parsed. + +Fortunately, we can do better and use automatic fixing! +All we need to do is to change from `aigenerate` -> `AIGenerate` (and use `airetry!`) + +The signature of `AIGenerate` is identical to `aigenerate` with the exception of `config` field, where we can influence the future `retry` behaviour. +```julia +result = AIGenerate(prompt; input="I just ate a delicious and juicy apple.", + schema=JSON3.write(sig["parameters"]), description=sig["description"], + ## We provide the JSON output requirement as per API docs: https://docs.together.ai/docs/json-mode + model, api_kwargs=(; response_format=Dict("type" => "json_object"), temperature=0.2), + ## limit the number of retries, default is 10 rounds + config=RetryConfig(; max_retries=3)) +run!(result) # run! triggers the generation step (to have some AI output to check) +``` + +Let's set up a retry mechanism with some practical feedback. We'll leverage `airetry!` to automatically retry the request and provide feedback to the model. +Think of `airetry!` as `@assert` on steroids: + +`@assert CONDITION MESSAGE` → `airetry! CONDITION MESSAGE` + +The main benefits of `airetry!` are: +- It can retry automatically, not just throw an error +- It manages the "conversation’ (list of messages) for you, including adding user-provided feedback to help generate better output + +```julia +feedback = "The output is not in the correct format. The keys should be $(join([string("\"$f\"") for f in fieldnames(Food)],", "))." +# We use do-syntax with provide the `CONDITION` (it must return Bool) +airetry!(result, feedback) do conv + ## try to convert + obj = try + JSON3.read(last_output(conv), Food) + catch e + ## you could save the error and provide as feedback (eg, into a slot in the `:memory` field of the AICall object) + e + end + ## Check if the conversion was successful; if it's `false`, it will retry + obj isa Food # -> Bool +end +food = JSON3.read(last_output(result), Food) +## [ Info: Condition not met. Retrying... +## Output: Food("apple", ["delicious", "juicy"]) +``` + +It took 1 retry (see `result.config.retries`) and we have the correct output from an open-source model! + +If you're interested in the `result` object, it's a struct (`AICall`) with a field `conversation`, which holds the conversation up to this point. +AIGenerate is an alias for AICall using `aigenerate` function. See `?AICall` (the underlying struct type) for more details on the fields and methods available. \ No newline at end of file diff --git a/src/PromptingTools.jl b/src/PromptingTools.jl index 8b0e96700..a4b0999f9 100644 --- a/src/PromptingTools.jl +++ b/src/PromptingTools.jl @@ -23,7 +23,7 @@ const RESERVED_KWARGS = [ :image_url, :image_path, :image_detail, - :model, + :model ] # export replace_words, split_by_length, call_cost, auth_header # for debugging only @@ -45,6 +45,7 @@ include("messages.jl") export aitemplates, AITemplate include("templates.jl") +const TEMPLATE_PATH = String[joinpath(@__DIR__, "..", "templates")] const TEMPLATE_STORE = Dict{Symbol, Any}() const TEMPLATE_METADATA = Vector{AITemplateMetadata}() diff --git a/src/templates.jl b/src/templates.jl index c9b82f313..4a7efd852 100644 --- a/src/templates.jl +++ b/src/templates.jl @@ -128,66 +128,107 @@ Removes all templates from `TEMPLATE_STORE` and `TEMPLATE_METADATA`. remove_templates!(; store = TEMPLATE_STORE, metadata_store = TEMPLATE_METADATA) = (empty!(store); empty!(metadata_store); nothing) """ - load_templates!(; remove_templates::Bool=true) + load_templates!(dir_templates::Union{String, Nothing} = nothing; + remember_path::Bool = true, + remove_templates::Bool = isnothing(dir_templates), + store::Dict{Symbol, <:Any} = TEMPLATE_STORE, + metadata_store::Vector{<:AITemplateMetadata} = TEMPLATE_METADATA) Loads templates from folder `templates/` in the package root and stores them in `TEMPLATE_STORE` and `TEMPLATE_METADATA`. Note: Automatically removes any existing templates and metadata from `TEMPLATE_STORE` and `TEMPLATE_METADATA` if `remove_templates=true`. + +# Arguments +- `dir_templates::Union{String, Nothing}`: The directory path to load templates from. If `nothing`, uses the default list of paths. It usually used only once "to register" a new template storage. +- `remember_path::Bool=true`: If true, remembers the path for future refresh (in `TEMPLATE_PATH`). +- `remove_templates::Bool=isnothing(dir_templates)`: If true, removes any existing templates and metadata from `store` and `metadata_store`. +- `store::Dict{Symbol, <:Any}=TEMPLATE_STORE`: The store to load the templates into. +- `metadata_store::Vector{<:AITemplateMetadata}=TEMPLATE_METADATA`: The metadata store to load the metadata into. + +# Example + +Load the default templates: +```julia +PT.load_templates!() # no path needed +``` + +Load templates from a new custom path: +```julia +PT.load_templates!("path/to/templates") # we will remember this path for future refresh +``` + +If you want to now refresh the default templates and the new path, just call `load_templates!()` without any arguments. """ -function load_templates!(dir_templates::String = joinpath(@__DIR__, "..", "templates"); - remove_templates::Bool = true, +function load_templates!(dir_templates::Union{String, Nothing} = nothing; + remember_path::Bool = true, + remove_templates::Bool = isnothing(dir_templates), store::Dict{Symbol, <:Any} = TEMPLATE_STORE, - metadata_store::Vector{<:AITemplateMetadata} = TEMPLATE_METADATA,) + metadata_store::Vector{<:AITemplateMetadata} = TEMPLATE_METADATA) + ## Init + global TEMPLATE_PATH + @assert isnothing(dir_templates)||isdir(dir_templates) "Invalid directory path provided! ($dir_templates)" + + # If no path is provided, use the default list + load_paths = isnothing(dir_templates) ? TEMPLATE_PATH : [dir_templates] # first remove any old templates and their metadata remove_templates && remove_templates!(; store, metadata_store) - # recursively load all templates from the `templates` folder - for (root, dirs, files) in walkdir(dir_templates) - for file in files - if endswith(file, ".json") - template_name = Symbol(split(basename(file), ".")[begin]) - template, metadata_msgs = load_template(joinpath(root, file)) - # add to store - if haskey(store, template_name) - @warn("Template $(template_name) already exists, overwriting! Metadata will be duplicated.") - end - store[template_name] = template - - # prepare the metadata - wordcount = 0 - system_preview = "" - user_preview = "" - variables = Symbol[] - for i in eachindex(template) - msg = template[i] - wordcount += length(msg.content) - if hasproperty(msg, :variables) - append!(variables, msg.variables) + # remember the path for future refresh + if remember_path && !isnothing(dir_templates) + if !(dir_templates in TEMPLATE_PATH) + push!(TEMPLATE_PATH, dir_templates) + end + end + + # recursively load all templates from the `load_paths` + for template_path in load_paths + for (root, dirs, files) in walkdir(template_path) + for file in files + if endswith(file, ".json") + template_name = Symbol(split(basename(file), ".")[begin]) + template, metadata_msgs = load_template(joinpath(root, file)) + # add to store + if haskey(store, template_name) + @warn("Template $(template_name) already exists, overwriting! Metadata will be duplicated.") end - # truncate previews to 100 characters - if msg isa SystemMessage && length(system_preview) < 100 - system_preview *= first(msg.content, 100) - elseif msg isa UserMessage && length(user_preview) < 100 - user_preview *= first(msg.content, 100) + store[template_name] = template + + # prepare the metadata + wordcount = 0 + system_preview = "" + user_preview = "" + variables = Symbol[] + for i in eachindex(template) + msg = template[i] + wordcount += length(msg.content) + if hasproperty(msg, :variables) + append!(variables, msg.variables) + end + # truncate previews to 100 characters + if msg isa SystemMessage && length(system_preview) < 100 + system_preview *= first(msg.content, 100) + elseif msg isa UserMessage && length(user_preview) < 100 + user_preview *= first(msg.content, 100) + end end + if !isempty(metadata_msgs) + # use the first metadata message found if available + meta = first(metadata_msgs) + metadata = AITemplateMetadata(; name = template_name, + meta.description, meta.version, meta.source, + wordcount, + system_preview = first(system_preview, 100), + user_preview = first(user_preview, 100), + variables = unique(variables)) + else + metadata = AITemplateMetadata(; name = template_name, + wordcount, + system_preview = first(system_preview, 100), + user_preview = first(user_preview, 100), + variables = unique(variables)) + end + # add metadata to store + push!(metadata_store, metadata) end - if !isempty(metadata_msgs) - # use the first metadata message found if available - meta = first(metadata_msgs) - metadata = AITemplateMetadata(; name = template_name, - meta.description, meta.version, meta.source, - wordcount, - system_preview = first(system_preview, 100), - user_preview = first(user_preview, 100), - variables = unique(variables)) - else - metadata = AITemplateMetadata(; name = template_name, - wordcount, - system_preview = first(system_preview, 100), - user_preview = first(user_preview, 100), - variables = unique(variables)) - end - # add metadata to store - push!(metadata_store, metadata) end end end @@ -249,7 +290,8 @@ function aitemplates(query_name::Symbol; limit::Int = 10, metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) query_str = lowercase(string(query_name)) - found_templates = filter(x -> occursin(query_str, + found_templates = filter( + x -> occursin(query_str, lowercase(string(x.name))), metadata_store) return first(found_templates, limit) end @@ -258,7 +300,8 @@ function aitemplates(query_key::AbstractString; limit::Int = 10, metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) query_str = lowercase(query_key) - found_templates = filter(x -> occursin(query_str, lowercase(string(x.name))) || + found_templates = filter( + x -> occursin(query_str, lowercase(string(x.name))) || occursin(query_str, lowercase(string(x.description))), metadata_store) return first(found_templates, limit) @@ -267,13 +310,14 @@ end function aitemplates(query_key::Regex; limit::Int = 10, metadata_store::Vector{AITemplateMetadata} = TEMPLATE_METADATA) - found_templates = filter(x -> occursin(query_key, - string(x.name)) || - occursin(query_key, - x.description) || - occursin(query_key, - x.system_preview) || - occursin(query_key, x.user_preview), + found_templates = filter( + x -> occursin(query_key, + string(x.name)) || + occursin(query_key, + x.description) || + occursin(query_key, + x.system_preview) || + occursin(query_key, x.user_preview), metadata_store) return first(found_templates, limit) end @@ -305,3 +349,82 @@ end function aiscan(schema::AbstractPromptSchema, template::Symbol; kwargs...) aiscan(schema, AITemplate(template); kwargs...) end + +## Utility for creating templates +""" + create_template(; user::AbstractString, system::AbstractString="Act as a helpful AI assistant.") + + create_template(system::AbstractString, user::AbstractString) + +Creates a simple template with a user and system message. Convenience function to prevent writing `[PT.UserMessage(...), ...]` + +# Arguments +- `system::AbstractString`: The system message. Usually defines the personality, style, instructions, output format, etc. +- `user::AbstractString`: The user message. Usually defines the input, query, request, etc. + +Use double handlebar placeholders (eg, `{{name}}`) to define variables that can be replaced by the `kwargs` during the AI call (see example). + +Returns a vector of `SystemMessage` and UserMessage objects. + +# Examples + +Let's generate a quick template for a simple conversation (only one placeholder: name) +```julia +# first system message, then user message (or use kwargs) +tpl=PT.create_template("You must speak like a pirate", "Say hi to {{name}}") + +## 2-element Vector{PromptingTools.AbstractChatMessage}: +## PromptingTools.SystemMessage("You must speak like a pirate") +## PromptingTools.UserMessage("Say hi to {{name}}") +``` + +You can immediately use this template in `ai*` functions: +```julia +aigenerate(tpl; name="Jack Sparrow") +# Output: AIMessage("Arr, me hearty! Best be sending me regards to Captain Jack Sparrow on the salty seas! May his compass always point true to the nearest treasure trove. Yarrr!") +``` + +If you want to save it in your project folder: +```julia +PT.save_template("templates/GreatingPirate.json", tpl; version="1.0") # optionally, add description +``` +It will be saved and accessed under its basename, ie, `GreatingPirate`. + +Now you can load it like all the other templates (provide the template directory): +``` +PT.load_templates!("templates") # it will remember the folder after the first run +# Note: If you save it again, overwrite it, etc., you need to explicitly reload all templates again! +``` + +You can verify that your template is loaded with a quick search for "pirate": +```julia +aitemplates("pirate") + +## 1-element Vector{AITemplateMetadata}: +## PromptingTools.AITemplateMetadata +## name: Symbol GreatingPirate +## description: String "" +## version: String "1.0" +## wordcount: Int64 46 +## variables: Array{Symbol}((1,)) +## system_preview: String "You must speak like a pirate" +## user_preview: String "Say hi to {{name}}" +## source: String "" +``` + +Now you can use it like any other template (notice it's a symbol, so `:GreatingPirate`): +```julia +aigenerate(:GreatingPirate; name="Jack Sparrow") +# Output: AIMessage("Arr, me hearty! Best be sending me regards to Captain Jack Sparrow on the salty seas! May his compass always point true to the nearest treasure trove. Yarrr!") +```` +""" +function create_template( + system::AbstractString, + user::AbstractString) + return [SystemMessage(system), UserMessage(user)] +end +# Kwarg version +function create_template(; + user::AbstractString, system::AbstractString = "Act as a helpful AI assistant.") + create_template(system, user) +end diff --git a/test/templates.jl b/test/templates.jl index 23d613ac7..a9a63ebb2 100644 --- a/test/templates.jl +++ b/test/templates.jl @@ -1,11 +1,12 @@ using PromptingTools: AbstractChatMessage, SystemMessage, UserMessage, MetadataMessage using PromptingTools: render -using PromptingTools: load_templates!, aitemplates +using PromptingTools: load_templates!, aitemplates, create_template using PromptingTools: TestEchoOpenAISchema @testset "Template rendering" begin template = AITemplate(:JudgeIsItTrue) - expected_output = AbstractChatMessage[SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), + expected_output = AbstractChatMessage[ + SystemMessage("You are an impartial AI judge evaluting whether the provided statement is \"true\" or \"false\". Answer \"unknown\" if you cannot decide."), UserMessage("# Statement\n\n{{it}}")] @test expected_output == render(PT.PROMPT_SCHEMA, template) @test expected_output == render(template) @@ -32,6 +33,41 @@ end @test length(tmps) >= 1 end +@testset "load_templates!" begin + load_templates!() + PT.TEMPLATE_PATH = PT.TEMPLATE_PATH[[1]] # reset + dir_name = joinpath(tempdir(), "templates") + dir_name in PT.TEMPLATE_PATH + mkpath(dir_name) + load_templates!(dir_name) + @test length(PT.TEMPLATE_PATH) == 2 + @test PT.TEMPLATE_PATH[2] == dir_name + # no more changes + load_templates!(dir_name) + load_templates!(dir_name) + @test length(PT.TEMPLATE_PATH) == 2 + @test PT.TEMPLATE_PATH[2] == dir_name + # reset to normal + PT.TEMPLATE_PATH = PT.TEMPLATE_PATH[[1]] # reset +end + +@testset "create_template" begin + tpl = create_template("You must speak like a pirate", "Say hi to {{name}}") + @test tpl[1].content == "You must speak like a pirate" + @test tpl[1] isa SystemMessage + @test tpl[2].content == "Say hi to {{name}}" + @test tpl[2].variables == [:name] + @test tpl[2] isa UserMessage + + # kwarg constructor + tpl = create_template(; user = "Say hi to {{chef}}") + @test tpl[1].content == "Act as a helpful AI assistant." + @test tpl[1] isa SystemMessage + @test tpl[2].content == "Say hi to {{chef}}" + @test tpl[2].variables == [:chef] + @test tpl[2] isa UserMessage +end + @testset "Templates - Echo aigenerate call" begin # E2E test for aigenerate with rendering template and filling the placeholders template_name = :JudgeIsItTrue diff --git a/test/utils.jl b/test/utils.jl index 5d6961cee..1e4fbbeb3 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -3,7 +3,7 @@ using PromptingTools: _extract_handlebar_variables, call_cost, _report_stats using PromptingTools: _string_to_vector, _encode_local_image using PromptingTools: DataMessage, AIMessage using PromptingTools: push_conversation!, - resize_conversation!, @timeout, preview, auth_header + resize_conversation!, @timeout, preview, auth_header @testset "replace_words" begin words = ["Disney", "Snow White", "Mickey Mouse"] @@ -243,7 +243,7 @@ end PT.SystemMessage("Welcome"), PT.UserMessage("Hello"), PT.AIMessage("World"), - PT.DataMessage(; content = ones(10)), + PT.DataMessage(; content = ones(10)) ] preview_output = preview(conversation) expected_output = Markdown.parse("# System Message\n\nWelcome\n\n---\n\n# User Message\n\nHello\n\n---\n\n# AI Message\n\nWorld\n\n---\n\n# Data Message\n\nData: Vector{Float64} (Size: (10,))\n") @@ -255,7 +255,7 @@ end @test headers == [ "Authorization" => "Bearer ", "Content-Type" => "application/json", - "Accept" => "application/json", + "Accept" => "application/json" ] @test_throws ArgumentError auth_header("") @test length(auth_header(nothing)) == 2 From 19cf980bdd24abab273cf1978ee24c75058cb997 Mon Sep 17 00:00:00 2001 From: J S <49557684+svilupp@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:24:08 +0000 Subject: [PATCH 129/251] update docs + version (#85) --- CHANGELOG.md | 8 +- Project.toml | 2 +- docs/src/frequently_asked_questions.md | 66 +++++++++++- docs/src/how_it_works.md | 2 + src/templates.jl | 135 +++++++++++++++++-------- test/templates.jl | 9 ++ 6 files changed, 178 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a592a8a17..ddbe50be7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +### Fixed + +## [0.14.0] + ### Added - Added a new documentation section "How it works" to explain the inner workings of the package. It's a work in progress, but it should give you a good idea of what's happening under the hood. - Improved template loading, so if you load your custom templates once with `load_templates!("my/template/folder)`, it will remember your folder for all future re-loads. -- Added convenience function `create_template` to create templates on the fly without having to deal with `PT.UserMessage` etc. See `?create_template` for more information. +- Added convenience function `create_template` to create templates on the fly without having to deal with `PT.UserMessage` etc. If you specify the keyword argument `load_as = "MyName"`, the template will be immediately loaded to the template registry. See `?create_template` for more information and examples. ### Fixed diff --git a/Project.toml b/Project.toml index da0c0e797..f9666002d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.14.0-DEV" +version = "0.14.0" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" diff --git a/docs/src/frequently_asked_questions.md b/docs/src/frequently_asked_questions.md index 923b3e915..824d26e5b 100644 --- a/docs/src/frequently_asked_questions.md +++ b/docs/src/frequently_asked_questions.md @@ -287,4 +287,68 @@ give_me_number("How many car seats are in Porsche 911T?") We ultimately received our custom type `SmallInt` with the number of car seats in the Porsche 911T (I hope it's correct!). -If you want to access the full conversation history (all the attempts and feedback), simply output the `response` object and explore `response.conversation`. \ No newline at end of file +If you want to access the full conversation history (all the attempts and feedback), simply output the `response` object and explore `response.conversation`. + +## How to quickly create a prompt template? + +Many times, you will want to create a prompt template that you can reuse with different inputs (eg, to create templates for AIHelpMe or LLMTextAnalysis). + +Previously, you would have to create a vector of `SystemMessage` and `UserMessage` objects and then save it to a disk and reload. +Now, you can use the `create_template` function to do it for you. It's designed for quick prototyping, so it skips the serialization step and loads it directly into the template store (ie, you can use it like any other templates - try `aitemplates()` search). + +The syntax is simple: `create_template(;user=, system=, load_as=