diff --git a/CHANGELOG.md b/CHANGELOG.md index db38bb788..0d75e209d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +## [0.61.0] + +### Added +- Added a new `extras` field to `ToolRef` to enable additional parameters in the tool signature (eg, `display_width_px`, `display_height_px` for the `:computer` tool). + +### Updated +- Updated the compat bounds for `StreamCallbacks` to enable both v0.4 and v0.5 (Fixes Julia 1.9 compatibility). +- Updated the return type of `tool_call_signature` to `Dict{String, AbstractTool}` to enable better interoperability with different tool types. + ## [0.60.0] ### Added diff --git a/Project.toml b/Project.toml index 8e18c2de4..83aac0270 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PromptingTools" uuid = "670122d1-24a8-4d70-bfce-740807c42192" authors = ["J S @svilupp and contributors"] -version = "0.60.0" +version = "0.61.0" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" @@ -55,9 +55,9 @@ REPL = "<0.0.1, 1" Random = "<0.0.1, 1" SparseArrays = "<0.0.1, 1" Statistics = "<0.0.1, 1" -StreamCallbacks = "0.4" +StreamCallbacks = "0.4, 0.5" Test = "<0.0.1, 1" -julia = "1.9,1.10" +julia = "1.9, 1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" diff --git a/src/extraction.jl b/src/extraction.jl index 0c2fa2514..8f155bc83 100644 --- a/src/extraction.jl +++ b/src/extraction.jl @@ -57,11 +57,12 @@ It can be rendered with a `render` method and a prompt schema. # Arguments - `ref::Symbol`: The symbolic name of the tool. - `callable::Any`: The callable object of the tool, eg, a type or a function. +- `extras::Dict{String, Any}`: Additional parameters to be included in the tool signature. # Examples ```julia # Define a tool with a symbolic name and a callable object -tool = ToolRef(:computer, println) +tool = ToolRef(;ref=:computer, callable=println) # Show the rendered tool signature PT.render(PT.AnthropicSchema(), tool) @@ -70,6 +71,7 @@ PT.render(PT.AnthropicSchema(), tool) Base.@kwdef struct ToolRef <: AbstractTool ref::Symbol callable::Any = identity + extras::Dict{String, Any} = Dict() end Base.show(io::IO, t::ToolRef) = print(io, "ToolRef($(t.ref))") @@ -444,7 +446,7 @@ Note: Fairly experimental, but works for combination of structs, arrays, strings - `hidden_fields::AbstractVector{<:Union{AbstractString, Regex}}`: A list of fields to hide from the LLM (eg, `["ctx_user_id"]` or `r"ctx"`). # Returns -- `Dict{String, Tool}`: A dictionary representing the function call signature schema. +- `Dict{String, AbstractTool}`: A dictionary representing the function call signature schema. # Tips - You can improve the quality of the extraction by writing a helpful docstring for your struct (or any nested struct). It will be provided as a description. @@ -464,7 +466,7 @@ struct MyMeasurement end tool_map = tool_call_signature(MyMeasurement) # -# Dict{String, PromptingTools.Tool}("MyMeasurement" => PromptingTools.Tool +# Dict{String, PromptingTools.AbstractTool}("MyMeasurement" => PromptingTools.Tool # name: String "MyMeasurement" # parameters: Dict{String, Any} # description: Nothing nothing @@ -563,7 +565,7 @@ function tool_call_signature( description = haskey(schema, "description") ? schema["description"] : nothing, strict = haskey(schema, "strict") ? schema["strict"] : nothing, callable = call_type) - return Dict(schema["name"] => tool) + return Dict{String, AbstractTool}(schema["name"] => tool) end ## Only thing you can change is the "strict" setting @@ -583,7 +585,7 @@ function tool_call_signature( end function tool_call_signature( tool::ToolRef; kwargs...) - Dict(string(tool.ref) => tool) + return Dict{String, AbstractTool}(string(tool.ref) => tool) end ## Add support for function signatures @@ -594,7 +596,7 @@ end function tool_call_signature( tools::Vector{<:T}; kwargs...) where {T <: Union{Type, Function, Method, AbstractTool}} - tool_map = Dict{String, Tool}() + tool_map = Dict{String, AbstractTool}() for tool in tools temp_map = tool_call_signature(tool; kwargs...) for (name, tool) in temp_map diff --git a/src/llm_anthropic.jl b/src/llm_anthropic.jl index b5fc83951..1a4e8c40b 100644 --- a/src/llm_anthropic.jl +++ b/src/llm_anthropic.jl @@ -130,13 +130,13 @@ function render(schema::AbstractAnthropicSchema, tool::ToolRef; kwargs...) ## WARNING: We ignore the tool name here, because the names are strict + (; extras) = tool rendered = if tool.ref == :computer Dict( "type" => "computer_20241022", "name" => "computer", - "display_width_px" => 1024, - "display_height_px" => 768, - "display_number" => 1 + "display_width_px" => get(extras, "display_width_px", 1024), + "display_height_px" => get(extras, "display_height_px", 768) ) elseif tool.ref == :str_replace_editor Dict( @@ -151,6 +151,9 @@ function render(schema::AbstractAnthropicSchema, else throw(ArgumentError("Unknown tool reference: $(tool.ref)")) end + if !isempty(extras) + merge!(rendered, extras) + end return rendered end @@ -836,6 +839,23 @@ conv = aitools( # UserMessage("And in New York?") # AIToolRequest("-"; Tool Requests: 1) ``` + +Using the the new Computer Use beta feature: +```julia +# Define tools (and associated functions to call) +tool_map = Dict("bash" => PT.ToolRef(; ref=:bash, callable=bash_tool), + "computer" => PT.ToolRef(; ref=:computer, callable=computer_tool, + extras=Dict("display_width_px" => 1920, "display_height_px" => 1080)), + "str_replace_editor" => PT.ToolRef(; ref=:str_replace_editor, callable=edit_tool)) + +msg = aitools(prompt; tools=collect(values(tool_map)), model="claude", betas=[:computer_use]) + +PT.pprint(msg) +# -------------------- +# AI Tool Request +# -------------------- +# Tool Request: computer, args: Dict{Symbol, Any}(:action => "screenshot") +``` """ function aitools(prompt_schema::AbstractAnthropicSchema, prompt::ALLOWED_PROMPT_TYPE; tools::Union{Type, Function, Method, AbstractTool, Vector} = Tool[], diff --git a/test/extraction.jl b/test/extraction.jl index 2c4efce8a..2c6590478 100644 --- a/test/extraction.jl +++ b/test/extraction.jl @@ -79,7 +79,7 @@ end @test isabstracttool(my_test_function) == false ## ToolRef - tool = ToolRef(:computer, println) + tool = ToolRef(; ref = :computer, callable = println) @test tool isa ToolRef @test tool.ref == :computer @test tool.callable == println @@ -757,7 +757,7 @@ end @test tool2.parameters["properties"]["weight"]["type"] == "number" ## ToolRef - tool = ToolRef(:computer, println) + tool = ToolRef(; ref = :computer, callable = println) tool_map = tool_call_signature(tool) @test tool_map == Dict("computer" => tool) end diff --git a/test/llm_anthropic.jl b/test/llm_anthropic.jl index 036f16a78..cfc7f2c51 100644 --- a/test/llm_anthropic.jl +++ b/test/llm_anthropic.jl @@ -282,6 +282,16 @@ end @test rendered["name"] == "computer" @test rendered["display_width_px"] == 1024 @test rendered["display_height_px"] == 768 + @test !haskey(rendered, "display_number") + + computer_tool2 = ToolRef(ref = :computer, + extras = Dict("display_width_px" => 1920, + "display_height_px" => 1080, "display_number" => 1)) + rendered = render(schema, computer_tool2) + @test rendered["type"] == "computer_20241022" + @test rendered["name"] == "computer" + @test rendered["display_width_px"] == 1920 + @test rendered["display_height_px"] == 1080 @test rendered["display_number"] == 1 # Test text editor tool rendering diff --git a/test/llm_shared.jl b/test/llm_shared.jl index 50951bda4..1688341de 100644 --- a/test/llm_shared.jl +++ b/test/llm_shared.jl @@ -1,4 +1,4 @@ -using PromptingTools: render, NoSchema, AbstractPromptSchema +using PromptingTools: render, NoSchema, AbstractPromptSchema, OpenAISchema using PromptingTools: AIMessage, SystemMessage, AbstractMessage, AbstractChatMessage using PromptingTools: UserMessage, UserMessageWithImages, DataMessage, AIToolRequest, ToolMessage, ToolRef @@ -222,7 +222,7 @@ using PromptingTools: finalize_outputs, role4render ## ToolRef schema = NoSchema() - tool = ToolRef(:computer, println) + tool = ToolRef(; ref = :computer) @test_throws ArgumentError render(schema, tool) end