From 54b1c657b5321c832750d47018869522ca47bf00 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Fri, 25 Oct 2024 12:16:17 +0200 Subject: [PATCH 01/44] todos --- TODO.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/TODO.txt b/TODO.txt index 702bfcf3f..99dfffbdb 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,7 +1,14 @@ # List of minor Todos +?? Default Translation Configuration in AEM? + https://wcm.io/caconfig/editor/usage.html translation config drop down +Generated with ... in conf pages +http://localhost:4502/editor.html/content/dam/wknd/en/magazine/arctic-surfing/aloha-spirits-in-northern-norway doesnt add page text or component text!!! + +- default icon for ai.composum site +!!! Dictation in Chat repeat in history bar. !! Empty prompt text !! http://localhost:5502/editor.html/content/xxx/com/en/about-us/going-forward.html "Explore the world..." RTE PDF and markdown as possible datatypes for the content creation dialog @@ -13,6 +20,8 @@ Why are the teasers in https://author-p43852-e197429.adobeaemcloud.com/editor.html/content/gfps/com/en/products-solutions/systems/primofit.html not included into the excerpt. +? Add history in content creation dialog, current component, last text, ... + ---------------- - Marker on page whenever a translation is in progress From 71df4ce48358fb6bc18ea8a47df3a043f6701dd0 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Sat, 12 Oct 2024 19:01:31 +0200 Subject: [PATCH 02/44] AITool interface --- .../slingbase/experimential/AITool.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java new file mode 100644 index 000000000..2900f60fe --- /dev/null +++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java @@ -0,0 +1,63 @@ +package com.composum.ai.backend.slingbase.experimential; + +import java.util.Locale; + +import org.apache.sling.api.resource.Resource; + +/** + * An action the AI can perform - likely from the sidebar chat. + * + * @see "https://platform.openai.com/docs/guides/function-calling" + */ +public interface AITool { + + /** + * Human readable name. + */ + String getName(Locale locale); + + /** + * Human readable description. + */ + String getDescription(Locale locale); + + /** + * The description to use for the OpenAI tool call. Will be inserted into the OpenAI tools array. E.g.: + * 
+     *       {
+     *         "type": "function",
+     *         "function": {
+     *           "name": "get_delivery_date",
+     *           "description": "Get the delivery date for a customer's order. Call this whenever you need to know the delivery date, for example when a customer asks 'Where is my package'",
+     *           "parameters": {
+     *             "type": "object",
+     *             "properties": {
+     *               "order_id": {
+     *                 "type": "string",
+     *                 "description": "The customer's order ID."
+     *               }
+     *             },
+     *             "required": ["order_id"],
+     *             "additionalProperties": false
+     *           }
+     *         },
+     *         "strict": true
+     *       }
+     *
+ * + * @see "https://platform.openai.com/docs/api-reference/chat/create" + */ + String getToolDeclaration(); + + /** + * Whether the tool is enabled for the given resource. + */ + public boolean isAllowedFor(Resource resource); + + /** + * Executes the tool call and returns the result to present to the AI. + * Must only be called if {@link #isAllowedFor(Resource)} returned true. + */ + public String execute(String arguments, Resource resource); + +} From c3473b4eca98aed99dc0a866b949870391adf090 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Mon, 14 Oct 2024 16:39:11 +0200 Subject: [PATCH 03/44] introduce GPTTool class --- .../base/service/chat/GPTConfiguration.java | 39 ++++++++++++- .../ai/backend/base/service/chat/GPTTool.java | 48 +++++++++++++++ ...GPTChatCompletionServiceImplWithTools.java | 58 +++++++++++++++++++ .../slingbase/experimential/AITool.java | 9 ++- 4 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java create mode 100644 backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTConfiguration.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTConfiguration.java index de180b806..f32b0909d 100644 --- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTConfiguration.java +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTConfiguration.java @@ -41,6 +41,16 @@ public enum Mode { CHAT } +// /** Controls which (if any) tool is called by the model. */ +// public enum ToolChoice { +// /** Do not call a tool, just output a message. */ +// NONE, +// /** Model can pick between generating a message and calling a tool. */ +// AUTO, +// /** Forces calling one or more tools. */ +// REQUIRED +// } + private final String apiKey; private final String organizationId; @@ -61,6 +71,8 @@ public enum Mode { private final List contexts; + private final List tools; + public GPTConfiguration(@Nullable String apiKey, @Nullable String organizationId, @Nullable AnswerType answerType) { this(apiKey, organizationId, answerType, null); @@ -91,6 +103,10 @@ public GPTConfiguration(@Nullable String apiKey, @Nullable String organizationId } public GPTConfiguration(@Nullable String apiKey, @Nullable String organizationId, @Nullable AnswerType answerType, @Nullable String additionalInstructions, @Nullable Mode mode, @Nullable Boolean highIntelligenceNeeded, @Nullable Boolean debug, @Nullable Double temperature, @Nullable Integer seed, @Nullable List contexts) { + this(apiKey, organizationId, answerType, additionalInstructions, mode, highIntelligenceNeeded, debug, temperature, seed, contexts, null); + } + + public GPTConfiguration(@Nullable String apiKey, @Nullable String organizationId, @Nullable AnswerType answerType, @Nullable String additionalInstructions, @Nullable Mode mode, @Nullable Boolean highIntelligenceNeeded, @Nullable Boolean debug, @Nullable Double temperature, @Nullable Integer seed, @Nullable List contexts, List tools) { this.apiKey = apiKey; this.answerType = answerType; this.organizationId = organizationId; @@ -101,6 +117,7 @@ public GPTConfiguration(@Nullable String apiKey, @Nullable String organizationId this.temperature = temperature; this.seed = seed; this.contexts = contexts; + this.tools = tools; } /** @@ -176,6 +193,13 @@ public Integer getSeed() { return seed; } + /** + * A list of tools the Model could use. + */ + public List getTools() { + return tools; + } + /** * Creates a configuration that joins the values. * @@ -228,7 +252,15 @@ public GPTConfiguration merge(@Nullable GPTConfiguration other, boolean override contextInfos.addAll(other.contexts); } contextInfos = contextInfos.isEmpty() ? null : Collections.unmodifiableList(contextInfos); - return new GPTConfiguration(apiKey, organizationId, answerType, additionalInstructions, mode, highIntelligenceNeeded, debug, temperature, seed, contextInfos); + List tools = new ArrayList<>(); + if (this.tools != null) { + tools.addAll(this.tools); + } + if (other.tools != null) { + tools.addAll(other.tools); + } + tools = tools.isEmpty() ? null : Collections.unmodifiableList(tools); + return new GPTConfiguration(apiKey, organizationId, answerType, additionalInstructions, mode, highIntelligenceNeeded, debug, temperature, seed, contextInfos, tools); } /** @@ -276,6 +308,11 @@ public static GPTConfiguration ofContext(@Nonnull String usermsg, @Nonnull Strin return ofContexts(Collections.singletonList(new GPTContextInfo(usermsg, assistantmsg))); } + @Nonnull + public static GPTConfiguration ofTools(@Nullable List tools) { + return new GPTConfiguration(null, null, null, null, null, null, null, null, null, null, tools); + } + @Override public String toString() { StringBuilder sb = new StringBuilder("GPTConfiguration{"); diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java new file mode 100644 index 000000000..ab51c320a --- /dev/null +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java @@ -0,0 +1,48 @@ +package com.composum.ai.backend.base.service.chat; + +/** + * An action the AI can perform - likely from the sidebar chat. + * + * @see "https://platform.openai.com/docs/guides/function-calling" + */ +public interface GPTTool { + + /** + * The name of the tool - must be exactly the name given in {@link #getToolDeclaration()}. + */ + String getName(); + + /** + * The description to use for the OpenAI tool call. Will be inserted into the OpenAI tools array. E.g.: + * 
+     *       {
+     *         "type": "function",
+     *         "function": {
+     *           "name": "get_delivery_date",
+     *           "description": "Get the delivery date for a customer's order. Call this whenever you need to know the delivery date, for example when a customer asks 'Where is my package'",
+     *           "parameters": {
+     *             "type": "object",
+     *             "properties": {
+     *               "order_id": {
+     *                 "type": "string",
+     *                 "description": "The customer's order ID."
+     *               }
+     *             },
+     *             "required": ["order_id"],
+     *             "additionalProperties": false
+     *           }
+     *         },
+     *         "strict": true
+     *       }
+     *
+ * + * @see "https://platform.openai.com/docs/api-reference/chat/create" + */ + String getToolDeclaration(); + + /** + * Executes the tool call and returns the result to present to the AI. + */ + public String execute(String arguments); + +} diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java new file mode 100644 index 000000000..71412c044 --- /dev/null +++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java @@ -0,0 +1,58 @@ +package com.composum.ai.backend.base.service.chat.impl; + +import com.composum.ai.backend.base.service.chat.GPTChatRequest; +import com.composum.ai.backend.base.service.chat.GPTCompletionCallback; +import com.composum.ai.backend.base.service.chat.GPTFinishReason; +import com.composum.ai.backend.base.service.chat.GPTMessageRole; + +/** + * Tries an actual call to ChatGPT with the streaming interface. Since that costs money (though much less than a cent), + * needs a secret key and takes a couple of seconds, we don't do that as an JUnit test. + */ +public class RunGPTChatCompletionServiceImplWithTools extends AbstractGPTRunner implements GPTCompletionCallback { + + StringBuilder buffer = new StringBuilder(); + private boolean isFinished; + + public static void main(String[] args) throws Exception { + RunGPTChatCompletionServiceImplWithTools instance = new RunGPTChatCompletionServiceImplWithTools(); + instance.setup(); + instance.run(); + instance.teardown(); + System.out.println("Done."); + } + + private void run() throws InterruptedException { + GPTChatRequest request = new GPTChatRequest(); + request.addMessage(GPTMessageRole.USER, "Make 2 haiku about the weather."); + chatCompletionService.streamingChatCompletion(request, this); + System.out.println("Call returned."); + while (!isFinished) Thread.sleep(1000); + System.out.println("Complete response:"); + System.out.println(buffer); + } + + @Override + public void onFinish(GPTFinishReason finishReason) { + isFinished = true; + System.out.println(); + System.out.println("Finished: " + finishReason); + } + + @Override + public void setLoggingId(String loggingId) { + System.out.println("Logging ID: " + loggingId); + } + + @Override + public void onNext(String item) { + buffer.append(item); + System.out.print(item); + } + + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(System.err); + isFinished = true; + } +} diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java index 2900f60fe..a99971302 100644 --- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java +++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java @@ -4,6 +4,8 @@ import org.apache.sling.api.resource.Resource; +import com.composum.ai.backend.base.service.chat.GPTTool; + /** * An action the AI can perform - likely from the sidebar chat. * @@ -52,12 +54,15 @@ public interface AITool { /** * Whether the tool is enabled for the given resource. */ - public boolean isAllowedFor(Resource resource); + boolean isAllowedFor(Resource resource); /** * Executes the tool call and returns the result to present to the AI. * Must only be called if {@link #isAllowedFor(Resource)} returned true. */ - public String execute(String arguments, Resource resource); + String execute(String arguments, Resource resource); + + /** The form useable by {@link com.composum.ai.backend.base.service.chat.GPTChatCompletionService}.*/ + GPTTool makeGPTTool(Resource resource); } From 8dfb59c2ddfd089c0c692af6d398075d53f81aec Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Mon, 14 Oct 2024 16:40:32 +0200 Subject: [PATCH 04/44] add tools attribute to ChatCompletionRequest --- .../impl/chatmodel/ChatCompletionRequest.java | 12 +++++ .../impl/chatmodel/ChatFunctionDetails.java | 52 +++++++++++++++++++ .../service/chat/impl/chatmodel/ChatTool.java | 30 +++++++++++ 3 files changed, 94 insertions(+) create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java index a808f93e6..574b86789 100644 --- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java @@ -4,6 +4,7 @@ import com.composum.ai.backend.base.service.chat.GPTMessageRole; import com.google.gson.annotations.SerializedName; +import java.util.List; public class ChatCompletionRequest { @@ -28,6 +29,9 @@ public class ChatCompletionRequest { @SerializedName("seed") private Integer seed; + @SerializedName("tools") + private List tools; + // Getters and setters public String getModel() { return model; @@ -77,6 +81,14 @@ public void setResponseFormat(ResponseFormat responseFormat) { this.responseFormat = responseFormat; } + public List getTools() { + return tools; + } + + public void setTools(List tools) { + this.tools = tools; + } + public Integer getSeed() { return seed; } diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java new file mode 100644 index 000000000..0e5a9121d --- /dev/null +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java @@ -0,0 +1,52 @@ +package com.composum.ai.backend.base.service.chat.impl.chatmodel; + +import com.google.gson.annotations.SerializedName; + +public class ChatFunctionDetails { + + @SerializedName("name") + private String name; + + @SerializedName("description") + private String description; + + @SerializedName("parameters") + private Object parameters; // Arbitrary JSON schema + + @SerializedName("strict") + private Boolean strict; + + // Getters and setters + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Object getParameters() { + return parameters; + } + + public void setParameters(Object parameters) { + this.parameters = parameters; + } + + public Boolean getStrict() { + return strict; + } + + public void setStrict(Boolean strict) { + this.strict = strict; + } +} diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java new file mode 100644 index 000000000..d3ec6b007 --- /dev/null +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java @@ -0,0 +1,30 @@ +package com.composum.ai.backend.base.service.chat.impl.chatmodel; + +import com.google.gson.annotations.SerializedName; + +public class ChatTool { + + @SerializedName("type") + private String type; // Always "function" + + @SerializedName("function") + private ChatFunctionDetails function; // Function details object + + // Getters and setters + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public ChatFunctionDetails getFunction() { + return function; + } + + public void setFunction(ChatFunctionDetails function) { + this.function = function; + } +} \ No newline at end of file From 54d8d76a6c22bf6e5a47f523dfff39f35281ace8 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Mon, 14 Oct 2024 17:00:28 +0200 Subject: [PATCH 05/44] co-developer adds comments to chat package --- .../impl/chatmodel/ChatCompletionChoice.java | 16 +++++ .../ChatCompletionChoiceMessage.java | 10 +++ .../impl/chatmodel/ChatCompletionMessage.java | 39 ++++++----- .../chatmodel/ChatCompletionMessagePart.java | 67 ++++++++++--------- .../impl/chatmodel/ChatCompletionRequest.java | 48 +++++++++---- .../chatmodel/ChatCompletionResponse.java | 25 +++++++ .../impl/chatmodel/ChatCompletionUsage.java | 13 ++++ .../impl/chatmodel/ChatFunctionDetails.java | 16 +++++ .../service/chat/impl/chatmodel/ChatTool.java | 12 +++- 9 files changed, 187 insertions(+), 59 deletions(-) diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoice.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoice.java index 051ab479b..18b172d72 100644 --- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoice.java +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoice.java @@ -2,17 +2,33 @@ import com.google.gson.annotations.SerializedName; +/** + * Represents a choice in the chat completion response. Each choice may include a message, + * a delta (for streaming responses), and a finish reason indicating why the completion stopped. + */ public class ChatCompletionChoice { + /** + * The position of this choice in the list of choices returned by the API. + */ @SerializedName("index") private int index; + /** + * The message content associated with this choice. + */ @SerializedName("message") private ChatCompletionChoiceMessage message; + /** + * Used for incremental updates (streaming responses), represents partial message content. + */ @SerializedName("delta") private ChatCompletionChoiceMessage delta; + /** + * The reason why the completion stopped (e.g., length, stop signal). + */ @SerializedName("finish_reason") private ChatCompletionResponse.FinishReason finishReason; diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java index 4913a8968..50cae77d5 100644 --- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java @@ -2,11 +2,21 @@ import com.google.gson.annotations.SerializedName; +/** + * Represents the message content in a chat completion choice, including the role (e.g., user, assistant) + * and the actual text content of the message. + */ public class ChatCompletionChoiceMessage { + /** + * The role of the message (e.g., user, assistant, or system). + */ @SerializedName("role") private ChatCompletionRequest.Role role; + /** + * The text content of the message. + */ @SerializedName("content") private String content; diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java index 720c8328d..3b3088ba8 100644 --- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java @@ -4,19 +4,42 @@ import java.util.List; import com.composum.ai.backend.base.service.chat.GPTChatMessage; -import com.google.gson.annotations.Expose; import com.google.gson.annotations.JsonAdapter; import com.google.gson.annotations.SerializedName; +/** + * Represents a message in a chat completion request, containing the role of the speaker + * (user, assistant, or system) and the message content, which may include text or other parts. + */ public class ChatCompletionMessage { + /** + * The role of the speaker for this message, such as 'user', 'assistant', or 'system'. + */ @SerializedName("role") private ChatCompletionRequest.Role role; + /** + * The content of the message, which may include text or other parts (like images). + */ @SerializedName("content") @JsonAdapter(ChatCompletionMessagePart.ChatCompletionMessagePartListDeSerializer.class) private List content; + public static ChatCompletionMessage make(GPTChatMessage message) { + ChatCompletionMessagePart part; + if (message.getImageUrl() != null && !message.getImageUrl().isEmpty()) { + part = ChatCompletionMessagePart.imageUrl(message.getImageUrl()); + } else { + part = ChatCompletionMessagePart.text(message.getContent()); + } + ChatCompletionRequest.Role role = ChatCompletionRequest.Role.make(message.getRole()); + ChatCompletionMessage result = new ChatCompletionMessage(); + result.setRole(role); + result.setContent(Collections.singletonList(part)); + return result; + } + // Getters and setters public ChatCompletionRequest.Role getRole() { return role; @@ -38,18 +61,4 @@ public boolean isEmpty(Void ignoreJustPreventSerialization) { return content == null || content.isEmpty() || !content.stream().anyMatch(m -> !m.isEmpty(null)); } - - public static ChatCompletionMessage make(GPTChatMessage message) { - ChatCompletionMessagePart part; - if (message.getImageUrl() != null && !message.getImageUrl().isEmpty()) { - part = ChatCompletionMessagePart.imageUrl(message.getImageUrl()); - } else { - part = ChatCompletionMessagePart.text(message.getContent()); - } - ChatCompletionRequest.Role role = ChatCompletionRequest.Role.make(message.getRole()); - ChatCompletionMessage result = new ChatCompletionMessage(); - result.setRole(role); - result.setContent(Collections.singletonList(part)); - return result; - } } diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessagePart.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessagePart.java index b6ec5630b..b9bab079d 100644 --- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessagePart.java +++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessagePart.java @@ -12,7 +12,8 @@ import com.google.gson.annotations.SerializedName; /** - * A text part or image part of a chat completion message. + * Represents a part of a chat completion message, which may be a text or an image URL. + * This allows messages to include multiple types of content. * 

  *          {
  *           "type": "text",
@@ -29,24 +30,40 @@
  */
 public class ChatCompletionMessagePart {
 
-    public enum Type {
-        @SerializedName("text")
-        TEXT,
-        @SerializedName("image_url")
-        IMAGE_URL
-    }
-
+    /**
+     * The type of this message part, either 'text' or 'image_url'.
+     */
     @SerializedName("type")
     private Type type;
-
+    /**
+     * The text content of this message part, used when the type is 'text'.
+     */
     @SerializedName("text")
     private String text;
-
+    /**
+     * The image URL content of this message part, used when the type is 'image_url'.
+     */
     @SerializedName("image_url")
     private ChatCompletionMessageUrlPart imageUrl;
 
+    public static ChatCompletionMessagePart text(String text) {
+        ChatCompletionMessagePart part = new ChatCompletionMessagePart();
+        part.setType(Type.TEXT);
+        part.setText(text);
+        return part;
+    }
+
     // Getters and setters
 
+    public static ChatCompletionMessagePart imageUrl(String imageUrl) {
+        ChatCompletionMessagePart part = new ChatCompletionMessagePart();
+        part.setType(Type.IMAGE_URL);
+        ChatCompletionMessageUrlPart urlpart = new ChatCompletionMessageUrlPart();
+        urlpart.setUrl(imageUrl);
+        part.setImageUrl(urlpart);
+        return part;
+    }
+
     public Type getType() {
         return type;
     }
@@ -76,20 +93,18 @@ public boolean isEmpty(Void ignoreJustPreventSerialization) {
                 (imageUrl == null || imageUrl.getUrl() == null || imageUrl.getUrl().isEmpty());
     }
 
-    public static ChatCompletionMessagePart text(String text) {
-        ChatCompletionMessagePart part = new ChatCompletionMessagePart();
-        part.setType(Type.TEXT);
-        part.setText(text);
-        return part;
+    public enum Type {
+        @SerializedName("text")
+        TEXT,
+        @SerializedName("image_url")
+        IMAGE_URL
     }
 
-    public static ChatCompletionMessagePart imageUrl(String imageUrl) {
-        ChatCompletionMessagePart part = new ChatCompletionMessagePart();
-        part.setType(Type.IMAGE_URL);
-        ChatCompletionMessageUrlPart urlpart = new ChatCompletionMessageUrlPart();
-        urlpart.setUrl(imageUrl);
-        part.setImageUrl(urlpart);
-        return part;
+    public enum ImageDetail {
+        @SerializedName("low")
+        LOW,
+        @SerializedName("high")
+        HIGH
     }
 
     /**
@@ -123,14 +138,6 @@ public void setDetail(ImageDetail detail) {
 
     }
 
-    public enum ImageDetail {
-        @SerializedName("low")
-        LOW,
-        @SerializedName("high")
-        HIGH
-    }
-
-
     public static class ChatCompletionMessagePartListDeSerializer implements JsonDeserializer>,
             JsonSerializer> {
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java
index 574b86789..c77229e08 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java
@@ -4,34 +4,61 @@
 
 import com.composum.ai.backend.base.service.chat.GPTMessageRole;
 import com.google.gson.annotations.SerializedName;
-import java.util.List;
 
+/**
+ * Represents a request to the OpenAI chat completion API, including model, messages,
+ * and optional parameters like max tokens, temperature, and response format.
+ */
 public class ChatCompletionRequest {
 
+    public static final ResponseFormat JSON = new ResponseFormat();
+    /**
+     * The AI model to use for the chat completion request, e.g., "gpt-4".
+     */
     @SerializedName("model")
     private String model;
-
+    /**
+     * The list of messages in the conversation, each with a role (user, assistant, system) and content.
+     */
     @SerializedName("messages")
     private List messages;
-
+    /**
+     * The maximum number of tokens to generate in the completion.
+     */
     @SerializedName("max_tokens")
     private Integer maxTokens;
-
+    /**
+     * Whether to stream the response incrementally.
+     */
     @SerializedName("stream")
     private Boolean stream;
-
+    /**
+     * The sampling temperature, used to control randomness. Values closer to 0 make the output more deterministic.
+     */
     @SerializedName("temperature")
     private Double temperature;
-
+    /**
+     * The format of the response. Possible values are "text" or "json_object".
+     */
     @SerializedName("response_format")
     private ResponseFormat responseFormat;
-
+    /**
+     * A seed for deterministic generation, useful for testing or reproducible results.
+     */
     @SerializedName("seed")
     private Integer seed;
-
+    /**
+     * A list of tools (functions) the model can call during the chat. Each tool contains a type and function details.
+     */
     @SerializedName("tools")
     private List tools;
 
+    {
+        {
+            JSON.setType(ResponseFormatType.JSON_OBJECT);
+        }
+    }
+
     // Getters and setters
     public String getModel() {
         return model;
@@ -150,9 +177,4 @@ public void setType(ResponseFormatType type) {
         }
     }
 
-    public static final ResponseFormat JSON = new ResponseFormat();
-    {{
-        JSON.setType(ResponseFormatType.JSON_OBJECT);
-    }};
-
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
index ce9eb147d..911f77f28 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
@@ -5,26 +5,51 @@
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.google.gson.annotations.SerializedName;
 
+/**
+ * Represents the response from the OpenAI chat completion API, containing details
+ * about the generated choices, token usage, and metadata like the model used and creation time.
+ */
 public class ChatCompletionResponse {
 
+    /**
+     * The unique identifier for this chat completion response.
+     */
     @SerializedName("id")
     private String id;
 
+    /**
+     * The type of object returned, typically 'chat.completion'.
+     */
     @SerializedName("object")
     private String object;
 
+    /**
+     * The timestamp (in epoch seconds) when this response was created.
+     */
     @SerializedName("created")
     private long created;
 
+    /**
+     * The model used for this chat completion, e.g., 'gpt-4'.
+     */
     @SerializedName("model")
     private String model;
 
+    /**
+     * An optional fingerprint of the system that generated this response.
+     */
     @SerializedName("system_fingerprint")
     private String systemFingerprint;
 
+    /**
+     * The list of choices the model generated, each with a message and finish reason.
+     */
     @SerializedName("choices")
     private List choices;
 
+    /**
+     * Token usage information for this completion, including total, prompt, and completion tokens.
+     */
     @SerializedName("usage")
     private ChatCompletionUsage usage;
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionUsage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionUsage.java
index 00abed741..62ad1d7db 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionUsage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionUsage.java
@@ -2,14 +2,27 @@
 
 import com.google.gson.annotations.SerializedName;
 
+/**
+ * Represents the token usage details in a chat completion response, including
+ * the number of tokens used for the prompt, the completion, and the total.
+ */
 public class ChatCompletionUsage {
 
+    /**
+     * The number of tokens used for the prompt (input) in this completion.
+     */
     @SerializedName("prompt_tokens")
     private int promptTokens;
 
+    /**
+     * The number of tokens generated in the completion (output).
+     */
     @SerializedName("completion_tokens")
     private int completionTokens;
 
+    /**
+     * The total number of tokens used (prompt + completion).
+     */
     @SerializedName("total_tokens")
     private int totalTokens;
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java
index 0e5a9121d..ca244db3e 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java
@@ -2,17 +2,33 @@
 
 import com.google.gson.annotations.SerializedName;
 
+/**
+ * Represents the details of a function used as a tool in the chat completion request.
+ * Includes the function's name, description, parameters, and an optional strict flag.
+ */
 public class ChatFunctionDetails {
 
+    /**
+     * The name of the function to be called. This must be unique and can only contain a-z, A-Z, 0-9, underscores, and dashes.
+     */
     @SerializedName("name")
     private String name;
 
+    /**
+     * A brief description of what the function does. Helps the model choose when to call it.
+     */
     @SerializedName("description")
     private String description;
 
+    /**
+     * The parameters accepted by the function, defined as an arbitrary JSON schema object.
+     */
     @SerializedName("parameters")
     private Object parameters;  // Arbitrary JSON schema
 
+    /**
+     * Whether to enforce strict schema adherence for the parameters.
+     */
     @SerializedName("strict")
     private Boolean strict;
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
index d3ec6b007..e0fb08bf5 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
@@ -2,11 +2,21 @@
 
 import com.google.gson.annotations.SerializedName;
 
+/**
+ * Represents a tool in the OpenAI chat completion request, currently limited to functions.
+ * Each tool contains details about the function including name, description, and parameters.
+ */
 public class ChatTool {
 
+    /**
+     * The type of the tool, currently fixed as "function".
+     */
     @SerializedName("type")
     private String type;  // Always "function"
 
+    /**
+     * The details of the function, such as its name, description, and parameters.
+     */
     @SerializedName("function")
     private ChatFunctionDetails function;  // Function details object
 
@@ -27,4 +37,4 @@ public ChatFunctionDetails getFunction() {
     public void setFunction(ChatFunctionDetails function) {
         this.function = function;
     }
-}
\ No newline at end of file
+}

From fe643235f5d92efeaf284320847ee6d14a681fd7 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Mon, 14 Oct 2024 21:53:07 +0200
Subject: [PATCH 06/44] add tool call to response

---
 .../ChatCompletionChoiceMessage.java          | 18 ++++++-
 .../chatmodel/ChatCompletionToolCall.java     | 54 +++++++++++++++++++
 2 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
index 50cae77d5..1da9b2b9b 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
@@ -20,7 +20,13 @@ public class ChatCompletionChoiceMessage {
     @SerializedName("content")
     private String content;
 
-    // Getters and setters
+    /**
+ * The tool calls generated by the model, such as function calls.
+ */
+@SerializedName("tool_calls")
+private List toolCalls;
+
+// Getters and setters
     public ChatCompletionRequest.Role getRole() {
         return role;
     }
@@ -29,7 +35,15 @@ public void setRole(ChatCompletionRequest.Role role) {
         this.role = role;
     }
 
-    public String getContent() {
+    public List getToolCalls() {
+    return toolCalls;
+}
+
+public void setToolCalls(List toolCalls) {
+    this.toolCalls = toolCalls;
+}
+
+public String getContent() {
         return content;
     }
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
new file mode 100644
index 000000000..ea2c343cf
--- /dev/null
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
@@ -0,0 +1,54 @@
+package com.composum.ai.backend.base.service.chat.impl.chatmodel;
+
+import com.google.gson.annotations.SerializedName;
+
+/**
+ * Represents a tool call generated by the model in a chat completion response.
+ * This can be a function call with specific arguments.
+ */
+public class ChatCompletionToolCall {
+
+    /**
+     * The ID of the tool call.
+     */
+    @SerializedName("id")
+    private String id;
+
+    /**
+     * The type of the tool, currently only "function" is supported.
+     */
+    @SerializedName("type")
+    private String type;
+
+    /**
+     * The function being called by the model, including its name and arguments.
+     */
+    @SerializedName("function")
+    private ChatFunctionDetails function;
+
+    // Getters and setters
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public ChatFunctionDetails getFunction() {
+        return function;
+    }
+
+    public void setFunction(ChatFunctionDetails function) {
+        this.function = function;
+    }
+}

From e3ffad565dd5d9482203dbc146d7e8d972d04618 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Mon, 14 Oct 2024 21:56:10 +0200
Subject: [PATCH 07/44] tool messages in input

---
 .../ChatCompletionChoiceMessage.java          | 24 ++++++++++---------
 .../impl/chatmodel/ChatCompletionMessage.java | 17 ++++++++++++-
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
index 1da9b2b9b..a7d2e6ccd 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
@@ -1,5 +1,7 @@
 package com.composum.ai.backend.base.service.chat.impl.chatmodel;
 
+import java.util.List;
+
 import com.google.gson.annotations.SerializedName;
 
 /**
@@ -21,12 +23,12 @@ public class ChatCompletionChoiceMessage {
     private String content;
 
     /**
- * The tool calls generated by the model, such as function calls.
- */
-@SerializedName("tool_calls")
-private List toolCalls;
+     * The tool calls generated by the model, such as function calls.
+     */
+    @SerializedName("tool_calls")
+    private List toolCalls;
 
-// Getters and setters
+    // Getters and setters
     public ChatCompletionRequest.Role getRole() {
         return role;
     }
@@ -36,14 +38,14 @@ public void setRole(ChatCompletionRequest.Role role) {
     }
 
     public List getToolCalls() {
-    return toolCalls;
-}
+        return toolCalls;
+    }
 
-public void setToolCalls(List toolCalls) {
-    this.toolCalls = toolCalls;
-}
+    public void setToolCalls(List toolCalls) {
+        this.toolCalls = toolCalls;
+    }
 
-public String getContent() {
+    public String getContent() {
         return content;
     }
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
index 3b3088ba8..44c7699d8 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
@@ -14,7 +14,7 @@
 public class ChatCompletionMessage {
 
     /**
-     * The role of the speaker for this message, such as 'user', 'assistant', or 'system'.
+     * The role of the speaker for this message, such as 'user', 'assistant', 'system', or 'tool'.
      */
     @SerializedName("role")
     private ChatCompletionRequest.Role role;
@@ -26,6 +26,13 @@ public class ChatCompletionMessage {
     @JsonAdapter(ChatCompletionMessagePart.ChatCompletionMessagePartListDeSerializer.class)
     private List content;
 
+    /**
+     * The ID of the tool call that this message is responding to.
+     * Only applicable when the role is 'tool'.
+     */
+    @SerializedName("tool_call_id")
+    private String toolCallId;
+
     public static ChatCompletionMessage make(GPTChatMessage message) {
         ChatCompletionMessagePart part;
         if (message.getImageUrl() != null && !message.getImageUrl().isEmpty()) {
@@ -49,6 +56,14 @@ public void setRole(ChatCompletionRequest.Role role) {
         this.role = role;
     }
 
+    public String getToolCallId() {
+        return toolCallId;
+    }
+
+    public void setToolCallId(String toolCallId) {
+        this.toolCallId = toolCallId;
+    }
+
     public List getContent() {
         return content;
     }

From 53cc6b6c73ed1533e7581a04a02162155493b534 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 15 Oct 2024 09:17:30 +0200
Subject: [PATCH 08/44] implement tool request

---
 .../impl/GPTChatCompletionServiceImpl.java    | 23 ++++++++
 .../service/chat/impl/chatmodel/ChatTool.java |  2 +-
 ...GPTChatCompletionServiceImplWithTools.java | 58 ++++++++++++++++++-
 3 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index 3704cb5f3..3d5a99997 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -75,11 +75,14 @@
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
+import com.composum.ai.backend.base.service.chat.GPTTool;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionChoice;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessage;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessagePart;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionRequest;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionResponse;
+import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatFunctionDetails;
+import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatTool;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.OpenAIEmbeddings;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.gson.Gson;
@@ -618,11 +621,31 @@ protected String createJsonRequest(GPTChatRequest request) throws JsonProcessing
             externalRequest.setMaxTokens(maxTokens);
         }
         externalRequest.setStream(Boolean.TRUE);
+        externalRequest.setTools(convertTools(request.getConfiguration()));
         String jsonRequest = gson.toJson(externalRequest);
         checkTokenCount(jsonRequest);
         return jsonRequest;
     }
 
+    private List convertTools(GPTConfiguration configuration) {
+        if (configuration == null || configuration.getTools() == null || configuration.getTools().isEmpty()) {
+            return null;
+        }
+        List result = new ArrayList<>();
+        for (GPTTool tool : configuration.getTools()) {
+            ChatTool toolDescr = new ChatTool();
+            ChatFunctionDetails details = new ChatFunctionDetails();
+            details.setName(tool.getName());
+            details.setStrict(true);
+            Map declaration = gson.fromJson(tool.getToolDeclaration(), Map.class);
+            details.setParameters(declaration.get("description"));
+            details.setParameters(((Map)declaration.get("function")).get("parameters"));
+            toolDescr.setFunction(details);
+            result.add(toolDescr);
+        }
+        return result;
+    }
+
     protected void checkEnabled() {
         if (!isEnabled()) {
             throw new IllegalStateException("Not enabled or no API key configured for the GPT chat completion service. Please configure the service.");
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
index e0fb08bf5..c7fdca415 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
@@ -12,7 +12,7 @@ public class ChatTool {
      * The type of the tool, currently fixed as "function".
      */
     @SerializedName("type")
-    private String type;  // Always "function"
+    private String type = "function";  // currently Always "function"
 
     /**
      * The details of the function, such as its name, description, and parameters.
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
index 71412c044..1a396438e 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
@@ -1,9 +1,16 @@
 package com.composum.ai.backend.base.service.chat.impl;
 
+import java.util.Arrays;
+import java.util.Map;
+
 import com.composum.ai.backend.base.service.chat.GPTChatRequest;
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
+import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.composum.ai.backend.base.service.chat.GPTMessageRole;
+import com.composum.ai.backend.base.service.chat.GPTTool;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 
 /**
  * Tries an actual call to ChatGPT with the streaming interface. Since that costs money (though much less than a cent),
@@ -12,6 +19,7 @@
 public class RunGPTChatCompletionServiceImplWithTools extends AbstractGPTRunner implements GPTCompletionCallback {
 
     StringBuilder buffer = new StringBuilder();
+    Gson gson = new GsonBuilder().setPrettyPrinting().create();
     private boolean isFinished;
 
     public static void main(String[] args) throws Exception {
@@ -24,7 +32,8 @@ public static void main(String[] args) throws Exception {
 
     private void run() throws InterruptedException {
         GPTChatRequest request = new GPTChatRequest();
-        request.addMessage(GPTMessageRole.USER, "Make 2 haiku about the weather.");
+        request.addMessage(GPTMessageRole.USER, "Make 2 haiku about the weather and then wobble them.");
+        request.setConfiguration(GPTConfiguration.ofTools(Arrays.asList(wobbler)));
         chatCompletionService.streamingChatCompletion(request, this);
         System.out.println("Call returned.");
         while (!isFinished) Thread.sleep(1000);
@@ -55,4 +64,51 @@ public void onError(Throwable throwable) {
         throwable.printStackTrace(System.err);
         isFinished = true;
     }
+
+    protected GPTTool wobbler = new GPTTool() {
+        @Override
+        public String getName() {
+            return "wobbler";
+        }
+
+        @Override
+        public String getToolDeclaration() {
+            return "{\n" +
+                    "  \"type\": \"function\",\n" +
+                    "  \"function\": {\n" +
+                    "    \"name\": \"wobbler\",\n" +
+                    "    \"description\": \"wobbles the string given as argument\",\n" +
+                    "    \"parameters\": {\n" +
+                    "      \"type\": \"object\",\n" +
+                    "      \"properties\": {\n" +
+                    "        \"towobble\": {\n" +
+                    "          \"type\": \"string\",\n" +
+                    "          \"description\": \"The string to wobble.\"\n" +
+                    "        }\n" +
+                    "      },\n" +
+                    "      \"required\": [\"towobble\"],\n" +
+                    "      \"additionalProperties\": false\n" +
+                    "    }\n" +
+                    "  },\n" +
+                    "  \"strict\": true\n" +
+                    "}";
+        }
+
+        @Override
+        public String execute(String arguments) {
+            Map parsedArguments = gson.fromJson(arguments, Map.class);
+            String towobble = (String) parsedArguments.get("towobble");
+            StringBuilder result = new StringBuilder();
+            for (int i = 0; i < towobble.length(); i++) {
+                char c = towobble.charAt(i);
+                if (i % 2 == 0) {
+                    result.append(Character.toUpperCase(c));
+                } else {
+                    result.append(Character.toLowerCase(c));
+                }
+            }
+            return result.toString();
+        }
+    };
+
 }

From a9c8263994faec84d89bb5c07fec4ca544e2a318 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 15 Oct 2024 20:20:41 +0200
Subject: [PATCH 09/44] start to parse tools response

---
 .../service/chat/GPTCompletionCallback.java   |  5 +++
 .../impl/GPTChatCompletionServiceImpl.java    |  9 ++++-
 .../ChatCompletionChoiceMessage.java          |  6 +--
 .../ChatCompletionFunctionCallDetails.java    | 38 +++++++++++++++++++
 ...ava => ChatCompletionFunctionDetails.java} |  2 +-
 .../chatmodel/ChatCompletionResponse.java     |  8 +++-
 .../chatmodel/ChatCompletionToolCall.java     |  6 +--
 .../service/chat/impl/chatmodel/ChatTool.java |  6 +--
 ...GPTChatCompletionServiceImplWithTools.java | 13 ++++++-
 9 files changed, 79 insertions(+), 14 deletions(-)
 create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
 rename backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/{ChatFunctionDetails.java => ChatCompletionFunctionDetails.java} (97%)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
index 521f41b77..5884086d4 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
@@ -35,6 +35,11 @@ public interface GPTCompletionCallback {
     default void setRequest(String json) {
     }
 
+    /** Notifies that the request is completely finished / closed, nothing more will arrive. */
+    default void close() {
+        // empty
+    }
+
     /**
      * A simple collector that just takes note of things.
      */
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index 3d5a99997..8b84434f2 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -1,6 +1,7 @@
 package com.composum.ai.backend.base.service.chat.impl;
 
 import static java.util.Collections.singletonList;
+import static java.util.Collections.sort;
 
 import java.io.IOException;
 import java.io.StringWriter;
@@ -81,7 +82,7 @@
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessagePart;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionRequest;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionResponse;
-import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatFunctionDetails;
+import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionFunctionDetails;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatTool;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.OpenAIEmbeddings;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -441,6 +442,7 @@ protected void handleStreamingEvent(GPTCompletionCallback callback, long id, Str
             try {
                 if (" [DONE]".equals(line)) {
                     LOG.debug("Response {} from GPT received DONE", id);
+                    callback.close();
                     return;
                 }
                 ChatCompletionResponse chunk = gson.fromJson(line, ChatCompletionResponse.class);
@@ -456,6 +458,9 @@ protected void handleStreamingEvent(GPTCompletionCallback callback, long id, Str
                     LOG.trace("Response {} from GPT: {}", id, content);
                     callback.onNext(content);
                 }
+                if (choice.getFinishReason() != null) {
+                    System.out.println("Response {} from GPT finished with reason {}" + id + choice.getFinishReason());
+                }
                 GPTFinishReason finishReason = ChatCompletionResponse.FinishReason.toGPTFinishReason(choice.getFinishReason());
                 if (finishReason != null) {
                     LOG.debug("Response {} from GPT finished with reason {}", id, finishReason);
@@ -634,7 +639,7 @@ private List convertTools(GPTConfiguration configuration) {
         List result = new ArrayList<>();
         for (GPTTool tool : configuration.getTools()) {
             ChatTool toolDescr = new ChatTool();
-            ChatFunctionDetails details = new ChatFunctionDetails();
+            ChatCompletionFunctionDetails details = new ChatCompletionFunctionDetails();
             details.setName(tool.getName());
             details.setStrict(true);
             Map declaration = gson.fromJson(tool.getToolDeclaration(), Map.class);
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
index a7d2e6ccd..433b1fd93 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
@@ -26,7 +26,7 @@ public class ChatCompletionChoiceMessage {
      * The tool calls generated by the model, such as function calls.
      */
     @SerializedName("tool_calls")
-    private List toolCalls;
+    private List toolCalls;
 
     // Getters and setters
     public ChatCompletionRequest.Role getRole() {
@@ -37,11 +37,11 @@ public void setRole(ChatCompletionRequest.Role role) {
         this.role = role;
     }
 
-    public List getToolCalls() {
+    public List getToolCalls() {
         return toolCalls;
     }
 
-    public void setToolCalls(List toolCalls) {
+    public void setToolCalls(List toolCalls) {
         this.toolCalls = toolCalls;
     }
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
new file mode 100644
index 000000000..5283acdf3
--- /dev/null
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
@@ -0,0 +1,38 @@
+package com.composum.ai.backend.base.service.chat.impl.chatmodel;
+
+import com.google.gson.annotations.SerializedName;
+
+/**
+ * Represents the a call of a function used as a tool in the chat completion request.
+ */
+public class ChatCompletionFunctionCallDetails {
+
+    /**
+     * The name of the function to be called. This must be unique and can only contain a-z, A-Z, 0-9, underscores, and dashes.
+     */
+    @SerializedName("name")
+    private String name;
+
+    /**
+     * A JSON for the arguments the function is called with.
+     */
+    @SerializedName("parameters")
+    private String arguments;
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getArguments() {
+        return arguments;
+    }
+
+    public void setArguments(String arguments) {
+        this.arguments = arguments;
+    }
+
+}
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionDetails.java
similarity index 97%
rename from backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java
rename to backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionDetails.java
index ca244db3e..37452299e 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatFunctionDetails.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionDetails.java
@@ -6,7 +6,7 @@
  * Represents the details of a function used as a tool in the chat completion request.
  * Includes the function's name, description, parameters, and an optional strict flag.
  */
-public class ChatFunctionDetails {
+public class ChatCompletionFunctionDetails {
 
     /**
      * The name of the function to be called. This must be unique and can only contain a-z, A-Z, 0-9, underscores, and dashes.
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
index 911f77f28..07fa86e45 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
@@ -116,7 +116,13 @@ public enum FinishReason {
         @SerializedName("length")
         LENGTH,
         @SerializedName("content_filter")
-        CONTENT_FILTER;
+        CONTENT_FILTER,
+        @SerializedName("tool_calls")
+        TOOL_CALLS,
+        @Deprecated
+        @SerializedName("function_call")
+        FUNCTION_CALL,
+        ;
 
         public static GPTFinishReason toGPTFinishReason(FinishReason finishReason) {
             if (finishReason == null) {
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
index ea2c343cf..1d7e24b42 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
@@ -24,7 +24,7 @@ public class ChatCompletionToolCall {
      * The function being called by the model, including its name and arguments.
      */
     @SerializedName("function")
-    private ChatFunctionDetails function;
+    private ChatCompletionFunctionDetails function;
 
     // Getters and setters
 
@@ -44,11 +44,11 @@ public void setType(String type) {
         this.type = type;
     }
 
-    public ChatFunctionDetails getFunction() {
+    public ChatCompletionFunctionDetails getFunction() {
         return function;
     }
 
-    public void setFunction(ChatFunctionDetails function) {
+    public void setFunction(ChatCompletionFunctionDetails function) {
         this.function = function;
     }
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
index c7fdca415..b2713f7e1 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatTool.java
@@ -18,7 +18,7 @@ public class ChatTool {
      * The details of the function, such as its name, description, and parameters.
      */
     @SerializedName("function")
-    private ChatFunctionDetails function;  // Function details object
+    private ChatCompletionFunctionDetails function;  // Function details object
 
     // Getters and setters
 
@@ -30,11 +30,11 @@ public void setType(String type) {
         this.type = type;
     }
 
-    public ChatFunctionDetails getFunction() {
+    public ChatCompletionFunctionDetails getFunction() {
         return function;
     }
 
-    public void setFunction(ChatFunctionDetails function) {
+    public void setFunction(ChatCompletionFunctionDetails function) {
         this.function = function;
     }
 }
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
index 1a396438e..b41afda29 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
@@ -32,7 +32,7 @@ public static void main(String[] args) throws Exception {
 
     private void run() throws InterruptedException {
         GPTChatRequest request = new GPTChatRequest();
-        request.addMessage(GPTMessageRole.USER, "Make 2 haiku about the weather and then wobble them.");
+        request.addMessage(GPTMessageRole.USER, "Wobble the string 'hi'.");
         request.setConfiguration(GPTConfiguration.ofTools(Arrays.asList(wobbler)));
         chatCompletionService.streamingChatCompletion(request, this);
         System.out.println("Call returned.");
@@ -112,3 +112,14 @@ public String execute(String arguments) {
     };
 
 }
+// sequence of data for a tool call:
+//  {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_ZkuSWztTvaOxQeT6TFITm3sa","type":"function","function":{"name":"wobbler","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"tow"}}]},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ob"}}]},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ble"}}]},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"hi"}}]},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}]}
+// {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]}
+// [DONE]

From b667f7125775394dcd606b5b0ecbb1bfe4f36f0c Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 15 Oct 2024 20:59:03 +0200
Subject: [PATCH 10/44] fixes

---
 .../ai/backend/base/service/chat/GPTFinishReason.java      | 6 ++++++
 .../service/chat/impl/GPTChatCompletionServiceImpl.java    | 4 ++--
 .../chat/impl/chatmodel/ChatCompletionChoiceMessage.java   | 6 +++---
 .../impl/chatmodel/ChatCompletionFunctionCallDetails.java  | 2 +-
 .../chat/impl/chatmodel/ChatCompletionResponse.java        | 4 ++++
 .../chat/impl/chatmodel/ChatCompletionToolCall.java        | 6 +++---
 .../impl/RunGPTChatCompletionServiceImplWithTools.java     | 7 ++++++-
 7 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFinishReason.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFinishReason.java
index 4fef2d30f..e7761dc5f 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFinishReason.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFinishReason.java
@@ -17,8 +17,14 @@ public enum GPTFinishReason {
     /**
      * Model decided to call a function.
      */
+    @Deprecated
     FUNCTION_CALL,
 
+    /**
+     * Model decided to call one or more tools.
+     */
+    TOOL_CALLS,
+
     /**
      * Omitted content due to a flag from our content filters.
      */
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index 8b84434f2..53f136bc4 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -1,7 +1,6 @@
 package com.composum.ai.backend.base.service.chat.impl;
 
 import static java.util.Collections.singletonList;
-import static java.util.Collections.sort;
 
 import java.io.IOException;
 import java.io.StringWriter;
@@ -78,11 +77,11 @@
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.composum.ai.backend.base.service.chat.GPTTool;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionChoice;
+import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionFunctionDetails;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessage;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessagePart;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionRequest;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionResponse;
-import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionFunctionDetails;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatTool;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.OpenAIEmbeddings;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -439,6 +438,7 @@ public void streamingChatCompletion(@Nonnull GPTChatRequest request, @Nonnull GP
     protected void handleStreamingEvent(GPTCompletionCallback callback, long id, String line) {
         if (line.startsWith("data:")) {
             line = line.substring(MAXTRIES);
+            System.out.println(line);
             try {
                 if (" [DONE]".equals(line)) {
                     LOG.debug("Response {} from GPT received DONE", id);
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
index 433b1fd93..a7d2e6ccd 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionChoiceMessage.java
@@ -26,7 +26,7 @@ public class ChatCompletionChoiceMessage {
      * The tool calls generated by the model, such as function calls.
      */
     @SerializedName("tool_calls")
-    private List toolCalls;
+    private List toolCalls;
 
     // Getters and setters
     public ChatCompletionRequest.Role getRole() {
@@ -37,11 +37,11 @@ public void setRole(ChatCompletionRequest.Role role) {
         this.role = role;
     }
 
-    public List getToolCalls() {
+    public List getToolCalls() {
         return toolCalls;
     }
 
-    public void setToolCalls(List toolCalls) {
+    public void setToolCalls(List toolCalls) {
         this.toolCalls = toolCalls;
     }
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
index 5283acdf3..755b78d87 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
@@ -16,7 +16,7 @@ public class ChatCompletionFunctionCallDetails {
     /**
      * A JSON for the arguments the function is called with.
      */
-    @SerializedName("parameters")
+    @SerializedName("arguments")
     private String arguments;
 
     public String getName() {
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
index 07fa86e45..ba7027afb 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionResponse.java
@@ -135,6 +135,10 @@ public static GPTFinishReason toGPTFinishReason(FinishReason finishReason) {
                     return GPTFinishReason.LENGTH;
                 case CONTENT_FILTER:
                     return GPTFinishReason.CONTENT_FILTER;
+                case TOOL_CALLS:
+                    return GPTFinishReason.TOOL_CALLS;
+                case FUNCTION_CALL:
+                    return GPTFinishReason.FUNCTION_CALL;
                 default:
                     throw new IllegalArgumentException("Unknown finish reason: " + finishReason);
             }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
index 1d7e24b42..099725b2c 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
@@ -24,7 +24,7 @@ public class ChatCompletionToolCall {
      * The function being called by the model, including its name and arguments.
      */
     @SerializedName("function")
-    private ChatCompletionFunctionDetails function;
+    private ChatCompletionFunctionCallDetails function;
 
     // Getters and setters
 
@@ -44,11 +44,11 @@ public void setType(String type) {
         this.type = type;
     }
 
-    public ChatCompletionFunctionDetails getFunction() {
+    public ChatCompletionFunctionCallDetails getFunction() {
         return function;
     }
 
-    public void setFunction(ChatCompletionFunctionDetails function) {
+    public void setFunction(ChatCompletionFunctionCallDetails function) {
         this.function = function;
     }
 }
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
index b41afda29..469abe9a2 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
@@ -32,7 +32,7 @@ public static void main(String[] args) throws Exception {
 
     private void run() throws InterruptedException {
         GPTChatRequest request = new GPTChatRequest();
-        request.addMessage(GPTMessageRole.USER, "Wobble the string 'hi'.");
+        request.addMessage(GPTMessageRole.USER, "Wobble the string 'hi' and the string 'ho'.");
         request.setConfiguration(GPTConfiguration.ofTools(Arrays.asList(wobbler)));
         chatCompletionService.streamingChatCompletion(request, this);
         System.out.println("Call returned.");
@@ -123,3 +123,8 @@ public String execute(String arguments) {
 // {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}]}
 // {"id":"chatcmpl-AIXBVjvIrBwLMcdUfqBsCEVwteQuK","object":"chat.completion.chunk","created":1728980829,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]}
 // [DONE]
+
+// or two calls:
+//  {"id":"chatcmpl-AIgs1gFG3stX6MiLD2jQkeam50HJQ","object":"chat.completion.chunk","created":1729018061,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_g44GU4kWpf8Q8T3pCmB54GWa","type":"function","function":{"name":"wobbler","arguments":""}}]},"logprobs":null,"finish_reason":null}]}
+// ... and then ...
+//  {"id":"chatcmpl-AIgs1gFG3stX6MiLD2jQkeam50HJQ","object":"chat.completion.chunk","created":1729018061,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"o\"}"}}]},"logprobs":null,"finish_reason":null}]}

From 035dbaad4bb2ff670cd6a73aee34e5fb099cd889 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Mon, 28 Oct 2024 16:40:21 +0100
Subject: [PATCH 11/44] add unittest for full stream

---
 .../GPTChatCompletionServiceImplTest.java     | 38 +++++++++++++++++--
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplTest.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplTest.java
index d2f2c2bb5..10c4fd496 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplTest.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplTest.java
@@ -1,18 +1,20 @@
 package com.composum.ai.backend.base.service.chat.impl;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 
-import org.apache.sling.commons.threads.ThreadPoolManager;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.osgi.framework.BundleContext;
 
 import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
+import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.fasterxml.jackson.core.JsonProcessingException;
 
 /**
@@ -143,7 +145,7 @@ public void testHandleStreamingEventOK() throws JsonProcessingException {
                 "\"model\":\"gpt-3.5-turbo-0613\"," +
                 "\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hi\"},\"finish_reason\":null}]}";
         service.handleStreamingEvent(callback, 123, dataline);
-        Mockito.verify(callback).onNext("Hi");
+        verify(callback).onNext("Hi");
     }
 
     @Test
@@ -155,7 +157,35 @@ public void testHandleStreamingEventAdditionalFields() throws JsonProcessingExce
                 "\"somethingunknown\":28," +
                 "\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hi\"},\"finish_reason\":null}]}";
         service.handleStreamingEvent(callback, 123, dataline);
-        Mockito.verify(callback).onNext("Hi");
+        verify(callback).onNext("Hi");
+    }
+
+
+    /**
+     * Streaming test from real example.
+     */
+    @Test
+    public void testWholeStream() {
+        String[] lines = ("" +
+                "{\"id\":\"chatcmpl-ANLyWhiTkkyRHtRTJcwxGzUTNxiG6\",\"object\":\"chat.completion.chunk\",\"created\":1730129380,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f59a81427f\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANLyWhiTkkyRHtRTJcwxGzUTNxiG6\",\"object\":\"chat.completion.chunk\",\"created\":1730129380,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f59a81427f\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANLyWhiTkkyRHtRTJcwxGzUTNxiG6\",\"object\":\"chat.completion.chunk\",\"created\":1730129380,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f59a81427f\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANLyWhiTkkyRHtRTJcwxGzUTNxiG6\",\"object\":\"chat.completion.chunk\",\"created\":1730129380,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_f59a81427f\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n" +
+                "[DONE]"
+        ).split("\n");
+        GPTCompletionCallback callback = mock(GPTCompletionCallback.class);
+        StringBuffer result = new StringBuffer();
+        doAnswer(invocation -> {
+            String arg = invocation.getArgument(0);
+            result.append(arg);
+            return null;
+        }).when(callback).onNext(anyString());
+        for (String line : lines) {
+            service.handleStreamingEvent(callback, 123, "data: " + line);
+        }
+        assertEquals("Hello!", result.toString());
+        verify(callback).onFinish(GPTFinishReason.STOP);
+        verify(callback).close();
     }
 
 }

From 8855b54679da226d175cbbfe12520363544eb6e3 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Mon, 28 Oct 2024 21:19:11 +0100
Subject: [PATCH 12/44] reading and merging tool calls

---
 .../service/chat/GPTCompletionCallback.java   | 71 +++++++++++++++++-
 .../impl/GPTChatCompletionServiceImpl.java    |  3 +
 .../ChatCompletionFunctionCallDetails.java    | 16 ++++
 .../chatmodel/ChatCompletionToolCall.java     | 73 +++++++++++++++++++
 ...GPTChatCompletionServiceImplToolsTest.java | 73 +++++++++++++++++++
 5 files changed, 235 insertions(+), 1 deletion(-)
 create mode 100644 backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
index 5884086d4..cf04d91a9 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
@@ -1,8 +1,14 @@
 package com.composum.ai.backend.base.service.chat;
 
+import java.util.List;
+
+import javax.annotation.Nonnull;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionToolCall;
+
 /**
  * For a streaming mode this is given as parameter for the method call and receives the streamed data; the method returns only when the response is complete.
  */
@@ -35,11 +41,63 @@ public interface GPTCompletionCallback {
     default void setRequest(String json) {
     }
 
-    /** Notifies that the request is completely finished / closed, nothing more will arrive. */
+    /**
+     * Notifies that the request is completely finished / closed, nothing more will arrive.
+     */
     default void close() {
         // empty
     }
 
+    /**
+     * Called when a tool call is made.
+     */
+    default void toolDelta(List toolCalls) {
+        // empty
+    }
+
+
+    /**
+     * Forwards all methods to a delegate.
+     */
+    public static class GPTCompletionCallbackWrapper {
+
+        @Nonnull
+        protected GPTCompletionCallback delegate;
+
+        public GPTCompletionCallbackWrapper(@Nonnull GPTCompletionCallback delegate) {
+            this.delegate = delegate;
+        }
+
+        public void onFinish(GPTFinishReason finishReason) {
+            delegate.onFinish(finishReason);
+        }
+
+        public void onNext(String chars) {
+            delegate.onNext(chars);
+        }
+
+        public void onError(Throwable throwable) {
+            delegate.onError(throwable);
+        }
+
+        public void setLoggingId(String loggingId) {
+            delegate.setLoggingId(loggingId);
+        }
+
+        public void setRequest(String json) {
+            delegate.setRequest(json);
+        }
+
+        public void close() {
+            delegate.close();
+        }
+
+        public void toolDelta(List toolCalls) {
+            delegate.toolDelta(toolCalls);
+        }
+
+    }
+
     /**
      * A simple collector that just takes note of things.
      */
@@ -50,6 +108,7 @@ public static class GPTCompletionCollector implements GPTCompletionCallback {
         private StringBuilder buffer = new StringBuilder();
         private Throwable throwable;
         private GPTFinishReason finishReason;
+        private List toolCalls;
 
         @Override
         public void onFinish(GPTFinishReason finishReason) {
@@ -83,6 +142,16 @@ public String getResult() {
         public Throwable getError() {
             return throwable;
         }
+
+        @Override
+        public void toolDelta(List toolCalls) {
+            this.toolCalls = ChatCompletionToolCall.mergeDelta(this.toolCalls, toolCalls);
+        }
+
+        public List getToolCalls() {
+            return toolCalls;
+        }
+
     }
 
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index 53f136bc4..df872f8da 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -458,6 +458,9 @@ protected void handleStreamingEvent(GPTCompletionCallback callback, long id, Str
                     LOG.trace("Response {} from GPT: {}", id, content);
                     callback.onNext(content);
                 }
+                if (choice.getDelta().getToolCalls() != null) {
+                    callback.toolDelta(choice.getDelta().getToolCalls());
+                }
                 if (choice.getFinishReason() != null) {
                     System.out.println("Response {} from GPT finished with reason {}" + id + choice.getFinishReason());
                 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
index 755b78d87..357b93a6a 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
@@ -1,5 +1,7 @@
 package com.composum.ai.backend.base.service.chat.impl.chatmodel;
 
+import javax.annotation.Nullable;
+
 import com.google.gson.annotations.SerializedName;
 
 /**
@@ -35,4 +37,18 @@ public void setArguments(String arguments) {
         this.arguments = arguments;
     }
 
+    public void mergeDelta(@Nullable ChatCompletionFunctionCallDetails function) {
+        if (function == null) {
+            return;
+        }
+        if (name == null) {
+            name = function.name;
+        }
+        if (arguments == null) {
+            arguments = function.arguments;
+        } else if (function.arguments != null) {
+            arguments = arguments + function.arguments;
+        }
+    }
+
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
index 099725b2c..41312d744 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
@@ -1,5 +1,10 @@
 package com.composum.ai.backend.base.service.chat.impl.chatmodel;
 
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
 import com.google.gson.annotations.SerializedName;
 
 /**
@@ -14,6 +19,12 @@ public class ChatCompletionToolCall {
     @SerializedName("id")
     private String id;
 
+    /**
+     * The index of the tool call in the list of tool calls.
+     */
+    @SerializedName("index")
+    private int index;
+
     /**
      * The type of the tool, currently only "function" is supported.
      */
@@ -36,6 +47,14 @@ public void setId(String id) {
         this.id = id;
     }
 
+    public int getIndex() {
+        return index;
+    }
+
+    public void setIndex(int index) {
+        this.index = index;
+    }
+
     public String getType() {
         return type;
     }
@@ -51,4 +70,58 @@ public ChatCompletionFunctionCallDetails getFunction() {
     public void setFunction(ChatCompletionFunctionCallDetails function) {
         this.function = function;
     }
+
+    @Nullable
+    public static List mergeDelta(@Nullable List calls1,
+                                                          @Nullable List calls2) {
+        if (calls1 == null || calls1.isEmpty()) {
+            return calls2;
+        }
+        if (calls2 == null || calls2.isEmpty()) {
+            return calls1;
+        }
+        int maxIndex = Math.max(
+                calls1.stream().mapToInt(ChatCompletionToolCall::getIndex).max().orElse(0),
+                calls2.stream().mapToInt(ChatCompletionToolCall::getIndex).max().orElse(0)
+        );
+        List calls = new ArrayList<>(maxIndex + 1);
+        for (int i = 0; i <= maxIndex; i++) {
+            calls.add(null);
+        }
+        for (ChatCompletionToolCall call : calls1) {
+            if (call != null) {
+                calls.set(call.getIndex(), call);
+            }
+        }
+        for (ChatCompletionToolCall call : calls2) {
+            if (call != null) {
+                if (calls.get(call.getIndex()) != null) {
+                    calls.get(call.getIndex()).mergeDelta(call);
+                } else {
+                    calls.set(call.getIndex(), call);
+                }
+            }
+        }
+        return calls;
+    }
+
+    public void mergeDelta(@Nullable ChatCompletionToolCall other) {
+        if (other == null) {
+            return;
+        }
+        if (index != other.index) {
+            throw new IllegalArgumentException("Cannot merge tool calls with different indices");
+        }
+        if (id == null) {
+            id = other.id;
+        }
+        if (type == null) {
+            type = other.type;
+        }
+        if (function == null) {
+            function = other.function;
+        } else {
+            function.mergeDelta(other.function);
+        }
+    }
 }
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java
new file mode 100644
index 000000000..90f91c462
--- /dev/null
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java
@@ -0,0 +1,73 @@
+package com.composum.ai.backend.base.service.chat.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.osgi.framework.BundleContext;
+
+import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
+import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
+import com.composum.ai.backend.base.service.chat.GPTFinishReason;
+import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionToolCall;
+
+/**
+ * Tests for {@link GPTChatCompletionService} with tool calls.
+ */
+public class GPTChatCompletionServiceImplToolsTest {
+
+    protected GPTChatCompletionServiceImpl service = new GPTChatCompletionServiceImpl();
+    private GPTChatCompletionServiceImpl.GPTChatCompletionServiceConfig config =
+            mock(GPTChatCompletionServiceImpl.GPTChatCompletionServiceConfig.class);
+    private BundleContext bundleContext = mock(BundleContext.class);
+
+    @Before
+    public void setUp() {
+        Mockito.when(config.openAiApiKey()).thenReturn("sk-abcdefg");
+        service.activate(config, bundleContext);
+    }
+
+    /**
+     * Streaming test from real example.
+     */
+    @Test
+    public void testHandleStreamingEventWithToolCall() {
+        String[] lines = ("" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_bZrlWEFQ7jbTybVQYHg0hDSn\",\"type\":\"function\",\"function\":{\"name\":\"wobbler\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"to\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"wobbl\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"e\\\": \\\"h\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"i\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"id\":\"call_om4hHNguyKYCOkhiZu6wo9xA\",\"type\":\"function\",\"function\":{\"name\":\"wobbler\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"{\\\"to\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"wobbl\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"e\\\": \\\"h\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"o\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}]}\n" +
+                "{\"id\":\"chatcmpl-ANM95HYArFdhLHNSljHzHawXXAiXT\",\"object\":\"chat.completion.chunk\",\"created\":1730130035,\"model\":\"gpt-4o-mini-2024-07-18\",\"system_fingerprint\":\"fp_8bfc6a7dc2\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}]}\n" +
+                "[DONE]"
+        ).split("\n");
+        GPTCompletionCallback.GPTCompletionCollector callback = new GPTCompletionCallback.GPTCompletionCollector();
+        for (String line : lines) {
+            service.handleStreamingEvent(callback, 123, "data: " + line);
+        }
+        assertEquals(GPTFinishReason.TOOL_CALLS, callback.getFinishReason());
+        List calls = callback.getToolCalls();
+        assertEquals(2, calls.size());
+        assertEquals("call_bZrlWEFQ7jbTybVQYHg0hDSn", calls.get(0).getId());
+        assertEquals("wobbler", calls.get(0).getFunction().getName());
+        assertEquals("{\"towobble\": \"hi\"}", calls.get(0).getFunction().getArguments());
+        assertEquals("call_om4hHNguyKYCOkhiZu6wo9xA", calls.get(1).getId());
+        assertEquals("wobbler", calls.get(1).getFunction().getName());
+        assertEquals("{\"towobble\": \"ho\"}", calls.get(1).getFunction().getArguments());
+    }
+
+}

From 5a8f51b68624c80d46b7b0edf8a266c9d762aa57 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Mon, 28 Oct 2024 22:09:22 +0100
Subject: [PATCH 13/44] add tool calls to message input

---
 .../base/service/chat/GPTChatMessage.java       | 17 ++++++++++++++---
 .../base/service/chat/GPTMessageRole.java       |  8 +++++++-
 .../impl/chatmodel/ChatCompletionMessage.java   |  1 +
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java
index 6302d1158..1145346be 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java
@@ -15,17 +15,21 @@ public class GPTChatMessage {
     private final GPTMessageRole role;
     private final String content;
     private final String imageUrl;
+    private final String tool_call_id;
 
     public GPTChatMessage(@Nonnull GPTMessageRole role, @Nonnull String content) {
-        this.role = role;
-        this.content = content;
-        this.imageUrl = null;
+        this(role, content, null);
     }
 
     public GPTChatMessage(@Nonnull GPTMessageRole role, @Nullable String content, @Nullable String imageUrl) {
+        this(role, content, imageUrl, null);
+    }
+
+    public GPTChatMessage(@Nonnull GPTMessageRole role, @Nullable String content, @Nullable String imageUrl, String tool_call_id) {
         this.role = role;
         this.content = content;
         this.imageUrl = imageUrl;
+        this.tool_call_id = tool_call_id;
     }
 
     /**
@@ -49,6 +53,13 @@ public String getImageUrl() {
         return imageUrl;
     }
 
+    /**
+     * The ID of a prior tool call.
+     */
+    public String getToolCallId() {
+        return tool_call_id;
+    }
+
     /**
      * String representation only for debugging.
      */
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTMessageRole.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTMessageRole.java
index 22c2b0889..6fa802004 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTMessageRole.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTMessageRole.java
@@ -23,7 +23,13 @@ public enum GPTMessageRole {
      * The assistant messages help store prior responses. It can also serve as an example of desired behavior.
      */
     @SerializedName("assistant")
-    ASSISTANT("assistant");
+    ASSISTANT("assistant"),
+
+    /**
+     * A result of a tool call the assistant made.
+     */
+    @SerializedName("tool")
+    TOOL("tool");
 
     private final String externalRepresentation;
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
index 44c7699d8..a4e7ad497 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
@@ -44,6 +44,7 @@ public static ChatCompletionMessage make(GPTChatMessage message) {
         ChatCompletionMessage result = new ChatCompletionMessage();
         result.setRole(role);
         result.setContent(Collections.singletonList(part));
+        result.setToolCallId(message.getToolCallId());
         return result;
     }
 

From 54d1294ef58ae372e8ec5112f6394500d93bdb4e Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 29 Oct 2024 09:39:01 +0100
Subject: [PATCH 14/44] add a (yet failing) archunit test that checks for API
 depencies on impl

---
 backend/base/pom.xml                          |  9 +++++-
 .../chat/impl/GPTChatMessagesTemplate.java    |  1 +
 .../ai/backend/base/BaseArchUnitTest.java     | 30 +++++++++++++++++++
 pom.xml                                       | 20 ++++++++++++-
 4 files changed, 58 insertions(+), 2 deletions(-)
 create mode 100644 backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java

diff --git a/backend/base/pom.xml b/backend/base/pom.xml
index e6f4d4453..e5fbecc5e 100644
--- a/backend/base/pom.xml
+++ b/backend/base/pom.xml
@@ -125,7 +125,14 @@
             org.mockito
             mockito-core
         
-
+        
+            com.tngtech.archunit
+            archunit
+        
+        
+            com.tngtech.archunit
+            archunit-junit4
+        
     
 
     
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplate.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplate.java
index 34e10e27d..795d6a40d 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplate.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplate.java
@@ -34,6 +34,7 @@
  * The messages can contain placeholders like ${name}, which are replaced by the values in the map given to getMessages.
  * If a placeholder is missing, we throw an error, as this is a misusage of the template.
  */
+// FIXME(hps,24/10/29) move this out of the impl package to make dependencies more sane
 public class GPTChatMessagesTemplate {
     static final Logger LOG = LoggerFactory.getLogger(GPTChatMessagesTemplate.class);
 
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java b/backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java
new file mode 100644
index 000000000..5d81ffdad
--- /dev/null
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java
@@ -0,0 +1,30 @@
+package com.composum.ai.backend.base;
+
+import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes;
+
+import org.junit.runner.RunWith;
+
+import com.tngtech.archunit.junit.AnalyzeClasses;
+import com.tngtech.archunit.junit.ArchTest;
+import com.tngtech.archunit.junit.ArchUnitRunner;
+import com.tngtech.archunit.lang.ArchRule;
+
+
+/**
+ * We try to cover some architecture failures that were there - package dependendies that should not be there.
+ *
+ * @see "https://www.archunit.org/userguide/html/000_Index.html"
+ */
+@RunWith(ArchUnitRunner.class)
+@AnalyzeClasses(packages = "com.composum.ai.backend.base")
+public class BaseArchUnitTest {
+
+    /**
+     * Classes from API packages should not import anything from the ..impl.. packages.
+     */
+    @ArchTest
+    public static final ArchRule noimportImpl = classes()
+            .that().resideInAPackage("com.composum.ai.backend.base.service.chat")
+            .should().onlyDependOnClassesThat().resideOutsideOfPackages("..impl..");
+
+}
diff --git a/pom.xml b/pom.xml
index 099a3f8bc..5a3dace8c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,5 +1,6 @@
 
-
+
     4.0.0
 
     
@@ -135,4 +136,21 @@
         
     
 
+    
+        
+            
+                com.tngtech.archunit
+                archunit
+                1.3.0
+                test
+            
+            
+                com.tngtech.archunit
+                archunit-junit4
+                1.3.0
+                test
+            
+        
+    
+
 

From d28eeba00e96561b5493db4fdc0973c55419d52e Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 29 Oct 2024 14:41:13 +0100
Subject: [PATCH 15/44] refactoring: GPTCompletionCallback must not use impl
 objects

---
 .../base/service/chat/GPTChatMessage.java     |  21 ++-
 .../service/chat/GPTCompletionCallback.java   |  12 +-
 .../service/chat/GPTFunctionCallDetails.java  |  78 +++++++++++
 .../base/service/chat/GPTFunctionDetails.java |  58 +++++++++
 .../base/service/chat/GPTToolCall.java        | 123 ++++++++++++++++++
 .../impl/GPTChatCompletionServiceImpl.java    |   4 +-
 .../ChatCompletionFunctionCallDetails.java    |   7 +
 .../chatmodel/ChatCompletionToolCall.java     |  29 +++++
 ...GPTChatCompletionServiceImplToolsTest.java |   3 +-
 9 files changed, 326 insertions(+), 9 deletions(-)
 create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionCallDetails.java
 create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionDetails.java
 create mode 100644 backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTToolCall.java

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java
index 1145346be..cc852ec63 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessage.java
@@ -1,5 +1,6 @@
 package com.composum.ai.backend.base.service.chat;
 
+import java.util.List;
 import java.util.Objects;
 
 import javax.annotation.Nonnull;
@@ -16,6 +17,7 @@ public class GPTChatMessage {
     private final String content;
     private final String imageUrl;
     private final String tool_call_id;
+    private final List tool_calls;
 
     public GPTChatMessage(@Nonnull GPTMessageRole role, @Nonnull String content) {
         this(role, content, null);
@@ -26,10 +28,15 @@ public GPTChatMessage(@Nonnull GPTMessageRole role, @Nullable String content, @N
     }
 
     public GPTChatMessage(@Nonnull GPTMessageRole role, @Nullable String content, @Nullable String imageUrl, String tool_call_id) {
+        this(role, content, imageUrl, tool_call_id, null);
+    }
+
+    public GPTChatMessage(@Nonnull GPTMessageRole role, @Nullable String content, @Nullable String imageUrl, String tool_call_id, List tool_calls) {
         this.role = role;
         this.content = content;
         this.imageUrl = imageUrl;
         this.tool_call_id = tool_call_id;
+        this.tool_calls = tool_calls;
     }
 
     /**
@@ -60,6 +67,14 @@ public String getToolCallId() {
         return tool_call_id;
     }
 
+    /**
+     * The tool calls that were made in the context of this message.
+     */
+    @Nullable
+    public List getToolCalls() {
+        return tool_calls;
+    }
+
     /**
      * String representation only for debugging.
      */
@@ -69,6 +84,8 @@ public String toString() {
                 "role=" + role +
                 (content != null ? ", text='" + content + '\'' : "") +
                 (imageUrl != null ? ", imageUrl='" + imageUrl + '\'' : "") +
+                (tool_call_id != null ? ", tool_call_id='" + tool_call_id + '\'' : "") +
+                (tool_calls != null ? ", tool_calls=" + tool_calls : "") +
                 '}';
     }
 
@@ -78,7 +95,9 @@ public boolean equals(Object o) {
         if (!(o instanceof GPTChatMessage)) return false;
         GPTChatMessage that = (GPTChatMessage) o;
         return getRole() == that.getRole() && Objects.equals(getContent(), that.getContent()) &&
-                Objects.equals(getImageUrl(), that.getImageUrl());
+                Objects.equals(getImageUrl(), that.getImageUrl()) &&
+                Objects.equals(getToolCallId(),  that.getToolCallId()) &&
+                Objects.equals(getToolCalls(), that.getToolCalls());
     }
 
     @Override
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
index cf04d91a9..c138d1b6b 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
@@ -51,7 +51,7 @@ default void close() {
     /**
      * Called when a tool call is made.
      */
-    default void toolDelta(List toolCalls) {
+    default void toolDelta(List toolCalls) {
         // empty
     }
 
@@ -92,7 +92,7 @@ public void close() {
             delegate.close();
         }
 
-        public void toolDelta(List toolCalls) {
+        public void toolDelta(List toolCalls) {
             delegate.toolDelta(toolCalls);
         }
 
@@ -108,7 +108,7 @@ public static class GPTCompletionCollector implements GPTCompletionCallback {
         private StringBuilder buffer = new StringBuilder();
         private Throwable throwable;
         private GPTFinishReason finishReason;
-        private List toolCalls;
+        private List toolCalls;
 
         @Override
         public void onFinish(GPTFinishReason finishReason) {
@@ -144,11 +144,11 @@ public Throwable getError() {
         }
 
         @Override
-        public void toolDelta(List toolCalls) {
-            this.toolCalls = ChatCompletionToolCall.mergeDelta(this.toolCalls, toolCalls);
+        public void toolDelta(List toolCalls) {
+            this.toolCalls = GPTToolCall.mergeDelta(this.toolCalls, toolCalls);
         }
 
-        public List getToolCalls() {
+        public List getToolCalls() {
             return toolCalls;
         }
 
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionCallDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionCallDetails.java
new file mode 100644
index 000000000..057493db3
--- /dev/null
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionCallDetails.java
@@ -0,0 +1,78 @@
+package com.composum.ai.backend.base.service.chat;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Represents the a call of a function used as a tool in the chat completion request.
+ */
+public class GPTFunctionCallDetails {
+
+    private final String name;
+    private final String arguments;
+
+    /**
+     * Creates the object
+     *
+     * @param name      The name of the function to be called. This must be unique and can only contain a-z, A-Z, 0-9, underscores, and dashes.
+     * @param arguments A JSON for the arguments the function is called with.
+     */
+    public GPTFunctionCallDetails(String name, String arguments) {
+        this.name = name;
+        this.arguments = arguments;
+    }
+
+    /**
+     * The name of the function to be called. This must be unique and can only contain a-z, A-Z, 0-9, underscores, and dashes.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * A JSON for the arguments the function is called with.
+     */
+    public String getArguments() {
+        return arguments;
+    }
+
+    /**
+     * String representation for debugging.
+     */
+    @Override
+    public String toString() {
+        return "{" +
+                "name='" + name + '\'' +
+                ", arguments='" + arguments + '\'' +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof GPTFunctionCallDetails)) return false;
+        GPTFunctionCallDetails that = (GPTFunctionCallDetails) o;
+        return Objects.equals(getName(), that.getName()) && Objects.equals(getArguments(), that.getArguments());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getName(), getArguments());
+    }
+
+    public GPTFunctionCallDetails mergeDelta(@Nullable GPTFunctionCallDetails function) {
+        if (function == null) {
+            return this;
+        }
+        String newName = name == null ? function.name : name;
+        String newArguments = arguments;
+        if (newArguments == null) {
+            newArguments = function.arguments;
+        } else if (function.arguments != null) {
+            newArguments = newArguments + function.arguments;
+        }
+        return new GPTFunctionCallDetails(newName, newArguments);
+    }
+
+}
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionDetails.java
new file mode 100644
index 000000000..2e0f1d46c
--- /dev/null
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTFunctionDetails.java
@@ -0,0 +1,58 @@
+package com.composum.ai.backend.base.service.chat;
+
+/**
+ * Represents the details of a function used as a tool in the chat completion request.
+ * Includes the function's name, description, parameters, and an optional strict flag.
+ */
+public class GPTFunctionDetails {
+
+    private final String name;
+    private final String description;
+    private final Object parameters;  // Arbitrary JSON schema
+    private final Boolean strict;
+
+    /**
+     * Creates the object
+     *
+     * @param name        The name of the function to be called. This must be unique and can only contain a-z, A-Z, 0-9, underscores, and dashes.
+     * @param description A brief description of what the function does. Helps the model choose when to call it.
+     *                    The parameters accepted by the function, defined as an arbitrary JSON schema object.
+     * @param parameters  The parameters accepted by the function, defined as an arbitrary JSON schema object.
+     * @param strict      Whether to enforce strict schema adherence for the parameters
+     */
+    public GPTFunctionDetails(String name, String description, Object parameters, Boolean strict) {
+        this.name = name;
+        this.description = description;
+        this.parameters = parameters;
+        this.strict = strict;
+    }
+
+    /**
+     * The name of the function to be called. This must be unique and can only contain a-z, A-Z, 0-9, underscores, and dashes.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * A brief description of what the function does. Helps the model choose when to call it.
+     */
+    public String getDescription() {
+        return description;
+    }
+
+    /**
+     * The parameters accepted by the function, defined as an arbitrary JSON schema object.
+     */
+    public Object getParameters() {
+        return parameters;
+    }
+
+    /**
+     * Whether to enforce strict schema adherence for the parameters.
+     */
+    public Boolean getStrict() {
+        return strict;
+    }
+
+}
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTToolCall.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTToolCall.java
new file mode 100644
index 000000000..d31f1989e
--- /dev/null
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTToolCall.java
@@ -0,0 +1,123 @@
+package com.composum.ai.backend.base.service.chat;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+/**
+ * Represents a tool call generated by the model in a chat completion response.
+ * This can be a function call with specific arguments.
+ */
+public class GPTToolCall {
+
+    private final String id;
+    private final String type;
+    private final GPTFunctionCallDetails function;
+
+    /**
+     * Creates the object
+     *
+     * @param id       The ID of the tool call.
+     * @param type     The type of the tool, currently only "function" is supported.
+     * @param function The function being called by the model, including its name and arguments.
+     */
+    public GPTToolCall(String id, String type, GPTFunctionCallDetails function) {
+        this.id = id;
+        this.type = type;
+        this.function = function;
+    }
+
+    /**
+     * The ID of the tool call.
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * The type of the tool, currently only "function" is supported.
+     */
+    public String getType() {
+        return type;
+    }
+
+    /**
+     * The function being called by the model, including its name and arguments.
+     */
+    public GPTFunctionCallDetails getFunction() {
+        return function;
+    }
+
+    /**
+     * String representation for debugging.
+     */
+    @Override
+    public String toString() {
+        return "GPTFunctionCallDetails{" +
+                "id='" + id + '\'' +
+                ", type='" + type + '\'' +
+                ", function=" + function +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof GPTToolCall)) return false;
+        GPTToolCall that = (GPTToolCall) o;
+        return Objects.equals(getId(), that.getId()) &&
+                Objects.equals(getType(), that.getType()) &&
+                Objects.equals(getFunction(), that.getFunction());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getId(), getType(), getFunction());
+    }
+
+    /**
+     * When streaming, this merges the delta received during the stream.
+     */
+    public GPTToolCall mergeDelta(@Nullable GPTToolCall other) {
+        if (other == null) {
+            return this;
+        }
+        if (id != null && other.id != null && !id.equals(other.id)) {
+            throw new IllegalArgumentException("Bug: trying to merge tool calls with different IDs: " + id + " and " + other.id);
+        }
+        String newId = id == null ? other.id : id;
+        String newType = type == null ? other.type : type;
+        GPTFunctionCallDetails newFunction = function == null ? other.function : function.mergeDelta(other.function);
+        return new GPTToolCall(newId, newType, newFunction);
+    }
+
+    /**
+     * When streaming, this merges the delta received during the stream.
+     */
+    @Nullable
+    public static List mergeDelta(@Nullable List calls1, @Nullable List calls2) {
+        if (calls1 == null || calls1.isEmpty()) {
+            return calls2;
+        }
+        if (calls2 == null || calls2.isEmpty()) {
+            return calls1;
+        }
+        int maxIndex = Math.max(calls1.size(), calls2.size());
+        List calls = new ArrayList<>(maxIndex + 1);
+        for (int i = 0; i < maxIndex; i++) {
+            GPTToolCall call1 = i < calls1.size() ? calls1.get(i) : null;
+            GPTToolCall call2 = i < calls2.size() ? calls2.get(i) : null;
+            if (call2 == null) {
+                calls.add(call1);
+            } else if (call1 == null) {
+                calls.add(call2);
+            } else {
+                calls.add(call1.mergeDelta(call2));
+            }
+        }
+        return calls;
+    }
+
+}
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index df872f8da..f51c4b741 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -76,12 +76,14 @@
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.composum.ai.backend.base.service.chat.GPTTool;
+import com.composum.ai.backend.base.service.chat.GPTToolCall;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionChoice;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionFunctionDetails;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessage;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessagePart;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionRequest;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionResponse;
+import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionToolCall;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatTool;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.OpenAIEmbeddings;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -459,7 +461,7 @@ protected void handleStreamingEvent(GPTCompletionCallback callback, long id, Str
                     callback.onNext(content);
                 }
                 if (choice.getDelta().getToolCalls() != null) {
-                    callback.toolDelta(choice.getDelta().getToolCalls());
+                    callback.toolDelta(ChatCompletionToolCall.toGptToolCallList(choice.getDelta().getToolCalls()));
                 }
                 if (choice.getFinishReason() != null) {
                     System.out.println("Response {} from GPT finished with reason {}" + id + choice.getFinishReason());
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
index 357b93a6a..a2f537397 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
@@ -1,7 +1,9 @@
 package com.composum.ai.backend.base.service.chat.impl.chatmodel;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import com.composum.ai.backend.base.service.chat.GPTFunctionCallDetails;
 import com.google.gson.annotations.SerializedName;
 
 /**
@@ -51,4 +53,9 @@ public void mergeDelta(@Nullable ChatCompletionFunctionCallDetails function) {
         }
     }
 
+    @Nonnull
+    public GPTFunctionCallDetails toGptFuctionCallDetails() {
+        return new GPTFunctionCallDetails(name, arguments);
+    }
+
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
index 41312d744..007d6cf1f 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
@@ -3,8 +3,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import com.composum.ai.backend.base.service.chat.GPTToolCall;
 import com.google.gson.annotations.SerializedName;
 
 /**
@@ -124,4 +126,31 @@ public void mergeDelta(@Nullable ChatCompletionToolCall other) {
             function.mergeDelta(other.function);
         }
     }
+
+    @Nonnull
+    public GPTToolCall toGptToolCall() {
+        return new GPTToolCall(id, type, function != null ? function.toGptFuctionCallDetails() : null);
+    }
+
+    /**
+     * Turns the list into a {@link GPTToolCall} list observing the {@link #getIndex()}.
+     */
+    @Nullable
+    public static List toGptToolCallList(List list) {
+        if (list == null) {
+            return null;
+        }
+        List res = new ArrayList<>();
+        int maxIndex = list.stream().mapToInt(ChatCompletionToolCall::getIndex).max().orElse(0);
+        for (int i = 0; i <= maxIndex; ++i) {
+            res.add(null);
+        }
+        for (ChatCompletionToolCall chatCompletionToolCall : list) {
+            if (chatCompletionToolCall != null) {
+                res.set(chatCompletionToolCall.index, chatCompletionToolCall.toGptToolCall());
+            }
+        }
+        return res;
+    }
+
 }
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java
index 90f91c462..37db36ae3 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImplToolsTest.java
@@ -17,6 +17,7 @@
 import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
+import com.composum.ai.backend.base.service.chat.GPTToolCall;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionToolCall;
 
 /**
@@ -60,7 +61,7 @@ public void testHandleStreamingEventWithToolCall() {
             service.handleStreamingEvent(callback, 123, "data: " + line);
         }
         assertEquals(GPTFinishReason.TOOL_CALLS, callback.getFinishReason());
-        List calls = callback.getToolCalls();
+        List calls = callback.getToolCalls();
         assertEquals(2, calls.size());
         assertEquals("call_bZrlWEFQ7jbTybVQYHg0hDSn", calls.get(0).getId());
         assertEquals("wobbler", calls.get(0).getFunction().getName());

From ca461740b2071020a92efd91e8c1638574572ee1 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 29 Oct 2024 14:45:37 +0100
Subject: [PATCH 16/44] Refactoring: move GPTChatCompletionTemplate out of impl
 package since that's used in the API

---
 .../backend/base/service/chat/GPTChatCompletionService.java   | 1 -
 .../base/service/chat/{impl => }/GPTChatMessagesTemplate.java | 4 +---
 .../base/service/chat/impl/GPTChatCompletionServiceImpl.java  | 2 +-
 .../base/service/chat/impl/GPTContentCreationServiceImpl.java | 1 +
 .../base/service/chat/impl/GPTTranslationServiceImpl.java     | 1 +
 .../java/com/composum/ai/backend/base/BaseArchUnitTest.java   | 2 +-
 .../service/chat/{impl => }/GPTChatMessagesTemplateTest.java  | 4 +---
 .../base/service/chat/impl/GPTTranslationServiceImplTest.java | 1 +
 8 files changed, 7 insertions(+), 9 deletions(-)
 rename backend/base/src/main/java/com/composum/ai/backend/base/service/chat/{impl => }/GPTChatMessagesTemplate.java (97%)
 rename backend/base/src/test/java/com/composum/ai/backend/base/service/chat/{impl => }/GPTChatMessagesTemplateTest.java (95%)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java
index 91bfb3525..a7a2b714d 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java
@@ -6,7 +6,6 @@
 import javax.annotation.Nullable;
 
 import com.composum.ai.backend.base.service.GPTException;
-import com.composum.ai.backend.base.service.chat.impl.GPTChatMessagesTemplate;
 
 /**
  * Raw abstraction of the ChatGPT chat interface, with only the details that are needed.
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplate.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplate.java
similarity index 97%
rename from backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplate.java
rename to backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplate.java
index 795d6a40d..9c71b59e9 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplate.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplate.java
@@ -1,4 +1,4 @@
-package com.composum.ai.backend.base.service.chat.impl;
+package com.composum.ai.backend.base.service.chat;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -21,8 +21,6 @@
 import org.slf4j.LoggerFactory;
 
 import com.composum.ai.backend.base.service.GPTException;
-import com.composum.ai.backend.base.service.chat.GPTChatMessage;
-import com.composum.ai.backend.base.service.chat.GPTMessageRole;
 
 /**
  * A template for chat messages, with placeholders.
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index f51c4b741..9c9f84538 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -71,12 +71,12 @@
 import com.composum.ai.backend.base.service.GPTException;
 import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
 import com.composum.ai.backend.base.service.chat.GPTChatMessage;
+import com.composum.ai.backend.base.service.chat.GPTChatMessagesTemplate;
 import com.composum.ai.backend.base.service.chat.GPTChatRequest;
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.composum.ai.backend.base.service.chat.GPTTool;
-import com.composum.ai.backend.base.service.chat.GPTToolCall;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionChoice;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionFunctionDetails;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessage;
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java
index 5c2e51aa5..dd31cb40d 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java
@@ -18,6 +18,7 @@
 import com.composum.ai.backend.base.service.GPTException;
 import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
 import com.composum.ai.backend.base.service.chat.GPTChatMessage;
+import com.composum.ai.backend.base.service.chat.GPTChatMessagesTemplate;
 import com.composum.ai.backend.base.service.chat.GPTChatRequest;
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImpl.java
index 5370fc6c2..512e4d15b 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImpl.java
@@ -34,6 +34,7 @@
 import com.composum.ai.backend.base.service.GPTException;
 import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
 import com.composum.ai.backend.base.service.chat.GPTChatMessage;
+import com.composum.ai.backend.base.service.chat.GPTChatMessagesTemplate;
 import com.composum.ai.backend.base.service.chat.GPTChatRequest;
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java b/backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java
index 5d81ffdad..066eeffff 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/BaseArchUnitTest.java
@@ -25,6 +25,6 @@ public class BaseArchUnitTest {
     @ArchTest
     public static final ArchRule noimportImpl = classes()
             .that().resideInAPackage("com.composum.ai.backend.base.service.chat")
-            .should().onlyDependOnClassesThat().resideOutsideOfPackages("..impl..");
+            .should().onlyDependOnClassesThat().resideOutsideOfPackages("com.composum.ai..impl..");
 
 }
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplateTest.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplateTest.java
similarity index 95%
rename from backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplateTest.java
rename to backend/base/src/test/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplateTest.java
index 06ac64015..18ecd5ae6 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTChatMessagesTemplateTest.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplateTest.java
@@ -1,4 +1,4 @@
-package com.composum.ai.backend.base.service.chat.impl;
+package com.composum.ai.backend.base.service.chat;
 
 import static org.junit.Assert.assertEquals;
 
@@ -12,8 +12,6 @@
 import org.slf4j.impl.SimpleLogger;
 
 import com.composum.ai.backend.base.service.GPTException;
-import com.composum.ai.backend.base.service.chat.GPTChatMessage;
-import com.composum.ai.backend.base.service.chat.GPTMessageRole;
 
 public class GPTChatMessagesTemplateTest {
 
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImplTest.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImplTest.java
index f2153a697..3580792a8 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImplTest.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/GPTTranslationServiceImplTest.java
@@ -33,6 +33,7 @@
 
 import com.composum.ai.backend.base.service.GPTException;
 import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
+import com.composum.ai.backend.base.service.chat.GPTChatMessagesTemplate;
 import com.composum.ai.backend.base.service.chat.GPTChatRequest;
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTResponseCheck;

From 29a3a34375470ac5b12fc9b8c8078ef9816df3d0 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 29 Oct 2024 14:57:09 +0100
Subject: [PATCH 17/44] remove / execute some FIXME comments

---
 .../autotranslate/AutoPageTranslateServiceImpl.java    |  7 -------
 .../base/service/chat/GPTChatMessagesTemplate.java     |  1 -
 .../composum/bundle/model/TranslationDialogModel.java  |  2 +-
 composum/config/pom.xml                                | 10 ----------
 featurespecs/AEMIntegrationIdeas.md                    |  1 -
 5 files changed, 1 insertion(+), 20 deletions(-)

diff --git a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoPageTranslateServiceImpl.java b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoPageTranslateServiceImpl.java
index 714d237c0..5c15d4e48 100644
--- a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoPageTranslateServiceImpl.java
+++ b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoPageTranslateServiceImpl.java
@@ -364,9 +364,6 @@ protected String remapPaths(String translatedValue, String blueprintPath, String
         Pattern pattern = Pattern.compile("href=\"" +
                 Pattern.quote(blueprintPath) + "(/[^\"]*)\"");
         String result = pattern.matcher(translatedValue).replaceAll("href=\"" + livecopyPath + "$1\"");
-        if (translatedValue.contains("href")) { // FIXME(hps,24/10/03) no checkin
-            LOG.trace("Remapping paths from {} to {} in {}", blueprintPath, livecopyPath, translatedValue);
-        }
         return result;
     }
 
@@ -581,10 +578,6 @@ protected boolean collectPropertiesToTranslate(
             stats.translateableProperties++;
             AITranslatePropertyWrapper targetWrapper = new AITranslatePropertyWrapper(sourceValueMap, targetValueMap, key);
 
-            if (StringUtils.contains(targetWrapper.getOriginalCopy(), "href")) { // FIXME(hps,24/10/03) no checkin
-                LOG.trace("Skipping {} in {} because it contains href", key, resource.getPath());
-            }
-
             // we will translate except if the property is cancelled and we don't want to touch cancelled properties,
             // or if we have a current translation.
             boolean isCancelled = isCancelled(resource, key, relationship);
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplate.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplate.java
index 9c71b59e9..a19a41b43 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplate.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatMessagesTemplate.java
@@ -32,7 +32,6 @@
  * The messages can contain placeholders like ${name}, which are replaced by the values in the map given to getMessages.
  * If a placeholder is missing, we throw an error, as this is a misusage of the template.
  */
-// FIXME(hps,24/10/29) move this out of the impl package to make dependencies more sane
 public class GPTChatMessagesTemplate {
     static final Logger LOG = LoggerFactory.getLogger(GPTChatMessagesTemplate.class);
 
diff --git a/composum/bundle/src/main/java/com/composum/ai/composum/bundle/model/TranslationDialogModel.java b/composum/bundle/src/main/java/com/composum/ai/composum/bundle/model/TranslationDialogModel.java
index 196576e24..b903a1a78 100644
--- a/composum/bundle/src/main/java/com/composum/ai/composum/bundle/model/TranslationDialogModel.java
+++ b/composum/bundle/src/main/java/com/composum/ai/composum/bundle/model/TranslationDialogModel.java
@@ -119,7 +119,7 @@ protected PropertyEditHandle makePropertyEditHandle(BeanContext context,
         Objects.requireNonNull(propertyI18nPath);
         PropertyEditHandle handle = new PropertyEditHandle(String.class);
         handle.setProperty(propertyName, propertyI18nPath, true);
-        // FIXME(hps,04.05.23) how to determine that?
+        // TODO(hps,04.05.23) how to determine that?
         handle.setMultiValue(false);
         handle.initialize(context, this.resource);
         // propertyEditHandle: resource e.g. /content/ist/software/home/test/jcr:content/main/text , propertyName = propertyPath = title
diff --git a/composum/config/pom.xml b/composum/config/pom.xml
index 383f27ead..0a083c854 100644
--- a/composum/config/pom.xml
+++ b/composum/config/pom.xml
@@ -74,16 +74,6 @@
                                 
                             
                         
-
-                        
-                            
-                            /libs/composum/ai/install
-                            
-                                
-                                    .*/com.composum.ai.backend.base.service.chat.impl.GPTChatCompletionServiceImpl.*
-                                
-                            
-                        
                     
 
                 
diff --git a/featurespecs/AEMIntegrationIdeas.md b/featurespecs/AEMIntegrationIdeas.md
index ab5bddcf4..c2796d27e 100644
--- a/featurespecs/AEMIntegrationIdeas.md
+++ b/featurespecs/AEMIntegrationIdeas.md
@@ -166,7 +166,6 @@ http://localhost:4502/bin/browser.html/libs/core/wcm/components/text/v2/text
 Test on wcm editors, too!
 
 http://experience-aem.blogspot.com/2016/07/aem-62-touch-ui-rich-text-editor-inplace-editing-open-in-fullscreen-editing-start-event.html
-editing-start is triggered but I have trouble to catch it. TODO FIXME CONTINUE THIS
 
 ### icon
 

From e02d224c6ca15bfa1862c47ce94c6edc62eaaa1e Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 29 Oct 2024 16:39:41 +0100
Subject: [PATCH 18/44] comment for auto translation rules

---
 .../core/impl/autotranslate/AutoTranslateCaConfig.java |  8 ++++++++
 .../impl/autotranslate/AutoTranslateRuleConfig.java    | 10 +++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateCaConfig.java b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateCaConfig.java
index 02f7096f1..d7a1eb5c1 100644
--- a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateCaConfig.java
+++ b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateCaConfig.java
@@ -57,4 +57,12 @@
             })
     String includeExistingTranslationsInRetranslation();
 
+    @Property(label = "Optional Comment", order = 7,
+            description = "An optional comment about the configuration, for documentation purposes (not used by the translation).",
+            property = {
+                    "widgetType=textarea",
+                    "textareaRows=2"
+            })
+    String comment();
+
 }
diff --git a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateRuleConfig.java b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateRuleConfig.java
index 32c965ed6..2620aa192 100644
--- a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateRuleConfig.java
+++ b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateRuleConfig.java
@@ -10,7 +10,7 @@
 public @interface AutoTranslateRuleConfig {
 
     @Property(label = "Path Regex", order = 1,
-            description = "A regular expression matching the absolute path to the page. " +
+            description = "A regular expression matching the absolute path to the page, incl. jcr:content. " +
                     "E.g. .*/home/products/.* will match all pages under .../home/products/. If empty every page will match" +
                     "if the content pattern condition is met.")
     String pathRegex();
@@ -30,4 +30,12 @@
             })
     String additionalInstructions();
 
+    @Property(label = "Optional Comment", order = 4,
+            description = "An optional comment for the rule, for documentation purposes (not used by the translation).",
+            property = {
+                    "widgetType=textarea",
+                    "textareaRows=2"
+            })
+    String comment();
+
 }

From 7d66aa8d6f60b471ddf179dcece0148b5ccdddca Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Tue, 29 Oct 2024 17:48:08 +0100
Subject: [PATCH 19/44] start implementing streamingChatCompletionWithToolCalls

---
 .../chat/GPTChatCompletionService.java        |  7 ++++
 .../service/chat/GPTCompletionCallback.java   |  2 +-
 .../impl/GPTChatCompletionServiceImpl.java    | 21 ++++++++++++
 ...GPTChatCompletionServiceImplWithTools.java | 32 +++++++++++++++----
 4 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java
index a7a2b714d..f54349737 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatCompletionService.java
@@ -29,6 +29,13 @@ public interface GPTChatCompletionService {
      */
     void streamingChatCompletion(@Nonnull GPTChatRequest request, @Nonnull GPTCompletionCallback callback) throws GPTException;
 
+    /**
+     * Give some messages and receive the streaming response via callback, to reduce waiting time.
+     * This implementation also performs tool calls if tools are given in {@link GPTChatRequest#getConfiguration()}.
+     * It possibly waits if a rate limit is reached, but otherwise returns immediately after scheduling an asynchronous call.
+     */
+    void streamingChatCompletionWithToolCalls(@Nonnull GPTChatRequest request, @Nonnull GPTCompletionCallback callback) throws GPTException;
+
     /**
      * Retrieves a (usually cached) chat template with that name. Mostly for backend internal use.
      * The templates are retrieved from the bundle resources at "chattemplates/", and are cached.
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
index c138d1b6b..05e180a3c 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
@@ -59,7 +59,7 @@ default void toolDelta(List toolCalls) {
     /**
      * Forwards all methods to a delegate.
      */
-    public static class GPTCompletionCallbackWrapper {
+    public static class GPTCompletionCallbackWrapper implements GPTCompletionCallback {
 
         @Nonnull
         protected GPTCompletionCallback delegate;
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index 9c9f84538..04ea22d14 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -77,6 +77,7 @@
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.composum.ai.backend.base.service.chat.GPTTool;
+import com.composum.ai.backend.base.service.chat.GPTToolCall;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionChoice;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionFunctionDetails;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionMessage;
@@ -429,6 +430,26 @@ public void streamingChatCompletion(@Nonnull GPTChatRequest request, @Nonnull GP
         }
     }
 
+    @Override
+    public void streamingChatCompletionWithToolCalls(@Nonnull GPTChatRequest request, @Nonnull GPTCompletionCallback callback)
+            throws GPTException {
+        List JatoolCalls = null;
+        boolean haveToolCalls;
+        GPTCompletionCallback callbackWrapper = new GPTCompletionCallback.GPTCompletionCallbackWrapper(callback) {
+            @Override
+            public void toolDelta(List toolCalls) {
+                toolCalls = GPTToolCall.mergeDelta(toolCalls, toolCalls);
+            }
+
+            @Override
+            public void onFinish(GPTFinishReason finishReason) {
+                if (GPTFinishReason.TOOL_CALLS == finishReason) {
+                    haveToolCalls = true;
+                }
+            }
+        };
+    }
+
     /**
      * Handle a single line of the streaming response.
      * 
    
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
index 469abe9a2..d43b06f64 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
@@ -25,15 +25,19 @@ public class RunGPTChatCompletionServiceImplWithTools extends AbstractGPTRunner
     public static void main(String[] args) throws Exception {
         RunGPTChatCompletionServiceImplWithTools instance = new RunGPTChatCompletionServiceImplWithTools();
         instance.setup();
-        instance.run();
+        instance.runWithoutAutomaticCall();
         instance.teardown();
-        System.out.println("Done.");
+        System.out.println("######################### Done without automatic call #########################");
+
+        instance = new RunGPTChatCompletionServiceImplWithTools();
+        instance.setup();
+        instance.runWithAutomaticCall();
+        instance.teardown();
+        System.out.println("######################### Done. ######################### ");
     }
 
-    private void run() throws InterruptedException {
-        GPTChatRequest request = new GPTChatRequest();
-        request.addMessage(GPTMessageRole.USER, "Wobble the string 'hi' and the string 'ho'.");
-        request.setConfiguration(GPTConfiguration.ofTools(Arrays.asList(wobbler)));
+    private void runWithoutAutomaticCall() throws InterruptedException {
+        GPTChatRequest request = makeRequest();
         chatCompletionService.streamingChatCompletion(request, this);
         System.out.println("Call returned.");
         while (!isFinished) Thread.sleep(1000);
@@ -41,6 +45,22 @@ private void run() throws InterruptedException {
         System.out.println(buffer);
     }
 
+    private void runWithAutomaticCall() throws InterruptedException {
+        GPTChatRequest request = makeRequest();
+        chatCompletionService.streamingChatCompletionWithToolCalls(request, this);
+        System.out.println("Call returned.");
+        while (!isFinished) Thread.sleep(1000);
+        System.out.println("Complete response:");
+        System.out.println(buffer);
+    }
+
+    private GPTChatRequest makeRequest() {
+        GPTChatRequest request = new GPTChatRequest();
+        request.addMessage(GPTMessageRole.USER, "Wobble the string 'hi' and the string 'ho'.");
+        request.setConfiguration(GPTConfiguration.ofTools(Arrays.asList(wobbler)));
+        return request;
+    }
+
     @Override
     public void onFinish(GPTFinishReason finishReason) {
         isFinished = true;

From 677a45116d84b3fb40a4e8d9fe348a50bb082a32 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Wed, 30 Oct 2024 16:28:22 +0100
Subject: [PATCH 20/44] more tool calling implementation

---
 .../base/service/chat/GPTChatRequest.java     | 22 ++++++++++
 .../impl/GPTChatCompletionServiceImpl.java    | 44 ++++++++++++++++---
 .../ChatCompletionFunctionCallDetails.java    | 10 +++++
 .../impl/chatmodel/ChatCompletionMessage.java | 17 ++++++-
 .../impl/chatmodel/ChatCompletionRequest.java |  6 ++-
 .../chatmodel/ChatCompletionToolCall.java     | 17 +++++++
 ...GPTChatCompletionServiceImplWithTools.java | 25 ++++++++---
 7 files changed, 127 insertions(+), 14 deletions(-)

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatRequest.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatRequest.java
index f240d6eb9..39295d073 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatRequest.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTChatRequest.java
@@ -36,6 +36,17 @@ public GPTChatRequest(GPTConfiguration configuration) {
         this.configuration = configuration;
     }
 
+    /**
+     * Returns a copy; the messages list is shallowly copied, but configuration is the same object.
+     */
+    public GPTChatRequest copy() {
+        GPTChatRequest result = new GPTChatRequest();
+        result.messages.addAll(messages);
+        result.maxTokens = maxTokens;
+        result.configuration = configuration;
+        return result;
+    }
+
     /**
      * Builder style adding of messages.
      *
@@ -46,6 +57,16 @@ public GPTChatRequest addMessage(GPTMessageRole role, String content) {
         return this;
     }
 
+    /**
+     * Builder style adding of messages.
+     *
+     * @return this
+     */
+    public GPTChatRequest addMessage(GPTChatMessage message) {
+        messages.add(message);
+        return this;
+    }
+
     /**
      * Builder style adding of messages.
      *
@@ -169,4 +190,5 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(getMessages(), getMaxTokens(), getConfiguration());
     }
+
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index 04ea22d14..8e9695c59 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -15,6 +15,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -76,6 +77,7 @@
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
+import com.composum.ai.backend.base.service.chat.GPTMessageRole;
 import com.composum.ai.backend.base.service.chat.GPTTool;
 import com.composum.ai.backend.base.service.chat.GPTToolCall;
 import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionChoice;
@@ -433,21 +435,51 @@ public void streamingChatCompletion(@Nonnull GPTChatRequest request, @Nonnull GP
     @Override
     public void streamingChatCompletionWithToolCalls(@Nonnull GPTChatRequest request, @Nonnull GPTCompletionCallback callback)
             throws GPTException {
-        List JatoolCalls = null;
-        boolean haveToolCalls;
         GPTCompletionCallback callbackWrapper = new GPTCompletionCallback.GPTCompletionCallbackWrapper(callback) {
+            List collectedToolcalls = null;
+
             @Override
             public void toolDelta(List toolCalls) {
-                toolCalls = GPTToolCall.mergeDelta(toolCalls, toolCalls);
+                collectedToolcalls = GPTToolCall.mergeDelta(collectedToolcalls, toolCalls);
             }
 
             @Override
             public void onFinish(GPTFinishReason finishReason) {
                 if (GPTFinishReason.TOOL_CALLS == finishReason) {
-                    haveToolCalls = true;
+                    LOG.info("Executing tool calls");
+                    LOG.debug("Tool calls: {}", collectedToolcalls);
+                    GPTChatRequest requestWithToolCalls = request.copy();
+                    GPTChatMessage assistantRequestsToolcallsMessage =
+                            new GPTChatMessage(GPTMessageRole.ASSISTANT, null, null, null, collectedToolcalls);
+                    requestWithToolCalls.addMessage(assistantRequestsToolcallsMessage);
+                    for (GPTToolCall toolCall : collectedToolcalls) {
+                        Optional toolOption = request.getConfiguration().getTools().stream()
+                                .filter(tool -> tool.getName().equals(toolCall.getFunction().getName()))
+                                .findAny();
+                        if (!toolOption.isPresent()) { // should be impossible
+                            LOG.error("Bug: Tool {} not found in configuration", toolCall.getFunction().getName());
+                            GPTException error = new GPTException("Bug: Tool " + toolCall.getFunction().getName() + " not found in configuration");
+                            this.onError(error);
+                            throw error;
+                        }
+                        GPTTool tool = toolOption.get();
+                        String toolresult = tool.execute(toolCall.getFunction().getArguments());
+                        if (null == toolresult) {
+                            toolresult = "";
+                        }
+                        LOG.debug("Tool {} with arguments {} returned {}", toolCall.getFunction().getName(),
+                                toolCall.getFunction().getArguments(),
+                                toolresult.substring(0, Math.min(100, toolresult.length())) + "...");
+                        GPTChatMessage toolResponseMessage = new GPTChatMessage(GPTMessageRole.TOOL, toolresult, null, toolCall.getId(), null);
+                        requestWithToolCalls.addMessage(toolResponseMessage);
+                    }
+                    streamingChatCompletionWithToolCalls(requestWithToolCalls, callback);
+                } else {
+                    super.onFinish(finishReason);
                 }
             }
         };
+        streamingChatCompletion(request, callbackWrapper);
     }
 
     /**
@@ -485,7 +517,7 @@ protected void handleStreamingEvent(GPTCompletionCallback callback, long id, Str
                     callback.toolDelta(ChatCompletionToolCall.toGptToolCallList(choice.getDelta().getToolCalls()));
                 }
                 if (choice.getFinishReason() != null) {
-                    System.out.println("Response {} from GPT finished with reason {}" + id + choice.getFinishReason());
+                    LOG.trace("Response {} from GPT finished with reason {}", id, choice.getFinishReason());
                 }
                 GPTFinishReason finishReason = ChatCompletionResponse.FinishReason.toGPTFinishReason(choice.getFinishReason());
                 if (finishReason != null) {
@@ -670,7 +702,7 @@ private List convertTools(GPTConfiguration configuration) {
             details.setStrict(true);
             Map declaration = gson.fromJson(tool.getToolDeclaration(), Map.class);
             details.setParameters(declaration.get("description"));
-            details.setParameters(((Map)declaration.get("function")).get("parameters"));
+            details.setParameters(((Map) declaration.get("function")).get("parameters"));
             toolDescr.setFunction(details);
             result.add(toolDescr);
         }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
index a2f537397..568d96eb8 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionFunctionCallDetails.java
@@ -58,4 +58,14 @@ public GPTFunctionCallDetails toGptFuctionCallDetails() {
         return new GPTFunctionCallDetails(name, arguments);
     }
 
+    public static ChatCompletionFunctionCallDetails make(GPTFunctionCallDetails function) {
+        if (function == null) {
+            return null;
+        }
+        ChatCompletionFunctionCallDetails result = new ChatCompletionFunctionCallDetails();
+        result.setName(function.getName());
+        result.setArguments(function.getArguments());
+        return result;
+    }
+
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
index a4e7ad497..c51032f2a 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionMessage.java
@@ -33,6 +33,10 @@ public class ChatCompletionMessage {
     @SerializedName("tool_call_id")
     private String toolCallId;
 
+    /** List of tool calls that should be executed. */
+    @SerializedName("tool_calls")
+    private List toolCalls;
+
     public static ChatCompletionMessage make(GPTChatMessage message) {
         ChatCompletionMessagePart part;
         if (message.getImageUrl() != null && !message.getImageUrl().isEmpty()) {
@@ -45,6 +49,7 @@ public static ChatCompletionMessage make(GPTChatMessage message) {
         result.setRole(role);
         result.setContent(Collections.singletonList(part));
         result.setToolCallId(message.getToolCallId());
+        result.setToolCalls(ChatCompletionToolCall.make(message.getToolCalls()));
         return result;
     }
 
@@ -73,8 +78,18 @@ public void setContent(List content) {
         this.content = content;
     }
 
+    public List getToolCalls() {
+        return toolCalls;
+    }
+
+    public void setToolCalls(List toolCalls) {
+        this.toolCalls = toolCalls;
+    }
+
     public boolean isEmpty(Void ignoreJustPreventSerialization) {
-        return content == null || content.isEmpty() ||
+        boolean contentIsEmpty = content == null || content.isEmpty() ||
                 !content.stream().anyMatch(m -> !m.isEmpty(null));
+        return contentIsEmpty && (toolCallId == null || toolCallId.isEmpty()) &&
+                (toolCalls == null || toolCalls.isEmpty());
     }
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java
index c77229e08..569a6e4c4 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionRequest.java
@@ -130,7 +130,9 @@ public enum Role {
         @SerializedName("assistant")
         ASSISTANT,
         @SerializedName("system")
-        SYSTEM;
+        SYSTEM,
+        @SerializedName("tool")
+        TOOL;
 
         public static Role make(GPTMessageRole role) {
             switch (role) {
@@ -140,6 +142,8 @@ public static Role make(GPTMessageRole role) {
                     return SYSTEM;
                 case ASSISTANT:
                     return ASSISTANT;
+                case TOOL:
+                    return TOOL;
                 default:
                     throw new IllegalArgumentException("Unknown role " + role);
             }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
index 007d6cf1f..5eb709379 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/chatmodel/ChatCompletionToolCall.java
@@ -153,4 +153,21 @@ public static List toGptToolCallList(List l
         return res;
     }
 
+    public static List make(List toolCalls) {
+        if (toolCalls == null) {
+            return null;
+        }
+        List res = new ArrayList<>(toolCalls.size());
+        for (int i = 0; i < toolCalls.size(); i++) {
+            GPTToolCall toolCall = toolCalls.get(i);
+            ChatCompletionToolCall call = new ChatCompletionToolCall();
+            call.setId(toolCall.getId());
+            call.setIndex(i);
+            call.setType(toolCall.getType());
+            call.setFunction(ChatCompletionFunctionCallDetails.make(toolCall.getFunction()));
+            res.add(call);
+        }
+        return res;
+    }
+
 }
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
index d43b06f64..1959a213e 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
@@ -1,6 +1,7 @@
 package com.composum.ai.backend.base.service.chat.impl;
 
 import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 
 import com.composum.ai.backend.base.service.chat.GPTChatRequest;
@@ -9,6 +10,7 @@
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
 import com.composum.ai.backend.base.service.chat.GPTMessageRole;
 import com.composum.ai.backend.base.service.chat.GPTTool;
+import com.composum.ai.backend.base.service.chat.GPTToolCall;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 
@@ -21,13 +23,17 @@ public class RunGPTChatCompletionServiceImplWithTools extends AbstractGPTRunner
     StringBuilder buffer = new StringBuilder();
     Gson gson = new GsonBuilder().setPrettyPrinting().create();
     private boolean isFinished;
+    List toolCalls;
 
     public static void main(String[] args) throws Exception {
-        RunGPTChatCompletionServiceImplWithTools instance = new RunGPTChatCompletionServiceImplWithTools();
-        instance.setup();
-        instance.runWithoutAutomaticCall();
-        instance.teardown();
-        System.out.println("######################### Done without automatic call #########################");
+        RunGPTChatCompletionServiceImplWithTools instance;
+        if (0 == 1) {
+            instance = new RunGPTChatCompletionServiceImplWithTools();
+            instance.setup();
+            instance.runWithoutAutomaticCall();
+            instance.teardown();
+            System.out.println("######################### Done without automatic call #########################");
+        }
 
         instance = new RunGPTChatCompletionServiceImplWithTools();
         instance.setup();
@@ -43,6 +49,8 @@ private void runWithoutAutomaticCall() throws InterruptedException {
         while (!isFinished) Thread.sleep(1000);
         System.out.println("Complete response:");
         System.out.println(buffer);
+        System.out.println("Tool calls:");
+        System.out.println(gson.toJson(toolCalls));
     }
 
     private void runWithAutomaticCall() throws InterruptedException {
@@ -56,7 +64,7 @@ private void runWithAutomaticCall() throws InterruptedException {
 
     private GPTChatRequest makeRequest() {
         GPTChatRequest request = new GPTChatRequest();
-        request.addMessage(GPTMessageRole.USER, "Wobble the string 'hi' and the string 'ho'.");
+        request.addMessage(GPTMessageRole.USER, "Wobble the string 'hihihi' and the string 'houhou'.");
         request.setConfiguration(GPTConfiguration.ofTools(Arrays.asList(wobbler)));
         return request;
     }
@@ -85,6 +93,11 @@ public void onError(Throwable throwable) {
         isFinished = true;
     }
 
+    @Override
+    public void toolDelta(List toolCalls) {
+        this.toolCalls = GPTToolCall.mergeDelta(this.toolCalls, toolCalls);
+    }
+
     protected GPTTool wobbler = new GPTTool() {
         @Override
         public String getName() {

From 676021d650fd82a3f4aeeff5c9846aae754a61e6 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Wed, 30 Oct 2024 22:53:03 +0100
Subject: [PATCH 21/44] tools for getting page markdown and searching pages

---
 .../ai/backend/base/service/chat/GPTTool.java |  10 +-
 .../impl/GPTChatCompletionServiceImpl.java    |   9 +-
 .../impl/GPTContentCreationServiceImpl.java   |   4 +-
 .../ai/backend/slingbase/AICreateServlet.java |  43 ++++++
 .../slingbase/experimential/AITool.java       |  52 ++++++-
 .../impl/GetPageMarkdownAITool.java           | 130 ++++++++++++++++++
 .../experimential/impl/SearchPageAITool.java  | 130 ++++++++++++++++++
 7 files changed, 367 insertions(+), 11 deletions(-)
 create mode 100644 backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
 create mode 100644 backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java
index ab51c320a..73f19e789 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java
@@ -1,5 +1,8 @@
 package com.composum.ai.backend.base.service.chat;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 /**
  * An action the AI can perform - likely from the sidebar chat.
  *
@@ -10,6 +13,7 @@ public interface GPTTool {
     /**
      * The name of the tool - must be exactly the name given in {@link #getToolDeclaration()}.
      */
+    @Nonnull
     String getName();
 
     /**
@@ -38,11 +42,15 @@ public interface GPTTool {
      *
      * @see "https://platform.openai.com/docs/api-reference/chat/create"
      */
+    @Nonnull
     String getToolDeclaration();
 
     /**
      * Executes the tool call and returns the result to present to the AI.
+     *
+     * @param arguments The arguments to the tool call, as JSON that matches the schema given in {@link #getToolDeclaration()}.
      */
-    public String execute(String arguments);
+    @Nonnull
+    public String execute(@Nullable String arguments);
 
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index 8e9695c59..c1bc41c0e 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -435,6 +435,10 @@ public void streamingChatCompletion(@Nonnull GPTChatRequest request, @Nonnull GP
     @Override
     public void streamingChatCompletionWithToolCalls(@Nonnull GPTChatRequest request, @Nonnull GPTCompletionCallback callback)
             throws GPTException {
+        if (request.getConfiguration() == null || request.getConfiguration().getTools() == null || request.getConfiguration().getTools().isEmpty()) {
+            streamingChatCompletion(request, callback);
+            return;
+        }
         GPTCompletionCallback callbackWrapper = new GPTCompletionCallback.GPTCompletionCallbackWrapper(callback) {
             List collectedToolcalls = null;
 
@@ -701,8 +705,9 @@ private List convertTools(GPTConfiguration configuration) {
             details.setName(tool.getName());
             details.setStrict(true);
             Map declaration = gson.fromJson(tool.getToolDeclaration(), Map.class);
-            details.setParameters(declaration.get("description"));
-            details.setParameters(((Map) declaration.get("function")).get("parameters"));
+            Map function = (Map) declaration.get("function");
+            details.setParameters(function.get("parameters"));
+            details.setDescription((String) function.get("description"));
             toolDescr.setFunction(details);
             result.add(toolDescr);
         }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java
index dd31cb40d..d5775a9ab 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTContentCreationServiceImpl.java
@@ -126,7 +126,7 @@ protected GPTChatRequest makeExecutePromptRequest(String prompt, @Nullable GPTCh
     @Override
     public void executePromptStreaming(@Nonnull String prompt, @Nullable GPTChatRequest additionalParameters, @Nonnull GPTCompletionCallback callback) throws GPTException {
         GPTChatRequest request = makeExecutePromptRequest(prompt, additionalParameters);
-        chatCompletionService.streamingChatCompletion(request, callback);
+        chatCompletionService.streamingChatCompletionWithToolCalls(request, callback);
     }
 
     @Nonnull
@@ -165,7 +165,7 @@ protected GPTChatRequest makeExecuteOnTextRequest(String prompt, String text, @N
     @Override
     public void executePromptOnTextStreaming(@Nonnull String prompt, @Nonnull String text, @Nullable GPTChatRequest additionalParameters, @Nonnull GPTCompletionCallback callback) throws GPTException {
         GPTChatRequest request = makeExecuteOnTextRequest(prompt, text, additionalParameters);
-        chatCompletionService.streamingChatCompletion(request, callback);
+        chatCompletionService.streamingChatCompletionWithToolCalls(request, callback);
     }
 
 }
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
index edbea16dd..20f798194 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
@@ -16,6 +16,7 @@
 import java.util.stream.Stream;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.servlet.Servlet;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletResponse;
@@ -33,6 +34,8 @@
 import org.osgi.service.component.annotations.Activate;
 import org.osgi.service.component.annotations.Component;
 import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.osgi.service.component.annotations.ReferencePolicy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,6 +44,8 @@
 import com.composum.ai.backend.base.service.chat.GPTConfiguration;
 import com.composum.ai.backend.base.service.chat.GPTContentCreationService;
 import com.composum.ai.backend.base.service.chat.GPTMessageRole;
+import com.composum.ai.backend.base.service.chat.GPTTool;
+import com.composum.ai.backend.slingbase.experimential.AITool;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonSyntaxException;
@@ -130,10 +135,25 @@ public class AICreateServlet extends SlingAllMethodsServlet {
     @Reference
     protected AIConfigurationService configurationService;
 
+    protected List tools = Collections.synchronizedList(new ArrayList<>());
+
     protected BundleContext bundleContext;
 
     protected Gson gson = new GsonBuilder().disableHtmlEscaping().create();
 
+    @Reference(service = AITool.class, policy = ReferencePolicy.DYNAMIC,
+            cardinality = ReferenceCardinality.MULTIPLE)
+    protected void addTool(@NotNull final AITool tool) {
+        LOG.info("addTool: {}", tool.getToolName());
+        tools.add(tool);
+    }
+
+    protected void removeTool(@NotNull final AITool tool) {
+        LOG.info("removeTool: {}", tool.getToolName());
+        tools.remove(tool);
+    }
+
+
     @Activate
     public void activate(final BundleContext bundleContext) {
         this.bundleContext = bundleContext;
@@ -259,6 +279,7 @@ protected void doPost(@NotNull SlingHttpServletRequest request, @NotNull SlingHt
         if ("undefined".equals(inputImagePath) || "null".equals(inputImagePath) || StringUtils.isBlank(inputImagePath)) {
             inputImagePath = null;
         }
+
         GPTConfiguration config = configurationService.getGPTConfiguration(request.getResourceResolver(), configBasePath);
         String chat = request.getParameter(PARAMETER_CHAT);
         if (Stream.of(sourcePath, sourceText, inputImagePath).filter(StringUtils::isNotBlank).count() > 1) {
@@ -268,6 +289,7 @@ protected void doPost(@NotNull SlingHttpServletRequest request, @NotNull SlingHt
                     " given, only one of them is allowed");
             return;
         }
+
         boolean richtext = Boolean.TRUE.toString().equalsIgnoreCase(request.getParameter(PARAMETER_RICHTEXT));
         GPTConfiguration mergedConfig = GPTConfiguration.ofRichText(richtext).merge(config);
         Integer maxTokensParam = getOptionalInt(request, response, PARAMETER_MAXTOKENS);
@@ -281,6 +303,12 @@ protected void doPost(@NotNull SlingHttpServletRequest request, @NotNull SlingHt
                 textLength = matcher.group(2);
             }
         }
+
+        List tools = collectTools(request.getResource(), request, response);
+        if (tools != null && !tools.isEmpty()) {
+            mergedConfig = GPTConfiguration.ofTools(tools).merge(mergedConfig);
+        }
+
         GPTChatRequest additionalParameters = makeAdditionalParameters(maxtokens, chat, response, mergedConfig);
 
         String fullPrompt = prompt;
@@ -351,4 +379,19 @@ protected GPTChatRequest makeAdditionalParameters(int maxtokens, String chat, Ht
         return additionalParameters;
     }
 
+    @Nullable
+    protected List collectTools(@Nonnull Resource resource,
+                                         @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+        if (tools == null || tools.isEmpty()) {
+            return null;
+        }
+        List tools = new ArrayList<>();
+        for (AITool tool : this.tools) {
+            if (tool.isAllowedFor(resource)) {
+                tools.add(tool.makeGPTTool(resource, request, response));
+            }
+        }
+        return tools;
+    }
+
 }
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java
index a99971302..0cbb59004 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java
@@ -2,6 +2,11 @@
 
 import java.util.Locale;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.sling.api.SlingHttpServletRequest;
+import org.apache.sling.api.SlingHttpServletResponse;
 import org.apache.sling.api.resource.Resource;
 
 import com.composum.ai.backend.base.service.chat.GPTTool;
@@ -16,12 +21,20 @@ public interface AITool {
     /**
      * Human readable name.
      */
-    String getName(Locale locale);
+    @Nonnull
+    String getName(@Nullable Locale locale);
 
     /**
      * Human readable description.
      */
-    String getDescription(Locale locale);
+    @Nonnull
+    String getDescription(@Nullable Locale locale);
+
+    /**
+     * Name for the purpose of calling - must match {@link #getToolDeclaration()}.
+     */
+    @Nonnull
+    String getToolName();
 
     /**
      * The description to use for the OpenAI tool call. Will be inserted into the OpenAI tools array. E.g.:
@@ -49,20 +62,47 @@ public interface AITool {
      *
      * @see "https://platform.openai.com/docs/api-reference/chat/create"
      */
+    @Nonnull
     String getToolDeclaration();
 
     /**
      * Whether the tool is enabled for the given resource.
      */
-    boolean isAllowedFor(Resource resource);
+    boolean isAllowedFor(@Nonnull Resource resource);
 
     /**
      * Executes the tool call and returns the result to present to the AI.
      * Must only be called if {@link #isAllowedFor(Resource)} returned true.
      */
-    String execute(String arguments, Resource resource);
+    @Nonnull
+    String execute(@Nullable String arguments, @Nonnull Resource resource,
+                   @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response);
+
+    /**
+     * The form useable by {@link com.composum.ai.backend.base.service.chat.GPTChatCompletionService}.
+     */
+    @Nullable
+    default GPTTool makeGPTTool(@Nonnull Resource resource,
+                       @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+        if (!isAllowedFor(resource)) {
+            return null;
+        }
+        return new GPTTool() {
+            @Override
+            public @Nonnull String getName() {
+                return AITool.this.getToolName();
+            }
+
+            @Override
+            public @Nonnull String getToolDeclaration() {
+                return AITool.this.getToolDeclaration();
+            }
 
-    /** The form useable by {@link com.composum.ai.backend.base.service.chat.GPTChatCompletionService}.*/
-    GPTTool makeGPTTool(Resource resource);
+            @Override
+            public @Nonnull String execute(@Nullable String arguments) {
+                return AITool.this.execute(arguments, resource, request, response);
+            }
+        };
+    }
 
 }
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
new file mode 100644
index 000000000..686cb5d89
--- /dev/null
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
@@ -0,0 +1,130 @@
+package com.composum.ai.backend.slingbase.experimential.impl;
+
+import java.util.Locale;
+import java.util.Map;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.sling.api.SlingHttpServletRequest;
+import org.apache.sling.api.SlingHttpServletResponse;
+import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceResolver;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.ConfigurationPolicy;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Modified;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.metatype.annotations.AttributeDefinition;
+import org.osgi.service.metatype.annotations.Designate;
+import org.osgi.service.metatype.annotations.ObjectClassDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.composum.ai.backend.slingbase.ApproximateMarkdownService;
+import com.composum.ai.backend.slingbase.experimential.AITool;
+import com.google.gson.Gson;
+
+@Component(service = AITool.class, configurationPolicy = ConfigurationPolicy.REQUIRE)
+@Designate(ocd = GetPageMarkdownAITool.Config.class)
+public class GetPageMarkdownAITool implements AITool {
+    private static final Logger LOG = LoggerFactory.getLogger(GetPageMarkdownAITool.class);
+    private Config config;
+    private Gson gson = new Gson();
+
+    @Reference
+    private ApproximateMarkdownService markdownService;
+
+    @Override
+    public @Nonnull String getName(@Nullable Locale locale) {
+        return "Get Text of Page";
+    }
+
+    @Override
+    public @Nonnull String getDescription(@Nullable Locale locale) {
+        return "Returns a markdown representation of the text of a given page";
+    }
+
+    @Override
+    public @Nonnull String getToolName() {
+        return "get_pagetext";
+    }
+
+    @Override
+    public @Nonnull String getToolDeclaration() {
+        return "{\n" +
+                "  \"type\": \"function\",\n" +
+                "  \"function\": {\n" +
+                "    \"name\": \"get_pagetext\",\n" +
+                "    \"description\": \"Get the text of a page with a given path.\",\n" +
+                "    \"parameters\": {\n" +
+                "      \"type\": \"object\",\n" +
+                "      \"properties\": {\n" +
+                "        \"path\": {\n" +
+                "          \"type\": \"string\",\n" +
+                "          \"description\": \"The path of the page to get the text of.\"\n" +
+                "        }\n" +
+                "      },\n" +
+                "      \"required\": [\"path\"],\n" +
+                "      \"additionalProperties\": false\n" +
+                "    }\n" +
+                "  },\n" +
+                "  \"strict\": true\n" +
+                "}";
+    }
+
+    @Override
+    public boolean isAllowedFor(@Nonnull Resource resource) {
+        return true;
+    }
+
+    /**
+     * Does a query with lucene and then rates the results with the embedding.
+     */
+    @Override
+    public @Nonnull String execute(@Nullable String arguments, @Nonnull Resource resource,
+                                   @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+        try {
+            Map parsedArguments = gson.fromJson(arguments, Map.class);
+            String path = (String) parsedArguments.get("path");
+            if (path == null || path.isEmpty()) {
+                return "Missing path parameter";
+            }
+            if (!config.allowedPathsRegex().matches(path)) {
+                return "Path not allowed";
+            }
+            ResourceResolver resolver = resource.getResourceResolver();
+            Resource pathResource = resolver.getResource(path);
+            String markdown = markdownService.approximateMarkdown(pathResource, request, response);
+            LOG.debug("Markdown of {} : {}", path, StringUtils.abbreviate(markdown, 80));
+            return markdown;
+        } catch (Exception e) {
+            LOG.error("Error in search page AI tool", e);
+            return "Error in search page AI tool: " + e;
+        }
+    }
+
+    // activate and deactivate methods
+    @Activate
+    @Modified
+    protected void activate(Config config) {
+        this.config = config;
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        this.config = null;
+    }
+
+    @ObjectClassDefinition(name = "Composum AI Tool Get Page Markdown",
+            description = "Provides the AI with a tool to search for page paths. Needs a lucene index for all pages.")
+    public @interface Config {
+
+        @AttributeDefinition(name = "Allowed paths regex",
+                description = "A regex to match the paths that this tool is allowed to be used on.")
+        String allowedPathsRegex() default "/content/.*";
+
+    }
+}
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
new file mode 100644
index 000000000..43efa9512
--- /dev/null
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
@@ -0,0 +1,130 @@
+package com.composum.ai.backend.slingbase.experimential.impl;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.sling.api.SlingHttpServletRequest;
+import org.apache.sling.api.SlingHttpServletResponse;
+import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceResolver;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.ConfigurationPolicy;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Modified;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.metatype.annotations.AttributeDefinition;
+import org.osgi.service.metatype.annotations.Designate;
+import org.osgi.service.metatype.annotations.ObjectClassDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.composum.ai.backend.slingbase.RAGService;
+import com.composum.ai.backend.slingbase.experimential.AITool;
+import com.google.gson.Gson;
+
+@Component(service = AITool.class, configurationPolicy = ConfigurationPolicy.REQUIRE)
+@Designate(ocd = SearchPageAITool.Config.class)
+public class SearchPageAITool implements AITool {
+    private static final Logger LOG = LoggerFactory.getLogger(SearchPageAITool.class);
+    private Config config;
+    private Gson gson = new Gson();
+
+    @Reference
+    private RAGService ragService;
+
+    @Override
+    public @Nonnull String getName(@Nullable Locale locale) {
+        return "Search Page";
+    }
+
+    @Override
+    public @Nonnull String getDescription(@Nullable Locale locale) {
+        return "Search for a page";
+    }
+
+    @Override
+    public @Nonnull String getToolName() {
+        return "search_page";
+    }
+
+    @Override
+    public @Nonnull String getToolDeclaration() {
+        return "{\n" +
+                "  \"type\": \"function\",\n" +
+                "  \"function\": {\n" +
+                "    \"name\": \"search_page\",\n" +
+                "    \"description\": \"Search for a page that best matches the given query\",\n" +
+                "    \"parameters\": {\n" +
+                "      \"type\": \"object\",\n" +
+                "      \"properties\": {\n" +
+                "        \"query\": {\n" +
+                "          \"type\": \"string\",\n" +
+                "          \"description\": \"The search query\"\n" +
+                "        }\n" +
+                "      },\n" +
+                "      \"required\": [\"query\"],\n" +
+                "      \"additionalProperties\": false\n" +
+                "    }\n" +
+                "  },\n" +
+                "  \"strict\": true\n" +
+                "}";
+    }
+
+    @Override
+    public boolean isAllowedFor(@Nonnull Resource resource) {
+        return true;
+    }
+
+    /**
+     * Does a query with lucene and then rates the results with the embedding.
+     */
+    @Override
+    public @Nonnull String execute(@Nullable String arguments, @Nonnull Resource resource,
+                                   @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+        try {
+            Map parsedArguments = gson.fromJson(arguments, Map.class);
+            String query = (String) parsedArguments.get("query");
+            if (query == null || query.isEmpty()) {
+                return "Missing query parameter";
+            }
+            ResourceResolver resolver = resource.getResourceResolver();
+            Resource rootResource = resolver.getResource(config.rootPath());
+            List paths = ragService.searchRelated(rootResource, query, 20);
+            List resources = paths.stream().map(resolver::getResource).collect(Collectors.toList());
+            List ordered = ragService.orderByEmbedding(query, resources, request, response, rootResource);
+            List result = ordered.stream().map(Resource::getPath).collect(Collectors.toList());
+            LOG.debug("Search page AI tool found for '{}' : {}", query, result);
+            return gson.toJson(result);
+        } catch (Exception e) {
+            LOG.error("Error in search page AI tool", e);
+            return "Error in search page AI tool: " + e;
+        }
+    }
+
+    // activate and deactivate methods
+    @Activate
+    @Modified
+    protected void activate(Config config) {
+        this.config = config;
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        this.config = null;
+    }
+
+    @ObjectClassDefinition(name = "Composum AI Tool Search Pageq",
+            description = "Provides the AI with a tool to search for page paths. Needs a lucene index for all pages.")
+    public @interface Config {
+
+        @AttributeDefinition(name = "Root path", description = "The root path to search for pages. Default is /content.")
+        String rootPath() default "/content";
+
+    }
+}

From 177cc29e174e2bb59fe37c114db700bb1be7128b Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Thu, 31 Oct 2024 06:32:04 +0100
Subject: [PATCH 22/44] first working version of tools

---
 .../service/chat/GPTCompletionCallback.java   | 24 +++++++++++--
 .../ai/backend/base/service/chat/GPTTool.java |  2 +-
 .../impl/GPTChatCompletionServiceImpl.java    |  2 +-
 ...GPTChatCompletionServiceImplWithTools.java |  7 +++-
 .../ai/backend/slingbase/AICreateServlet.java |  2 ++
 .../ai/backend/slingbase/EventStream.java     | 13 +++++++
 .../slingbase/experimential/AITool.java       |  8 +++--
 .../impl/GetPageMarkdownAITool.java           |  9 +++--
 .../experimential/impl/SearchPageAITool.java  | 12 +++++--
 .../model/SlingGPTExecutionContext.java       | 36 +++++++++++++++++++
 10 files changed, 102 insertions(+), 13 deletions(-)
 create mode 100644 backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/model/SlingGPTExecutionContext.java

diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
index 05e180a3c..5d625e90b 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTCompletionCallback.java
@@ -7,8 +7,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.composum.ai.backend.base.service.chat.impl.chatmodel.ChatCompletionToolCall;
-
 /**
  * For a streaming mode this is given as parameter for the method call and receives the streamed data; the method returns only when the response is complete.
  */
@@ -35,6 +33,13 @@ public interface GPTCompletionCallback {
      */
     void setLoggingId(String loggingId);
 
+    /**
+     * For tool calls: set the context to execute actions in.
+     */
+    default GPTToolExecutionContext getToolExecutionContext() {
+        return null;
+    }
+
     /**
      * For debugging - the request that was sent to ChatGPT as JSON.
      */
@@ -55,6 +60,12 @@ default void toolDelta(List toolCalls) {
         // empty
     }
 
+    /**
+     * For tool calls: context to execute actions in.
+     */
+    public interface GPTToolExecutionContext {
+        // empty here - has to be specifiec in other layers
+    }
 
     /**
      * Forwards all methods to a delegate.
@@ -96,6 +107,10 @@ public void toolDelta(List toolCalls) {
             delegate.toolDelta(toolCalls);
         }
 
+        public GPTToolExecutionContext getToolExecutionContext() {
+            return delegate.getToolExecutionContext();
+        }
+
     }
 
     /**
@@ -152,6 +167,11 @@ public List getToolCalls() {
             return toolCalls;
         }
 
+        @Override
+        public GPTToolExecutionContext getToolExecutionContext() {
+            return null;
+        }
+
     }
 
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java
index 73f19e789..c820ab88c 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/GPTTool.java
@@ -51,6 +51,6 @@ public interface GPTTool {
      * @param arguments The arguments to the tool call, as JSON that matches the schema given in {@link #getToolDeclaration()}.
      */
     @Nonnull
-    public String execute(@Nullable String arguments);
+    public String execute(@Nullable String arguments, @Nullable GPTCompletionCallback.GPTToolExecutionContext context);
 
 }
diff --git a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
index c1bc41c0e..37e03c805 100644
--- a/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
+++ b/backend/base/src/main/java/com/composum/ai/backend/base/service/chat/impl/GPTChatCompletionServiceImpl.java
@@ -467,7 +467,7 @@ public void onFinish(GPTFinishReason finishReason) {
                             throw error;
                         }
                         GPTTool tool = toolOption.get();
-                        String toolresult = tool.execute(toolCall.getFunction().getArguments());
+                        String toolresult = tool.execute(toolCall.getFunction().getArguments(), getToolExecutionContext());
                         if (null == toolresult) {
                             toolresult = "";
                         }
diff --git a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
index 1959a213e..b84414a14 100644
--- a/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
+++ b/backend/base/src/test/java/com/composum/ai/backend/base/service/chat/impl/RunGPTChatCompletionServiceImplWithTools.java
@@ -81,6 +81,11 @@ public void setLoggingId(String loggingId) {
         System.out.println("Logging ID: " + loggingId);
     }
 
+    @Override
+    public GPTToolExecutionContext getToolExecutionContext() {
+        return null;
+    }
+
     @Override
     public void onNext(String item) {
         buffer.append(item);
@@ -128,7 +133,7 @@ public String getToolDeclaration() {
         }
 
         @Override
-        public String execute(String arguments) {
+        public String execute(String arguments, GPTToolExecutionContext context) {
             Map parsedArguments = gson.fromJson(arguments, Map.class);
             String towobble = (String) parsedArguments.get("towobble");
             StringBuilder result = new StringBuilder();
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
index 20f798194..5ae1499e0 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
@@ -46,6 +46,7 @@
 import com.composum.ai.backend.base.service.chat.GPTMessageRole;
 import com.composum.ai.backend.base.service.chat.GPTTool;
 import com.composum.ai.backend.slingbase.experimential.AITool;
+import com.composum.ai.backend.slingbase.model.SlingGPTExecutionContext;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonSyntaxException;
@@ -207,6 +208,7 @@ protected void doGet(@NotNull SlingHttpServletRequest request, @NotNull SlingHtt
         if (stream == null) {
             response.sendError(HttpServletResponse.SC_GONE, "Stream " + streamId + " not found (anymore?)");
         } else {
+            stream.setContext(new SlingGPTExecutionContext(request, response));
             response.setCharacterEncoding("UTF-8");
             response.setContentType("text/event-stream");
             response.setHeader("Cache-Control", "no-cache");
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/EventStream.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/EventStream.java
index 4be0344f3..f63263c54 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/EventStream.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/EventStream.java
@@ -20,6 +20,7 @@
 import com.composum.ai.backend.base.service.StringstreamSlowdown;
 import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.base.service.chat.GPTFinishReason;
+import com.composum.ai.backend.slingbase.model.SlingGPTExecutionContext;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 
@@ -49,6 +50,8 @@ public class EventStream implements GPTCompletionCallback {
 
     private final StringstreamSlowdown slowdown = new StringstreamSlowdown(this::writeData, 250);
 
+    private SlingGPTExecutionContext context;
+
     public void setId(String id) {
         this.id = id;
     }
@@ -163,4 +166,14 @@ public void onError(Throwable throwable) {
         queue.add("");
         queue.add(QUEUEEND);
     }
+
+    public void setContext(SlingGPTExecutionContext context) {
+        this.context = context;
+    }
+
+    @Override
+    public SlingGPTExecutionContext getToolExecutionContext() {
+        return context;
+    }
+
 }
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java
index 0cbb59004..94c19e864 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/AITool.java
@@ -9,6 +9,8 @@
 import org.apache.sling.api.SlingHttpServletResponse;
 import org.apache.sling.api.resource.Resource;
 
+import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
+import com.composum.ai.backend.base.service.chat.GPTCompletionCallback.GPTToolExecutionContext;
 import com.composum.ai.backend.base.service.chat.GPTTool;
 
 /**
@@ -76,7 +78,7 @@ public interface AITool {
      */
     @Nonnull
     String execute(@Nullable String arguments, @Nonnull Resource resource,
-                   @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response);
+                   @Nullable GPTCompletionCallback.GPTToolExecutionContext context);
 
     /**
      * The form useable by {@link com.composum.ai.backend.base.service.chat.GPTChatCompletionService}.
@@ -99,8 +101,8 @@ default GPTTool makeGPTTool(@Nonnull Resource resource,
             }
 
             @Override
-            public @Nonnull String execute(@Nullable String arguments) {
-                return AITool.this.execute(arguments, resource, request, response);
+            public @Nonnull String execute(@Nullable String arguments, @Nullable GPTToolExecutionContext context) {
+                return AITool.this.execute(arguments, resource, context);
             }
         };
     }
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
index 686cb5d89..602975e96 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
@@ -23,8 +23,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.slingbase.ApproximateMarkdownService;
 import com.composum.ai.backend.slingbase.experimential.AITool;
+import com.composum.ai.backend.slingbase.model.SlingGPTExecutionContext;
 import com.google.gson.Gson;
 
 @Component(service = AITool.class, configurationPolicy = ConfigurationPolicy.REQUIRE)
@@ -77,7 +79,8 @@ public class GetPageMarkdownAITool implements AITool {
 
     @Override
     public boolean isAllowedFor(@Nonnull Resource resource) {
-        return true;
+        return config != null && config.allowedPathsRegex() != null &&
+                config.allowedPathsRegex().matches(resource.getPath());
     }
 
     /**
@@ -85,8 +88,10 @@ public boolean isAllowedFor(@Nonnull Resource resource) {
      */
     @Override
     public @Nonnull String execute(@Nullable String arguments, @Nonnull Resource resource,
-                                   @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+                                   @Nullable GPTCompletionCallback.GPTToolExecutionContext context) {
         try {
+            SlingHttpServletRequest request = ((SlingGPTExecutionContext) context).getRequest();
+            SlingHttpServletResponse response = ((SlingGPTExecutionContext) context).getResponse();
             Map parsedArguments = gson.fromJson(arguments, Map.class);
             String path = (String) parsedArguments.get("path");
             if (path == null || path.isEmpty()) {
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
index 43efa9512..5955d81bc 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
@@ -24,8 +24,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
 import com.composum.ai.backend.slingbase.RAGService;
 import com.composum.ai.backend.slingbase.experimential.AITool;
+import com.composum.ai.backend.slingbase.model.SlingGPTExecutionContext;
 import com.google.gson.Gson;
 
 @Component(service = AITool.class, configurationPolicy = ConfigurationPolicy.REQUIRE)
@@ -86,19 +88,23 @@ public boolean isAllowedFor(@Nonnull Resource resource) {
      */
     @Override
     public @Nonnull String execute(@Nullable String arguments, @Nonnull Resource resource,
-                                   @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+                                   @Nullable GPTCompletionCallback.GPTToolExecutionContext context) {
         try {
+            SlingHttpServletRequest request = ((SlingGPTExecutionContext) context).getRequest();
+            SlingHttpServletResponse response = ((SlingGPTExecutionContext) context).getResponse();
             Map parsedArguments = gson.fromJson(arguments, Map.class);
             String query = (String) parsedArguments.get("query");
             if (query == null || query.isEmpty()) {
                 return "Missing query parameter";
             }
-            ResourceResolver resolver = resource.getResourceResolver();
+            ResourceResolver resolver = request.getResourceResolver();
             Resource rootResource = resolver.getResource(config.rootPath());
             List paths = ragService.searchRelated(rootResource, query, 20);
             List resources = paths.stream().map(resolver::getResource).collect(Collectors.toList());
             List ordered = ragService.orderByEmbedding(query, resources, request, response, rootResource);
-            List result = ordered.stream().map(Resource::getPath).collect(Collectors.toList());
+            List result = ordered.stream().map(Resource::getPath)
+                    .map(path -> path.replaceAll("/jcr:content$", ""))
+                    .collect(Collectors.toList());
             LOG.debug("Search page AI tool found for '{}' : {}", query, result);
             return gson.toJson(result);
         } catch (Exception e) {
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/model/SlingGPTExecutionContext.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/model/SlingGPTExecutionContext.java
new file mode 100644
index 000000000..54e7c6bb1
--- /dev/null
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/model/SlingGPTExecutionContext.java
@@ -0,0 +1,36 @@
+package com.composum.ai.backend.slingbase.model;
+
+import javax.annotation.Nonnull;
+
+import org.apache.sling.api.SlingHttpServletRequest;
+import org.apache.sling.api.SlingHttpServletResponse;
+
+import com.composum.ai.backend.base.service.chat.GPTCompletionCallback;
+
+/**
+ * For Sling tools: the request and response of the streaming as executin context for tool calls.
+ */
+public class SlingGPTExecutionContext implements GPTCompletionCallback.GPTToolExecutionContext {
+
+    @Nonnull
+    private final SlingHttpServletRequest request;
+
+    @Nonnull
+    private final SlingHttpServletResponse response;
+
+    public SlingGPTExecutionContext(@Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+        this.request = request;
+        this.response = response;
+    }
+
+    @Nonnull
+    public SlingHttpServletRequest getRequest() {
+        return request;
+    }
+
+    @Nonnull
+    public SlingHttpServletResponse getResponse() {
+        return response;
+    }
+
+}

From 4583d1f7a362291c7fffe862da002c352d334c02 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Thu, 31 Oct 2024 11:58:35 +0100
Subject: [PATCH 23/44] make URLs clickable

---
 .../src/main/webpack/site/SidePanelDialog.js  | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js b/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js
index f3379fedb..86996805c 100644
--- a/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js
+++ b/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js
@@ -300,6 +300,7 @@ class SidePanelDialog {
 
     doneCallback(text, event) {
         if (this.debug) console.log("SidePanelDialog doneCallback", arguments);
+        this.addLinksToResponse();
         this.ensurePromptCount(this.$promptContainer.find('.composum-ai-response').length + 1);
         this.$promptContainer.find('.composum-ai-prompt:last').focus();
         console.log("SidePanelDialog doneCallback", arguments);
@@ -316,6 +317,29 @@ class SidePanelDialog {
         this.history.maybeSaveToHistory();
     }
 
+    /** There might be content paths or URLs created by tools in the response. We turn them into clickable anchors. */
+    addLinksToResponse() {
+        const lastResponse = this.$promptContainer.find('.composum-ai-response:last');
+        try {
+            lastResponse.html(lastResponse.html().replace(/(https?:\/\/[^\s<]+|\/content\/[\w\-\/]+(?:\.html)?)/g, function (match) {
+                if (match.startsWith('http://') || match.startsWith('https://')) {
+                    // Handle full URLs
+                    return '' + match + '';
+                } else if (match.startsWith('/content/')) {
+                    // Handle AEM content paths
+                    const editorPathMatch = location.pathname.match(/\/[^.]*\.[^/]*/);
+                    const editorPath = editorPathMatch ? editorPathMatch[0] : '';
+                    const cleanedMatch = match.replace(/\/jcr:content$/, '').replace(/\.html$/, '');
+                    return '' + match + '';
+                } else {
+                    return match;
+                }
+            }));
+        } catch (e) {
+            console.error("addLinksToResponse error for ", lastResponse.html(), e);
+        }
+    }
+
     errorCallback(data) {
         console.error("SidePanelDialog errorCallback", arguments);
         this.showError(data);

From 1c9868f40a6396914450299a6b49cce3ab3b24bd Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Thu, 31 Oct 2024 13:31:26 +0100
Subject: [PATCH 24/44] fixes and improvements

---
 .../ai/backend/slingbase/AICreateServlet.java |  2 +-
 .../impl/GetPageMarkdownAITool.java           |  5 +-
 .../experimential/impl/SearchPageAITool.java  | 52 ++++++++++++++-----
 3 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
index 5ae1499e0..9bf439014 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/AICreateServlet.java
@@ -306,7 +306,7 @@ protected void doPost(@NotNull SlingHttpServletRequest request, @NotNull SlingHt
             }
         }
 
-        List tools = collectTools(request.getResource(), request, response);
+        List tools = collectTools(request.getResourceResolver().getResource(configBasePath), request, response);
         if (tools != null && !tools.isEmpty()) {
             mergedConfig = GPTConfiguration.ofTools(tools).merge(mergedConfig);
         }
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
index 602975e96..b14f4190e 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/GetPageMarkdownAITool.java
@@ -124,11 +124,12 @@ protected void deactivate() {
     }
 
     @ObjectClassDefinition(name = "Composum AI Tool Get Page Markdown",
-            description = "Provides the AI with a tool to search for page paths. Needs a lucene index for all pages.")
+            description = "Provides the AI with a tool to search for page paths. Needs a lucene index for all pages." +
+                    "If there is no configuration, the tool is not active.")
     public @interface Config {
 
         @AttributeDefinition(name = "Allowed paths regex",
-                description = "A regex to match the paths that this tool is allowed to be used on.")
+                description = "A regex to match the paths that this tool is allowed to be used on. Default: /content/.*")
         String allowedPathsRegex() default "/content/.*";
 
     }
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
index 5955d81bc..e393f5c3a 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
@@ -35,7 +35,7 @@
 public class SearchPageAITool implements AITool {
     private static final Logger LOG = LoggerFactory.getLogger(SearchPageAITool.class);
     private Config config;
-    private Gson gson = new Gson();
+    private final Gson gson = new Gson();
 
     @Reference
     private RAGService ragService;
@@ -61,7 +61,8 @@ public class SearchPageAITool implements AITool {
                 "  \"type\": \"function\",\n" +
                 "  \"function\": {\n" +
                 "    \"name\": \"search_page\",\n" +
-                "    \"description\": \"Search for a page that best matches the given query\",\n" +
+                "    \"description\": \"Search for titles and JCR paths for pages that best match the given query. " +
+                "Never add a protocol / host / port to JCR paths (/content/...)!\",\n" +
                 "    \"parameters\": {\n" +
                 "      \"type\": \"object\",\n" +
                 "      \"properties\": {\n" +
@@ -98,22 +99,45 @@ public boolean isAllowedFor(@Nonnull Resource resource) {
                 return "Missing query parameter";
             }
             ResourceResolver resolver = request.getResourceResolver();
-            Resource rootResource = resolver.getResource(config.rootPath());
-            List paths = ragService.searchRelated(rootResource, query, 20);
+            // go up to site resource starting from resource
+            Resource rootResource = resolver.getResource(resource.getPath()); // original resource resolver is already closed.
+            while (rootResource != null && rootResource.getPath().split("/").length > config.siteLevel() + 1) {
+                rootResource = rootResource.getParent();
+            }
+
+            List paths = ragService.searchRelated(rootResource, query, config.resultCount());
             List resources = paths.stream().map(resolver::getResource).collect(Collectors.toList());
             List ordered = ragService.orderByEmbedding(query, resources, request, response, rootResource);
-            List result = ordered.stream().map(Resource::getPath)
+            List resultPaths = ordered.stream().map(Resource::getPath)
                     .map(path -> path.replaceAll("/jcr:content$", ""))
                     .collect(Collectors.toList());
-            LOG.debug("Search page AI tool found for '{}' : {}", query, result);
-            return gson.toJson(result);
+            LOG.debug("Search page AI tool found for '{}' : {}", query, resultPaths);
+
+            // collect titles (properties "jcr:title" / "title") of resource and make itemized list of markdown links
+            StringBuilder result = new StringBuilder("Here are the JCR paths for the " + config.resultCount() +
+                    " pages best matching the query. Never add a protocol / host / port to JCR paths (/content/...)!\n\n");
+            for (String path : resultPaths) {
+                Resource res = resolver.getResource(path);
+                if (res != null) {
+                    res = res.getChild("jcr:content") != null ? res.getChild("jcr:content") : res;
+                    String title = res.getValueMap().get("jcr:title",
+                            res.getValueMap().get("title", String.class));
+                    if (title == null || title.startsWith("/")) {
+                        result.append("- ").append(path).append("\n");
+                    } else {
+                        result.append("- [").append(title).append("](").append(path).append(")\n");
+                    }
+                } else {
+                    result.append("- ").append(path).append("\n");
+                }
+            }
+            return result.toString();
         } catch (Exception e) {
             LOG.error("Error in search page AI tool", e);
             return "Error in search page AI tool: " + e;
         }
     }
 
-    // activate and deactivate methods
     @Activate
     @Modified
     protected void activate(Config config) {
@@ -125,12 +149,16 @@ protected void deactivate() {
         this.config = null;
     }
 
-    @ObjectClassDefinition(name = "Composum AI Tool Search Pageq",
-            description = "Provides the AI with a tool to search for page paths. Needs a lucene index for all pages.")
+    @ObjectClassDefinition(name = "Composum AI Tool Search Pages",
+            description = "Provides the AI with a tool to search for page paths. Needs a lucene index for all pages." +
+                    "If there is no configuration the tool is not active.")
     public @interface Config {
 
-        @AttributeDefinition(name = "Root path", description = "The root path to search for pages. Default is /content.")
-        String rootPath() default "/content";
+        @AttributeDefinition(name = "Result count", description = "The number of results to return. Default is 20.")
+        int resultCount() default 20;
+
+        @AttributeDefinition(name = "Site level", description = "The number of path segments a site has, used to identify the site root. Default is 2, for sites like /content/my-site.")
+        int siteLevel() default 2;
 
     }
 }

From 7328e95cfa303a13171e2d124256f381e0ead437 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Thu, 31 Oct 2024 17:12:01 +0100
Subject: [PATCH 25/44] page templating workflow process

---
 .../PageTemplatingWorkflowProcess.java        | 91 +++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100644 aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/workflow/PageTemplatingWorkflowProcess.java

diff --git a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/workflow/PageTemplatingWorkflowProcess.java b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/workflow/PageTemplatingWorkflowProcess.java
new file mode 100644
index 000000000..d3db15cb2
--- /dev/null
+++ b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/workflow/PageTemplatingWorkflowProcess.java
@@ -0,0 +1,91 @@
+package com.composum.ai.aem.core.impl.autotranslate.workflow;
+
+import static com.adobe.granite.workflow.PayloadMap.TYPE_JCR_PATH;
+
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceResolver;
+import org.osgi.service.component.annotations.Reference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.adobe.granite.workflow.WorkflowException;
+import com.adobe.granite.workflow.WorkflowSession;
+import com.adobe.granite.workflow.exec.WorkItem;
+import com.adobe.granite.workflow.exec.WorkflowData;
+import com.adobe.granite.workflow.exec.WorkflowProcess;
+import com.adobe.granite.workflow.metadata.MetaDataMap;
+import com.adobe.granite.workflow.model.ValidationException;
+import com.composum.ai.backend.slingbase.experimential.AITemplatingService;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonSyntaxException;
+
+/**
+ * Triggers a call of the {@link AITemplatingService} on the current page.
+ * If parameter "reset" is given, it does a resetToPrompts instead.
+ * As process arguments it can be given a json {"reset":true} to reset the page to prompts.
+ * The URL as source needs to be in the page somewhere.
+ *
+ * @see "https://ai.composum.com/aiPageTemplating.html"
+ */
+public class PageTemplatingWorkflowProcess implements WorkflowProcess {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PageTemplatingWorkflowProcess.class);
+
+    @Reference
+    protected AITemplatingService aiTemplatingService;
+
+    protected final Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+
+    @Override
+    public void execute(WorkItem workItem, WorkflowSession workflowSession, MetaDataMap metaDataMap) throws WorkflowException {
+        String path = null;
+        try (ResourceResolver resourceResolver = workflowSession.adaptTo(ResourceResolver.class)) {
+            WorkflowData workflowData = workItem.getWorkflowData();
+            if (workflowData.getPayloadType().equals(TYPE_JCR_PATH)) {
+                path = workflowData.getPayload().toString();
+                if (path == null || !path.startsWith("/content/")) {
+                    LOG.error("Workflow started with wrong payload path {}", path);
+                    throw new IllegalArgumentException("Workflow started with wrong payload path: " + path);
+                }
+                Resource resource = resourceResolver.getResource(path);
+                if (resource != null) {
+                    if (isReset(workItem, metaDataMap)) {
+                        aiTemplatingService.resetToPrompts(resource);
+                    } else {
+                        aiTemplatingService.replacePromptsInResource(resource, null, null, null);
+                    }
+                } else {
+                    LOG.error("Autotranslate workflow started with wrong payload path - no resource found: {}", path);
+                }
+            } else {
+                LOG.error("Autotranslate workflow started with wrong payload type: {}", workflowData.getPayloadType());
+            }
+
+        } catch (Exception e) {
+            LOG.error("Failed to process page templating for {}", path, e);
+            throw new WorkflowException("Failed to process page templating for " + path, e);
+        }
+    }
+
+
+    protected boolean isReset(WorkItem workItem, MetaDataMap metaDataMap) throws WorkflowException {
+        Object payload = workItem.getWorkflowData().getPayload();
+        String processArguments = metaDataMap.get("PROCESS_ARGS", String.class); // e.g. {"reset":true}
+        LOG.info("TriggerRollout workflow receiver {} , args {}", payload, processArguments);
+        if (StringUtils.isNotBlank(processArguments)) {
+            try {
+                Map parameters = gson.fromJson(processArguments, Map.class);
+                return parameters.containsKey("reset") && (Boolean) parameters.get("reset");
+            } catch (JsonSyntaxException | ClassCastException | IllegalArgumentException e) {
+                LOG.error("Failed to parse process arguments: {} , ", processArguments, e);
+                throw new ValidationException("Failed to parse process arguments " + processArguments, e);
+            }
+        }
+        return false;
+    }
+
+}

From 36ff05f366728dd452f59b6a5f4a89f81e97c15a Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Thu, 31 Oct 2024 17:12:25 +0100
Subject: [PATCH 26/44] change format of search page to hopefully avoid url
 "completion" by the model

---
 .../slingbase/experimential/impl/SearchPageAITool.java     | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
index e393f5c3a..36d8f78b1 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
@@ -61,8 +61,7 @@ public class SearchPageAITool implements AITool {
                 "  \"type\": \"function\",\n" +
                 "  \"function\": {\n" +
                 "    \"name\": \"search_page\",\n" +
-                "    \"description\": \"Search for titles and JCR paths for pages that best match the given query. " +
-                "Never add a protocol / host / port to JCR paths (/content/...)!\",\n" +
+                "    \"description\": \"Search for titles and JCR paths for pages that best match the given query.\",\n" +
                 "    \"parameters\": {\n" +
                 "      \"type\": \"object\",\n" +
                 "      \"properties\": {\n" +
@@ -115,7 +114,7 @@ public boolean isAllowedFor(@Nonnull Resource resource) {
 
             // collect titles (properties "jcr:title" / "title") of resource and make itemized list of markdown links
             StringBuilder result = new StringBuilder("Here are the JCR paths for the " + config.resultCount() +
-                    " pages best matching the query. Never add a protocol / host / port to JCR paths (/content/...)!\n\n");
+                    " pages best matching the query.\n\n");
             for (String path : resultPaths) {
                 Resource res = resolver.getResource(path);
                 if (res != null) {
@@ -125,7 +124,7 @@ public boolean isAllowedFor(@Nonnull Resource resource) {
                     if (title == null || title.startsWith("/")) {
                         result.append("- ").append(path).append("\n");
                     } else {
-                        result.append("- [").append(title).append("](").append(path).append(")\n");
+                        result.append("- ").append(title).append(": ").append(path).append("\n");
                     }
                 } else {
                     result.append("- ").append(path).append("\n");

From 453c50f421bbbb65ec144c019686aa74454e3b9b Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Thu, 31 Oct 2024 17:28:34 +0100
Subject: [PATCH 27/44] import comparetool

---
 .../components/tool/comparetool/.content.xml  |   6 +
 .../tool/comparetool/comparetool.html         | 126 ++++++++++++++++++
 2 files changed, 132 insertions(+)
 create mode 100644 aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/.content.xml
 create mode 100644 aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html

diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/.content.xml b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/.content.xml
new file mode 100644
index 000000000..f1e1885eb
--- /dev/null
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/.content.xml
@@ -0,0 +1,6 @@
+
+
+
diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
new file mode 100644
index 000000000..0503c415c
--- /dev/null
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
@@ -0,0 +1,126 @@
+
+
+
+    
+    Compare Tool
+    
+
+
+    
    
+        
+        
+    
    
+    
    
+        
+        
+    
    
+    
+
+

From 07d46f45add292ef029ce5f06bf53895e5fe826c Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Thu, 31 Oct 2024 18:20:02 +0100
Subject: [PATCH 28/44] mostly fix problem of garbled content links

---
 .../src/main/webpack/site/SidePanelDialog.js         | 12 +++++++++---
 .../experimential/impl/SearchPageAITool.java         |  8 ++++----
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js b/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js
index 86996805c..e05be7dc8 100644
--- a/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js
+++ b/aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js
@@ -321,15 +321,21 @@ class SidePanelDialog {
     addLinksToResponse() {
         const lastResponse = this.$promptContainer.find('.composum-ai-response:last');
         try {
-            lastResponse.html(lastResponse.html().replace(/(https?:\/\/[^\s<]+|\/content\/[\w\-\/]+(?:\.html)?)/g, function (match) {
+            lastResponse.html(lastResponse.html().replace(/(https?:\/\/[^\s<]+|\/?content\/[\w\-\/]+(?:\.html)?)/g, function (match) {
+                if (match.startsWith('http://localhost:4502')) {
+                    match = match.replace('^http://localhost:4502', '');
+                }
                 if (match.startsWith('http://') || match.startsWith('https://')) {
                     // Handle full URLs
                     return '' + match + '';
-                } else if (match.startsWith('/content/')) {
+                } else if (match.startsWith('/content/') || match.startsWith('content/')) {
                     // Handle AEM content paths
                     const editorPathMatch = location.pathname.match(/\/[^.]*\.[^/]*/);
                     const editorPath = editorPathMatch ? editorPathMatch[0] : '';
-                    const cleanedMatch = match.replace(/\/jcr:content$/, '').replace(/\.html$/, '');
+                    var cleanedMatch = match.replace(/\/jcr:content$/, '').replace(/\.html$/, '');
+                    if (cleanedMatch.startsWith('content')) {
+                        cleanedMatch = '/' + cleanedMatch;
+                     }
                     return '' + match + '';
                 } else {
                     return match;
diff --git a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
index 36d8f78b1..0fd71912e 100644
--- a/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
+++ b/backend/slingbase/src/main/java/com/composum/ai/backend/slingbase/experimential/impl/SearchPageAITool.java
@@ -110,11 +110,11 @@ public boolean isAllowedFor(@Nonnull Resource resource) {
             List resultPaths = ordered.stream().map(Resource::getPath)
                     .map(path -> path.replaceAll("/jcr:content$", ""))
                     .collect(Collectors.toList());
-            LOG.debug("Search page AI tool found for '{}' : {}", query, resultPaths);
+            LOG.debug("Search page AI tool found for '{}' at {} : {}", query, rootResource.getPath(), resultPaths);
 
             // collect titles (properties "jcr:title" / "title") of resource and make itemized list of markdown links
-            StringBuilder result = new StringBuilder("Here are the JCR paths for the " + config.resultCount() +
-                    " pages best matching the query.\n\n");
+            StringBuilder result = new StringBuilder("Here are the AEM JCR paths for the " + config.resultCount() +
+                    " pages best matching the query. They should be printed as root-relative URLs but can be turned into full URLs by adding them as a suffix after http://localhost:4502/ , but do not print that information. DO NOT change the returned links into full URLs - print them as root-relative URLs starting with /content! They will automatically be translated into full URLs later.\n\n");
             for (String path : resultPaths) {
                 Resource res = resolver.getResource(path);
                 if (res != null) {
@@ -124,7 +124,7 @@ public boolean isAllowedFor(@Nonnull Resource resource) {
                     if (title == null || title.startsWith("/")) {
                         result.append("- ").append(path).append("\n");
                     } else {
-                        result.append("- ").append(title).append(": ").append(path).append("\n");
+                        result.append("- [").append(title).append("](").append(path).append(")\n");
                     }
                 } else {
                     result.append("- ").append(path).append("\n");

From f23eaf1dd7e86fd4f111d76609307846cc745c38 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Fri, 1 Nov 2024 07:38:24 +0100
Subject: [PATCH 29/44] comparetool fix

---
 .../tool/comparetool/comparetool.html         | 185 ++++++++++--------
 .../tool/comparetool/comparetool.md           |  48 +++++
 2 files changed, 149 insertions(+), 84 deletions(-)
 create mode 100644 aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.md

diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
index 0503c415c..b61f530c3 100644
--- a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
@@ -9,11 +9,13 @@
             margin: 0;
             height: 100vh;
         }
+
         .iframe-container {
             width: 50%;
             display: flex;
             flex-direction: column;
         }
+
         .url-field {
             width: 100%;
             box-sizing: border-box;
@@ -21,6 +23,7 @@
             font-size: 16px;
             border: 1px solid #ccc;
         }
+
         iframe {
             width: 100%;
             height: calc(100% - 40px); /* Adjust based on input field height */
@@ -29,98 +32,112 @@
     
 
 
-    
    
-        
-        
-    
    
-    
    
-        
-        
-    
    
-    
+        isLeftScrolling = true;
+        const leftWindow = leftIframe.contentWindow;
+        const rightWindow = rightIframe.contentWindow;
+
+        const scrollPercent = rightWindow.scrollY / (rightWindow.document.documentElement.scrollHeight - rightWindow.innerHeight);
+        leftWindow.scrollTo(0, scrollPercent * (leftWindow.document.documentElement.scrollHeight - leftWindow.innerHeight));
+    }
+
 
 
diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.md b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.md
new file mode 100644
index 000000000..ed545d267
--- /dev/null
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.md
@@ -0,0 +1,48 @@
+# Compare Tool Specification
+
+## Overview
+
+This document outlines the specifications for a single-page HTML application named `comparetool.html`. The application
+will display two web pages side by side, allowing for an efficient comparison.
+
+## Features
+
+1. **Two Iframes Side by Side**: The application will consist of two iframes, each occupying 50% of the browser width
+   and the full height of the window.
+
+2. **URL Fields**: Each iframe will have a dedicated URL field above it for users to input the URLs they want displayed
+   within the iframes.
+
+3. **Synchronized Scrolling**: If one iframe is scrolled, the other iframe will scroll in proportion. For instance, if
+   the left iframe is scrolled to 42%, the right iframe will also scroll to 42%.
+
+4. **Parameters**: the URL parameters passed to the application are:
+   - `url1`: the URL that is initially loaded into the left iframe
+   - `url2`: the URL that is initially loaded into the right iframe
+
+The tool is meant to be run in a desktop browser, no responsive design necessary.
+
+## Implementation Details
+
+- **HTML Structure**: The HTML will consist of two main sections, each containing an iframe and a URL input field.
+- **CSS Styling**: The iframes will be styled to take up 50% of the width and 100% height of the viewport.
+- **JavaScript Functionality**: A script will be implemented to handle the synchronization of scrolling between the two
+  iframes.
+
+## Conclusion
+
+The `comparetool.html` application is designed to provide users with an efficient means of comparing two web pages side
+by side. With synchronized scrolling and dedicated URL fields, users can easily navigate and assess differences between
+the two sources.
+
+## TODOs / ideas
+
+- if url1/url2 are changed, the location needs to be updated (push state!)
+- url1/url2 should set the fields
+- default url1 to blueprint of url2 ; button to do that
+- styling
+- checkbox to break scroll synchronization
+- shortcut to editor
+- bookmarklet to open comparison
+- help texts
+

From e236ba8ef0ff29b9f567a72dd10f3d0167dfecf8 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Fri, 1 Nov 2024 12:32:04 +0100
Subject: [PATCH 30/44] fix autotranslation: force committing delayed
 translations

---
 .../aem/core/impl/autotranslate/AutoTranslateServiceImpl.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateServiceImpl.java b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateServiceImpl.java
index 2eb0fa4ed..8f04d1e9e 100644
--- a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateServiceImpl.java
+++ b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateServiceImpl.java
@@ -131,7 +131,8 @@ public TranslationRun startTranslation(
         TranslationRunImpl run = new TranslationRunImpl();
         run.id = "" + Math.abs(System.nanoTime());
         run.rootPath = path;
-        run.translationParameters = translationParameters;
+        run.translationParameters = translationParameters.clone();
+        run.translationParameters.autoSave = true; // otherwise it'll be just rolled back
         run.translatedPages = resources.stream()
                 .map(r -> new TranslationPageImpl(r.getPath()))
                 .collect(Collectors.toList());

From ae4cc20570e4b83f62d5637e76ac24a68a63991a Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Mon, 4 Nov 2024 15:29:44 +0100
Subject: [PATCH 31/44] translation tool: offer to make a copy

---
 .../autotranslate/AutoTranslateListModel.java | 46 ++++++++++++++++++-
 .../autotranslate-experiments/list/list.html  | 20 +++++++-
 .../components/autotranslate/list/list.html   | 19 +++++++-
 3 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateListModel.java b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateListModel.java
index 44b1c5798..132ba527a 100644
--- a/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateListModel.java
+++ b/aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateListModel.java
@@ -12,6 +12,7 @@
 import org.apache.sling.api.resource.LoginException;
 import org.apache.sling.api.resource.PersistenceException;
 import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceResolver;
 import org.apache.sling.models.annotations.Model;
 import org.apache.sling.models.annotations.injectorspecific.OSGiService;
 import org.apache.sling.models.annotations.injectorspecific.Self;
@@ -19,7 +20,10 @@
 import org.slf4j.LoggerFactory;
 
 import com.composum.ai.backend.slingbase.AIConfigurationService;
+import com.day.cq.wcm.api.Page;
+import com.day.cq.wcm.api.PageManager;
 import com.day.cq.wcm.api.WCMException;
+import com.day.cq.wcm.msm.api.LiveRelationshipManager;
 
 @Model(adaptables = SlingHttpServletRequest.class)
 public class AutoTranslateListModel {
@@ -35,6 +39,9 @@ public class AutoTranslateListModel {
     @OSGiService
     private AutoTranslateConfigService autoTranslateConfigService;
 
+    @OSGiService
+    private LiveRelationshipManager liveRelationshipManager;
+
     @Self
     private SlingHttpServletRequest request;
 
@@ -55,7 +62,7 @@ public boolean inProgress() {
         return runs.stream().filter(run -> run.isInProgress()).findAny().isPresent();
     }
 
-    public AutoTranslateService.TranslationRun createRun() throws LoginException, PersistenceException {
+    public AutoTranslateService.TranslationRun createRun() throws LoginException, PersistenceException, WCMException {
         if (run == null) {
             String path = request.getParameter("path");
             if (path == null || path.isEmpty()) {
@@ -64,6 +71,7 @@ public AutoTranslateService.TranslationRun createRun() throws LoginException, Pe
             path = path.replaceAll("_jcr_content", "jcr:content").replaceAll("\\.html$", "").trim();
             boolean recursive = request.getParameter("recursive") != null;
             boolean changed = request.getParameter("translateWhenChanged") != null;
+            boolean copyOriginalPage = request.getParameter("copyOriginalPage") != null;
             String additionalInstructions = request.getParameter("additionalInstructions");
             boolean debugaddinstructions = request.getParameter("debugaddinstructions") != null;
             if (debugaddinstructions) {
@@ -93,11 +101,47 @@ public AutoTranslateService.TranslationRun createRun() throws LoginException, Pe
             parms.translateWhenChanged = changed;
             parms.additionalInstructions = additionalInstructions;
             parms.breakInheritance = breakInheritance;
+            if (copyOriginalPage) {
+                copyOriginalPage(request, path);
+            }
             run = autoTranslateService.startTranslation(request.getResourceResolver(), path, parms);
         }
         return run;
     }
 
+    /**
+     * If parameter copyOriginalPage is set, we create a copy of the original page with this suffix
+     * before doing the translation.
+     */
+    public static final String SUFFIX_TRANSLATECOPY = "_aitranslate_bak";
+
+    /**
+     * Make a copy of the original page for comparison purposes.
+     */
+    protected void copyOriginalPage(SlingHttpServletRequest request, String path) throws WCMException, PersistenceException {
+        ResourceResolver resolver = request.getResourceResolver();
+        PageManager pageManager = resolver.adaptTo(PageManager.class);
+        Page originalPage = pageManager.getContainingPage(path);
+        path = originalPage.getPath();
+        if (originalPage != null) {
+            String newPath = path + SUFFIX_TRANSLATECOPY;
+            if (resolver.getResource(newPath) != null) {
+                resolver.delete(resolver.getResource(newPath));
+            }
+            Page copy = pageManager.copy(originalPage, newPath, null, true, true, false);
+            if (copy != null) {
+                liveRelationshipManager.endRelationship(copy.getContentResource(), false);
+                LOG.info("Created copy of {} at {}", originalPage.getPath(), newPath);
+                resolver.commit();
+            } else {
+                LOG.error("Failed to create copy of {} at {}", originalPage.getPath(), newPath);
+                throw new IllegalArgumentException("Failed to create copy of " + originalPage.getPageTitle() + " at " + newPath);
+            }
+        } else {
+            throw new IllegalArgumentException("No page exists at " + path);
+        }
+    }
+
     public String rollback() throws WCMException, PersistenceException {
         String path = request.getParameter("path");
         try {
diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/autotranslate-experiments/list/list.html b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/autotranslate-experiments/list/list.html
index e108437d4..379e55962 100644
--- a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/autotranslate-experiments/list/list.html
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/autotranslate-experiments/list/list.html
@@ -87,6 +87,11 @@
                                                 
                                             
                                         
+                                        
    
+                                            
+                                                
+                                            
+                                        
    
                                         
                                         
    
                                             
    
diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
index b61f530c3..cf901ae05 100644
--- a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
@@ -77,8 +77,14 @@
     // Load the initial URLs if available in the parameters
     const initialUrl1 = getParameterByName('url1');
     const initialUrl2 = getParameterByName('url2');
-    if (initialUrl1) leftIframe.src = initialUrl1;
-    if (initialUrl2) rightIframe.src = initialUrl2;
+    if (initialUrl1) {
+        leftIframe.src = initialUrl1;
+        leftField.value = initialUrl1;
+    }
+    if (initialUrl2) {
+        rightIframe.src = initialUrl2;
+        rightField.value = initialUrl2;
+    }
 
     // Load event listeners for iframes
     leftIframe.addEventListener('load', function () {

From d4755b00df0771f86d110c7bf09e64c33d75934f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dr=2E=20Hans-Peter=20St=C3=B6rr?=
 <999184+stoerr@users.noreply.github.com>
Date: Wed, 13 Nov 2024 10:12:47 +0100
Subject: [PATCH 41/44] Fix code scanning alert no. 21: DOM text reinterpreted
 as HTML
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
Signed-off-by: Dr. Hans-Peter Störr <999184+stoerr@users.noreply.github.com>
---
 .../components/tool/comparetool/comparetool.html   | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
index cf901ae05..086eb235a 100644
--- a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
@@ -60,12 +60,22 @@
     // Event listeners for URL input fields
     leftField.addEventListener('change', function () {
         leftLoaded = false;
-        leftIframe.src = this.value;
+        try {
+            const url = new URL(this.value);
+            leftIframe.src = url.href;
+        } catch (e) {
+            console.error('Invalid URL:', this.value);
+        }
     });
 
     rightField.addEventListener('change', function () {
         rightLoaded = false;
-        rightIframe.src = this.value;
+        try {
+            const url = new URL(this.value);
+            rightIframe.src = url.href;
+        } catch (e) {
+            console.error('Invalid URL:', this.value);
+        }
     });
 
     // Function to parse URL parameters

From 678f89c2757ca4c177e329fbc3193d846b2ab6f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dr=2E=20Hans-Peter=20St=C3=B6rr?=
 <999184+stoerr@users.noreply.github.com>
Date: Wed, 13 Nov 2024 10:22:26 +0100
Subject: [PATCH 42/44] Fix code scanning alert no. 18: Client-side cross-site
 scripting
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
Signed-off-by: Dr. Hans-Peter Störr <999184+stoerr@users.noreply.github.com>
---
 .../components/tool/comparetool/comparetool.html         | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
index 086eb235a..f12820b4f 100644
--- a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
@@ -30,6 +30,7 @@
             border: none;
         }
     
+    
 
 
 
    
@@ -62,7 +63,7 @@
         leftLoaded = false;
         try {
             const url = new URL(this.value);
-            leftIframe.src = url.href;
+            leftIframe.src = DOMPurify.sanitize(url.href);
         } catch (e) {
             console.error('Invalid URL:', this.value);
         }
@@ -72,7 +73,7 @@
         rightLoaded = false;
         try {
             const url = new URL(this.value);
-            rightIframe.src = url.href;
+            rightIframe.src = DOMPurify.sanitize(url.href);
         } catch (e) {
             console.error('Invalid URL:', this.value);
         }
@@ -88,11 +89,11 @@
     const initialUrl1 = getParameterByName('url1');
     const initialUrl2 = getParameterByName('url2');
     if (initialUrl1) {
-        leftIframe.src = initialUrl1;
+        leftIframe.src = DOMPurify.sanitize(initialUrl1);
         leftField.value = initialUrl1;
     }
     if (initialUrl2) {
-        rightIframe.src = initialUrl2;
+        rightIframe.src = DOMPurify.sanitize(initialUrl2);
         rightField.value = initialUrl2;
     }
 

From e0612d301fa48632d312990dbce0b85b3f12effc Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Wed, 13 Nov 2024 10:24:55 +0100
Subject: [PATCH 43/44] check integrity of dompurify

---
 .../composum-ai/components/tool/comparetool/comparetool.html    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
index f12820b4f..3aa92c8c3 100644
--- a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/comparetool/comparetool.html
@@ -30,7 +30,7 @@
             border: none;
         }
     
-    
+    
 
 
 
    

From 14c33e93fd16d2734fa86a3f1dc4ddefcc0e1732 Mon Sep 17 00:00:00 2001
From: "Dr. Hans-Peter Stoerr" 
Date: Wed, 13 Nov 2024 10:30:57 +0100
Subject: [PATCH 44/44] one more integrity check

---
 .../components/tool/mergetool/translation-merge-tool-html.htm   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/mergetool/translation-merge-tool-html.htm b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/mergetool/translation-merge-tool-html.htm
index 769df5e32..1ab28ff8d 100644
--- a/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/mergetool/translation-merge-tool-html.htm
+++ b/aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/tool/mergetool/translation-merge-tool-html.htm
@@ -73,7 +73,7 @@ 
    Translation Merge Tool
    
         
    
     
    
 
-    
+