Refactoring, AI Actions, Compare tool (#157)

This implements a couple of smaller and larger features.
- pages are marked with attribute ai_translationError if the translation
fails
- move GPTChatCompletionTemplate out of impl package since that's used
in the API
- add support for tools to the site panel AI: the interface AITool can
be implemented by services that do offer functionality the side panel AI
can call by itself (OpenAI tool calls).
- include two tools that need to be OSGI configured to be available:
- SearchPageAITool performs a search in the current site. That needs a
lucene fulltext indext to be configured, though.
- GetPageMarkdownAITool allows the AI to retrieve the text of a given
page.

TODO.txt

@@ -1,7 +1,14 @@
 # List of minor Todos
 
+?? Default Translation Configuration in AEM?
+
 https://wcm.io/caconfig/editor/usage.html translation config drop down
 
+Generated with ... in conf pages
+http://localhost:4502/editor.html/content/dam/wknd/en/magazine/arctic-surfing/aloha-spirits-in-northern-norway doesnt add page text or component text!!!
+
+- default icon for ai.composum site
+!!! Dictation in Chat repeat in history bar.
 !! Empty prompt text
 !! http://localhost:5502/editor.html/content/xxx/com/en/about-us/going-forward.html "Explore the world..." RTE
 PDF and markdown as possible datatypes for the content creation dialog
@@ -13,6 +20,8 @@ Why are the teasers in
 https://author-p43852-e197429.adobeaemcloud.com/editor.html/content/gfps/com/en/products-solutions/systems/primofit.html
 not included into the excerpt.
 
+? Add history in content creation dialog, current component, last text, ...
+
 ----------------
 
 - Marker on page whenever a translation is in progress

aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AITranslatePropertyWrapper.java

@@ -11,7 +11,7 @@
 import org.apache.sling.api.resource.ModifiableValueMap;
 import org.apache.sling.api.resource.ValueMap;
 
-class AITranslatePropertyWrapper {
+public class AITranslatePropertyWrapper {
 
     /**
      * PageContent only property: saves the additional instructions the page was translated with.
@@ -62,6 +62,12 @@ class AITranslatePropertyWrapper {
      */
     public static final String AI_MANUAL_CHANGE_SUFFIX = "_manualChange";
 
+    /**
+     * Attribute that is set on jcr:content of a page when the translation of a page failed, to make it easy to find such pages. Not set by {@link AITranslatePropertyWrapper}, but since all property names are defined here...
+     * Is set to the time at which the error occurred, to make it easy to find in the logs.
+     */
+    public static final String AI_TRANSLATION_ERRORMARKER = "ai_translationError";
+
     private final ModifiableValueMap targetValueMap;
     private final String propertyName;
     private final ValueMap sourceValueMap;

aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoPageTranslateServiceImpl.java

@@ -1,5 +1,6 @@
 package com.composum.ai.aem.core.impl.autotranslate;
 
+import static com.composum.ai.aem.core.impl.autotranslate.AITranslatePropertyWrapper.AI_TRANSLATION_ERRORMARKER;
 import static com.composum.ai.backend.base.service.chat.impl.GPTTranslationServiceImpl.LASTID;
 import static com.composum.ai.backend.base.service.chat.impl.GPTTranslationServiceImpl.MULTITRANSLATION_SEPARATOR_END;
 import static com.composum.ai.backend.base.service.chat.impl.GPTTranslationServiceImpl.MULTITRANSLATION_SEPARATOR_START;
@@ -131,9 +132,23 @@ public Stats translateLiveCopy(@Nonnull Resource resource,
             if (translationParameters.rules != null) {
                 allRules.addAll(translationParameters.rules);
             }
+
             if (autoTranslateCaConfig.rules() != null) {
                 allRules.addAll(Arrays.asList(autoTranslateCaConfig.rules()));
             }
+            if (autoTranslateCaConfig.temperature() != null && !autoTranslateCaConfig.temperature().trim().isEmpty()) {
+                try {
+                    double temperature = Double.parseDouble(autoTranslateCaConfig.temperature());
+                    configuration = GPTConfiguration.ofTemperature(temperature).merge(configuration);
+                } catch (NumberFormatException e) {
+                    LOG.error("Invalid temperature value {} for path {}", autoTranslateCaConfig.temperature(), resource.getPath());
+                }
+            }
+            if (autoTranslateCaConfig.preferHighIntelligenceModel()) {
+                configuration = GPTConfiguration.HIGH_INTELLIGENCE.merge(configuration);
+            } else if (autoTranslateCaConfig.preferStandardModel()) {
+                configuration = GPTConfiguration.STANDARD_INTELLIGENCE.merge(configuration);
+            }
 
             // collect translation rules that apply
             List<PropertyToTranslate> allTranslateableProperties = new ArrayList<>();
@@ -364,9 +379,6 @@ protected String remapPaths(String translatedValue, String blueprintPath, String
         Pattern pattern = Pattern.compile("href=\"" +
                 Pattern.quote(blueprintPath) + "(/[^\"]*)\"");
         String result = pattern.matcher(translatedValue).replaceAll("href=\"" + livecopyPath + "$1\"");
-        if (translatedValue.contains("href")) { // FIXME(hps,24/10/03) no checkin
-            LOG.trace("Remapping paths from {} to {} in {}", blueprintPath, livecopyPath, translatedValue);
-        }
         return result;
     }
 
@@ -392,6 +404,7 @@ protected void markAsAiTranslated(Resource resource, LiveRelationship liveRelati
             liveRelationshipManager.cancelPropertyRelationship(resource.getResourceResolver(),
                     liveRelationship, targetWrapper.allGeneralKeys(), false);
         }
+        valueMap.put(AI_TRANSLATION_ERRORMARKER, Boolean.FALSE); // reset error marker if there was one.
     }
 
     /**
@@ -581,10 +594,6 @@ protected boolean collectPropertiesToTranslate(
             stats.translateableProperties++;
             AITranslatePropertyWrapper targetWrapper = new AITranslatePropertyWrapper(sourceValueMap, targetValueMap, key);
 
-            if (StringUtils.contains(targetWrapper.getOriginalCopy(), "href")) { // FIXME(hps,24/10/03) no checkin
-                LOG.trace("Skipping {} in {} because it contains href", key, resource.getPath());
-            }
-
             // we will translate except if the property is cancelled and we don't want to touch cancelled properties,
             // or if we have a current translation.
             boolean isCancelled = isCancelled(resource, key, relationship);
@@ -621,6 +630,10 @@ protected boolean collectPropertiesToTranslate(
             propertyToTranslate.propertyName = key;
             propertyToTranslate.isAlreadyCorrectlyTranslated = isAlreadyCorrectlyTranslated;
             propertiesToTranslate.add(propertyToTranslate);
+
+            if (targetWrapper.getOriginal().contains("THROWUPRIGHTNOW49e43jwsdsg")) {
+                throw new IllegalStateException("THROWUPRIGHTNOW49e43jwsdsg requested for " + sourceResource.getPath());
+            }
         }
         for (Resource child : resource.getChildren()) {
             if (!PATTERN_IGNORED_SUBNODE_NAMES.matcher(child.getName()).matches()) {

aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateCaConfig.java

@@ -9,8 +9,8 @@
 // is also added to Sling-ContextAware-Configuration-Classes bnd header in pom.xml
 public @interface AutoTranslateCaConfig {
 
-    @Property(label = "Additional Instructions", order = 1,
-            description = "Additional instructions for the automatic translation.")
+    @Property(label = "Additional Instructions (Deprecated)", order = 1,
+            description = "Additional instructions for the automatic translation. Deprecated, please use 'Rules for additional Instructions' instead - if you do not give a path regex nor a content pattern the instructions will be used everywhere.")
     String additionalInstructions();
 
     @Property(label = "Rules for additional Instructions", order = 2,
@@ -57,4 +57,16 @@
             })
     String includeExistingTranslationsInRetranslation();
 
+    @Property(label = "Optional Comment (for documentation, not used by AI)", order = 7,
+            description = "An optional comment about the configuration, for documentation purposes (not used by the translation).",
+            property = {
+                    "widgetType=textarea",
+                    "textareaRows=2"
+            })
+    String comment();
+
+    @Property(label = "Temperature", order = 8,
+            description = "Optional temperature setting that determines variability and creativity as a floating point between 0.0 and 1.0")
+    String temperature();
+
 }

aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateListModel.java

@@ -12,14 +12,18 @@
 import org.apache.sling.api.resource.LoginException;
 import org.apache.sling.api.resource.PersistenceException;
 import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceResolver;
 import org.apache.sling.models.annotations.Model;
 import org.apache.sling.models.annotations.injectorspecific.OSGiService;
 import org.apache.sling.models.annotations.injectorspecific.Self;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.composum.ai.backend.slingbase.AIConfigurationService;
+import com.day.cq.wcm.api.Page;
+import com.day.cq.wcm.api.PageManager;
 import com.day.cq.wcm.api.WCMException;
+import com.day.cq.wcm.msm.api.LiveRelationshipManager;
 
 @Model(adaptables = SlingHttpServletRequest.class)
 public class AutoTranslateListModel {
@@ -35,6 +39,9 @@ public class AutoTranslateListModel {
     @OSGiService
     private AutoTranslateConfigService autoTranslateConfigService;
 
+    @OSGiService
+    private LiveRelationshipManager liveRelationshipManager;
+
     @Self
     private SlingHttpServletRequest request;
 
@@ -55,7 +62,7 @@ public boolean inProgress() {
         return runs.stream().filter(run -> run.isInProgress()).findAny().isPresent();
     }
 
-    public AutoTranslateService.TranslationRun createRun() throws LoginException, PersistenceException {
+    public AutoTranslateService.TranslationRun createRun() throws LoginException, PersistenceException, WCMException {
         if (run == null) {
             String path = request.getParameter("path");
             if (path == null || path.isEmpty()) {
@@ -64,6 +71,7 @@ public AutoTranslateService.TranslationRun createRun() throws LoginException, Pe
             path = path.replaceAll("_jcr_content", "jcr:content").replaceAll("\\.html$", "").trim();
             boolean recursive = request.getParameter("recursive") != null;
             boolean changed = request.getParameter("translateWhenChanged") != null;
+            boolean copyOriginalPage = request.getParameter("copyOriginalPage") != null;
             String additionalInstructions = request.getParameter("additionalInstructions");
             boolean debugaddinstructions = request.getParameter("debugaddinstructions") != null;
             if (debugaddinstructions) {
@@ -93,11 +101,48 @@ public AutoTranslateService.TranslationRun createRun() throws LoginException, Pe
             parms.translateWhenChanged = changed;
             parms.additionalInstructions = additionalInstructions;
             parms.breakInheritance = breakInheritance;
+            if (copyOriginalPage) {
+                copyOriginalPage(request, path);
+            }
             run = autoTranslateService.startTranslation(request.getResourceResolver(), path, parms);
         }
         return run;
     }
 
+    /**
+     * If parameter copyOriginalPage is set, we create a copy of the original page with this suffix
+     * before doing the translation.
+     */
+    public static final String SUFFIX_TRANSLATECOPY = "_aitranslate_bak";
+
+    /**
+     * Make a copy of the original page for comparison purposes.
+     */
+    protected void copyOriginalPage(SlingHttpServletRequest request, String path) throws WCMException, PersistenceException {
+        ResourceResolver resolver = request.getResourceResolver();
+        PageManager pageManager = resolver.adaptTo(PageManager.class);
+        Page originalPage = pageManager.getContainingPage(path);
+        path = originalPage.getPath();
+        if (originalPage != null) {
+            String newPath = path + SUFFIX_TRANSLATECOPY;
+            if (resolver.getResource(newPath) != null) {
+                resolver.delete(resolver.getResource(newPath));
+            }
+            Page copy = pageManager.copy(originalPage, newPath, null, true, true, false);
+            if (copy != null) {
+                liveRelationshipManager.endRelationship(copy.getContentResource(), true);
+                liveRelationshipManager.detach(copy.getContentResource(), true); // end doesn't seem to work
+                LOG.info("Created copy of {} at {}", originalPage.getPath(), newPath);
+                resolver.commit();
+            } else {
+                LOG.error("Failed to create copy of {} at {}", originalPage.getPath(), newPath);
+                throw new IllegalArgumentException("Failed to create copy of " + originalPage.getPageTitle() + " at " + newPath);
+            }
+        } else {
+            throw new IllegalArgumentException("No page exists at " + path);
+        }
+    }
+
     public String rollback() throws WCMException, PersistenceException {
         String path = request.getParameter("path");
         try {

aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateRuleConfig.java

@@ -10,7 +10,7 @@
 public @interface AutoTranslateRuleConfig {
 
     @Property(label = "Path Regex", order = 1,
-            description = "A regular expression matching the absolute path to the page. " +
+            description = "A regular expression matching the absolute path to the page, incl. jcr:content. " +
                     "E.g. .*/home/products/.* will match all pages under .../home/products/. If empty every page will match" +
                     "if the content pattern condition is met.")
     String pathRegex();
@@ -30,4 +30,12 @@
             })
     String additionalInstructions();
 
+    @Property(label = "Optional Comment (for documentation, not used by AI)", order = 4,
+            description = "An optional comment for the rule, for documentation purposes (not used by the translation).",
+            property = {
+                    "widgetType=textarea",
+                    "textareaRows=2"
+            })
+    String comment();
+
 }

aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateService.java

@@ -159,6 +159,8 @@ abstract class TranslationPage {
                 Pattern.compile("\\.(png|jpg|jpeg|gif|svg|mp3|mov|mp4)(/|$)", Pattern.CASE_INSENSITIVE);
 
         public String pagePath;
+        public String translateCopyPagePath;
+
         public String status;
         public AutoPageTranslateService.Stats stats;
 
@@ -172,6 +174,17 @@ public String editorUrl() {
                 return "/editor.html" + pagePath + ".html";
             }
         }
+
+        /** If a translate copy is present, this would open a diff view. */
+        public String diffToCopyUrl() {
+            if (startsWith(pagePath, "/content/dam") || translateCopyPagePath == null) {
+                return null;
+            } else {
+                return "/mnt/overlay/wcm/core/content/sites/diffresources.html" + pagePath +
+                        "?item=" + translateCopyPagePath + "&sideBySide";
+            }
+        }
+
     }
 
 }

aem/core/src/main/java/com/composum/ai/aem/core/impl/autotranslate/AutoTranslateServiceImpl.java

@@ -1,6 +1,10 @@
 package com.composum.ai.aem.core.impl.autotranslate;
 
+import static com.composum.ai.aem.core.impl.autotranslate.AITranslatePropertyWrapper.AI_TRANSLATION_ERRORMARKER;
+import static org.apache.commons.lang3.StringUtils.startsWith;
+
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
@@ -10,6 +14,7 @@
 import javax.annotation.Nonnull;
 
 import org.apache.sling.api.resource.LoginException;
+import org.apache.sling.api.resource.ModifiableValueMap;
 import org.apache.sling.api.resource.PersistenceException;
 import org.apache.sling.api.resource.Resource;
 import org.apache.sling.api.resource.ResourceResolver;
@@ -131,9 +136,10 @@ public TranslationRun startTranslation(
         TranslationRunImpl run = new TranslationRunImpl();
         run.id = "" + Math.abs(System.nanoTime());
         run.rootPath = path;
-        run.translationParameters = translationParameters;
+        run.translationParameters = translationParameters.clone();
+        run.translationParameters.autoSave = true; // otherwise it'll be just rolled back
         run.translatedPages = resources.stream()
-                .map(r -> new TranslationPageImpl(r.getPath()))
+                .map(r -> new TranslationPageImpl(r))
                 .collect(Collectors.toList());
         run.waituntil = System.currentTimeMillis() + 1000; // when triggered during live copy creation.
         run.status = TranslationStatus.QUEUED;
@@ -144,7 +150,7 @@ public TranslationRun startTranslation(
     }
 
     protected List<Resource> collectPages(Resource root, int maxDepth) {
-        if (maxDepth < 0) {
+        if (maxDepth < 0 || root.getName().endsWith(AutoTranslateListModel.SUFFIX_TRANSLATECOPY)) {
             return Collections.emptyList();
         }
         if (root.getPath().contains("/jcr:content")) {
@@ -233,10 +239,21 @@ public void execute(ResourceResolver callResourceResolver) {
                         resourceResolver.revert();
                         resourceResolver.refresh();
                         Resource resource = resourceResolver.getResource(page.resourcePath);
-                        if (resource != null) {
-                            AutoPageTranslateService.Stats stats = pageTranslateService.translateLiveCopy(resource, translationParameters);
-                            page.stats = stats;
-                            page.status = stats.hasChanges() ? "done" : "unchanged";
+                        try {
+                            if (resource != null) {
+                                AutoPageTranslateService.Stats stats = pageTranslateService.translateLiveCopy(resource, translationParameters);
+                                page.stats = stats;
+                                page.status = stats.hasChanges() ? "done" : "unchanged";
+                            }
+                        } catch (GPTException.GPTUserNotificationException e) {
+                            throw e;
+                        } catch (Exception e) {
+                            resourceResolver.revert();
+                            resourceResolver.refresh();
+                            // mark translation as failed.
+                            resource.adaptTo(ModifiableValueMap.class).put(AI_TRANSLATION_ERRORMARKER, Calendar.getInstance());
+                            resourceResolver.commit();
+                            throw e;
                         }
                     } catch (GPTException.GPTUserNotificationException e) {
                         page.status = "cancelled - user notification";
@@ -273,11 +290,15 @@ public void execute(ResourceResolver callResourceResolver) {
     public static class TranslationPageImpl extends TranslationPage {
         String resourcePath;
 
-        public TranslationPageImpl(String resourcePath) {
-            this.resourcePath = resourcePath;
+        public TranslationPageImpl(Resource resource) {
+            this.resourcePath = resource.getPath();
             pagePath = ResourceUtil.getParent(resourcePath); // remove jcr:content
             this.status = "queued";
+            Resource translateCopyResource = resource.getParent().getParent()
+                    .getChild(resource.getParent().getName() + AutoTranslateListModel.SUFFIX_TRANSLATECOPY);
+            translateCopyPagePath = translateCopyResource != null ? translateCopyResource.getPath() : null;
         }
+
     }
 
 }