Feature/22 rework approximate markdown servlet (#29)

- reverse heuristical strategy from JCR to print attributes except if
(heuristically) forbidden, to better handle custom components
- introduce plugin to create markdown from the actual HTML rendering and
use that as new default (OSGI configurable)
- extend html to markdown converter with many HTML tags to allow good
markdown from component renderings

.chatgpt.codeRules.md

@@ -25,4 +25,5 @@
   action after making changes if the changes are complete in the sense that the tests should work.
 - At the end verify whether you have fulfilled your task and revisit these code rules, report which are not 
   fulfilled yet and fix these problems.
-- Always complain if something is incorrect, contradictory or incomplete.
+- Always point out when something is incorrect, contradictory or incomplete.
+- You do not need to ask for permission to read files you'd need, just do it!

.chatgpt.intro.md

@@ -1,5 +1,7 @@
-List all files except ui.content.sample using the plugin. Read the .chatgpt.codeRules.md and observe the rules that 
+Read the .chatgpt.codeRules.md and observe the rules that 
 are laid out there during this whole chat, but you don't need to repeat them.
 
+List all files except ui.content.sample using the plugin. 
+
 Use the grep operation to search for comments starting with `ChatGPTTask:` and execute them. The comment should be
 replaced by the new code or deleted when it's completely done.

NextSteps.md

@@ -1,6 +1,5 @@
 # Next steps in the implementation
 
-- Update AEM dialog with history
 - AEM announcement and installation and usage description
 - AEM 6.5 version
 - Update Pages content creation dialog with separate content field like AEM
@@ -81,3 +80,4 @@
 - DONE: use library to count the tokens -> JTokkit
 - DONE: save scroll position, scroll the chat field to the top.
 - DONE Somehow implement streaming to make result more responsive.
+- DONE: Update AEM dialog with history

TODO.txt

@@ -1,14 +1,13 @@
-Restrictions
+Restrictions : ??? component
 Announcement, Installation, Doc überarbeiten
-Refactor Aprrox Markdown for HTML
 AEM for content fragments?
 
 Done:
 Markdown to HTML transformation in sidebar -> not necessary, white-space: break-spaces is enough.
 implement AEM Historie -> DONE
-Help-Pages -> done for creation assistant; for the side panel ai the help buttons and the initial text should be enough for now.
-Enter in Composum Variante
-Tell ChatGPT when we expect HTML.
+Help-Pages -> DONE for creation assistant; for the side panel ai the help buttons and the initial text should be enough for now.
+Enter in Composum Variante -> DONE
+Tell ChatGPT when we expect HTML. -> DONE
 
 Deferred:
 AEM 6.5

aem/core/src/main/java/com/composum/ai/aem/core/impl/AemApproximateMarkdownServicePlugin.java

@@ -16,8 +16,11 @@
 import javax.annotation.Nullable;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.sling.api.SlingHttpServletRequest;
+import org.apache.sling.api.SlingHttpServletResponse;
 import org.apache.sling.api.resource.Resource;
 import org.apache.sling.api.resource.ValueMap;
+import org.osgi.framework.Constants;
 import org.osgi.service.component.annotations.Component;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -30,7 +33,10 @@
 /**
  * Special handling for cq:PageContent and components
  */
-@Component(service = ApproximateMarkdownServicePlugin.class)
+@Component(service = ApproximateMarkdownServicePlugin.class,
+        // lower priority than HtmlToApproximateMarkdownServicePlugin since that does do a better job on experience fragments / content fragments if enabled
+        property = Constants.SERVICE_RANKING + ":Integer=2000"
+)
 public class AemApproximateMarkdownServicePlugin implements ApproximateMarkdownServicePlugin {
 
     private static final Logger LOG = LoggerFactory.getLogger(AemApproximateMarkdownServicePlugin.class);
@@ -47,14 +53,18 @@ public class AemApproximateMarkdownServicePlugin implements ApproximateMarkdownS
     protected static final Pattern CONTENTFRAGMENT_TYPES = Pattern.compile("core/wcm/components/contentfragment/v./contentfragment");
 
     @Override
-    public @Nonnull PluginResult maybeHandle(@Nonnull Resource resource, @Nonnull PrintWriter out, @Nonnull ApproximateMarkdownService service) {
+    public @Nonnull PluginResult maybeHandle(
+            @Nonnull Resource resource, @Nonnull PrintWriter out,
+            @Nonnull ApproximateMarkdownService service,
+            @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
         if (resourceRendersAsComponentMatching(resource, FULLY_IGNORED_TYPES)) {
             return PluginResult.HANDLED_ALL;
         }
         if (pageHandling(resource, out, service)) {
             return PluginResult.HANDLED_ATTRIBUTES;
         }
-        if (handleTeaser(resource, out, service) || handleExperienceFragment(resource, out, service) || handleContentFragment(resource, out, service)) {
+        if (handleTeaser(resource, out, service) || handleExperienceFragment(resource, out, service, request, response)
+                || handleContentFragment(resource, out, service)) {
             return PluginResult.HANDLED_ALL;
         }
         return PluginResult.NOT_HANDLED;
@@ -78,6 +88,7 @@ protected boolean pageHandling(Resource resource, PrintWriter out, @Nonnull Appr
             }
             outputIfNotBlank(out, vm, "shortDescription", service);
             outputIfNotBlank(out, vm, JCR_DESCRIPTION, service);
+            out.println();
         }
         return isPage;
     }
@@ -153,7 +164,8 @@ private void outputIfNotBlank(@Nonnull PrintWriter out, @Nonnull ValueMap vm, @N
      *
      * @see "https://github.com/adobe/aem-core-wcm-components/blob/main/content/src/content/jcr_root/apps/core/wcm/components/experiencefragment/v2/experiencefragment/README.md"
      */
-    protected boolean handleExperienceFragment(Resource resource, PrintWriter out, ApproximateMarkdownService service) {
+    protected boolean handleExperienceFragment(Resource resource, PrintWriter out, ApproximateMarkdownService service,
+                                               SlingHttpServletRequest request, SlingHttpServletResponse response) {
         if (resourceRendersAsComponentMatching(resource, EXPERIENCEFRAGMENT_TYPES)) {
             String reference = resource.getValueMap().get("fragmentVariationPath", String.class);
             if (StringUtils.startsWith(reference, "/content/")) {
@@ -165,7 +177,7 @@ protected boolean handleExperienceFragment(Resource resource, PrintWriter out, A
                             referencedResource = referencedResource.getChild("root");
                         }
                     }
-                    service.approximateMarkdown(referencedResource, out);
+                    service.approximateMarkdown(referencedResource, out, request, response);
                 } else {
                     LOG.warn("Resource {} referenced from {} attribute {} not found.", reference, resource.getPath(), "fragmentVariationPath");
                 }
@@ -202,7 +214,9 @@ protected boolean handleContentFragment(Resource resource, PrintWriter out, Appr
         return false;
     }
 
-    private void renderReferencedContentFragment(Resource resource, PrintWriter out, ApproximateMarkdownService service, Resource referencedResource, String variation, String reference, String[] elementNames) {
+    private void renderReferencedContentFragment(
+            Resource resource, PrintWriter out, ApproximateMarkdownService service,
+            Resource referencedResource, String variation, String reference, String[] elementNames) {
         Resource dataNode = referencedResource.getChild("jcr:content/data");
         String title = referencedResource.getValueMap().get("jcr:content/jcr:title", String.class);
         if (StringUtils.isNotBlank(title)) {

aem/core/src/test/java/com/composum/ai/aem/core/impl/AemApproximateMarkdownServicePluginTest.java

@@ -7,6 +7,7 @@
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
 
 import java.io.ByteArrayInputStream;
 import java.io.PrintWriter;
@@ -16,12 +17,15 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.sling.api.SlingHttpServletRequest;
+import org.apache.sling.api.SlingHttpServletResponse;
 import org.apache.sling.api.resource.ModifiableValueMap;
 import org.apache.sling.api.resource.Resource;
 import org.apache.sling.testing.mock.sling.ResourceResolverType;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mockito;
 
 import com.composum.ai.backend.base.service.chat.GPTChatCompletionService;
 import com.composum.ai.backend.slingbase.impl.ApproximateMarkdownServiceImpl;
@@ -37,21 +41,28 @@
 public class AemApproximateMarkdownServicePluginTest {
 
     private ApproximateMarkdownServiceImpl service;
+    private SlingHttpServletRequest request = Mockito.mock(SlingHttpServletRequest.class);
+    private SlingHttpServletResponse response = Mockito.mock(SlingHttpServletResponse.class);
     private Resource component;
     private StringWriter writer;
     private PrintWriter printWriter;
 
     private AemContext context;
+    private ApproximateMarkdownServiceImpl.Config config;
 
     @BeforeEach
     public void setUp() {
         context = new AemContext(ResourceResolverType.JCR_MOCK);
+        config = mock(ApproximateMarkdownServiceImpl.Config.class,
+                withSettings().defaultAnswer(invocation -> invocation.getMethod().getDefaultValue()));
+        when(config.labelledAttributeOrder()).thenReturn(new String[]{"thefirst", "asecond"});
         service = new ApproximateMarkdownServiceImpl() {
             {
                 chatCompletionService = mock(GPTChatCompletionService.class);
                 plugins = Collections.singletonList(new AemApproximateMarkdownServicePlugin());
                 when(chatCompletionService.htmlToMarkdown(anyString()))
                         .then(invocation -> "markdownOf(" + invocation.getArgument(0) + ")");
+                this.activate(config);
             }
         };
         writer = new StringWriter();
@@ -61,7 +72,7 @@ public void setUp() {
     @Test
     public void testPageHandlingWithNonPageResource() {
         component = createMockResource("not/page", new HashMap<>());
-        service.approximateMarkdown(component, printWriter);
+        service.approximateMarkdown(component, printWriter, request, response);
         assertEquals("", writer.toString());
     }
 
@@ -72,10 +83,10 @@ public void testPageHandlingWithPageResource() {
                         "jcr:description", "The best page!",
                         "category", "test, dummy"));
 
-        service.approximateMarkdown(component, printWriter);
+        service.approximateMarkdown(component, printWriter, request, response);
         String expectedOutput =
                 "# myPage\n\n" +
-                        "The best page!\n";
+                        "The best page!\n\n";
         assertThat(writer.toString(), is(expectedOutput));
     }
 
@@ -117,7 +128,7 @@ public void testTeaser() {
                 "    \"sling:resourceType\": \"wknd/components/page\"\n" +
                 "  }\n" +
                 "}");
-        service.approximateMarkdown(teaser, printWriter);
+        service.approximateMarkdown(teaser, printWriter, request, response);
         String expectedOutput = "Downhill Skiing Wyoming\n" +
                 "markdownOf(<p>A skiers paradise far from crowds and close to nature with terrain so vast it appears uncharted.</p>\n" +
                 ")\n" +
@@ -135,7 +146,7 @@ public void testExperienceFragment() {
         component = createMockResource("core/wcm/components/experiencefragment/v1/experiencefragment",
                 ImmutableMap.of("fragmentVariationPath", "/content/experience-fragments/foo/master"));
 
-        service.approximateMarkdown(component, printWriter);
+        service.approximateMarkdown(component, printWriter, request, response);
         String expectedOutput = "## thetitle\n\n";
         assertEquals(expectedOutput, writer.toString());
     }
@@ -149,7 +160,7 @@ public void testContentFragment() {
                 ImmutableMap.of("fragmentPath", "/content/dam/cf/foo",
                         "variationName", "variation", "elementNames", new String[]{"a", "b"}));
 
-        service.approximateMarkdown(component, printWriter);
+        service.approximateMarkdown(component, printWriter, request, response);
         String expectedOutput = "theA\n" +
                 "markdownOf(<p>theB</p>)\n";
         assertEquals(expectedOutput, writer.toString());
@@ -185,7 +196,7 @@ public void testContentFragmentAllElements() {
         component = createMockResource("core/wcm/components/contentfragment/v1/contentfragment",
                 ImmutableMap.of("fragmentPath", "/content/dam/cf/foo"));
 
-        service.approximateMarkdown(component, printWriter);
+        service.approximateMarkdown(component, printWriter, request, response);
         String expectedOutput = "An B: theB\n" +
                 "An A: theA\n";
         assertEquals(expectedOutput, writer.toString());

aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/contentcreation/_cq_dialog/.content.xml

@@ -14,6 +14,8 @@
         <items jcr:primaryType="nt:unstructured" sling:resourceType="nt:unstructured">
             <promptColumns
                     jcr:primaryType="nt:unstructured"
+                    maximized="{Boolean}true"
+                    granite:class="composum-ai-prompt-columns"
                     sling:resourceType="granite/ui/components/coral/foundation/fixedcolumns">
                 <items jcr:primaryType="nt:unstructured">
                     <container

aem/ui.config/src/main/content/jcr_root/apps/composum-ai/osgiconfig/config.author/com.composum.ai.backend.slingbase.impl.HtmlToApproximateMarkdownServicePlugin.cfg.json

@@ -0,0 +1,7 @@
+{
+  "allowedResourceTypes": [
+    ".*"
+  ],
+  "deniedResourceTypes": [
+  ]
+}

aem/ui.content/src/test/content/jcr_root/content/wknd/language-masters/test/composum-ai-testpages/.content.xml

@@ -25,11 +25,16 @@
                 <text
                         jcr:primaryType="nt:unstructured"
                         sling:resourceType="wknd/components/text"
-                        text="&lt;p>Text before&lt;/p>&#xd;&#xa;"
+                        text="&lt;p style=&quot;text-align: left;&quot;>Normal&amp;nbsp;&lt;b>bold&lt;/b>&amp;nbsp;&lt;i>italic&lt;/i>&amp;nbsp;&lt;u>underlined&lt;/u>&amp;nbsp;text&lt;/p>&#xd;&#xa;&lt;table cellpadding=&quot;1&quot; cellspacing=&quot;0&quot; border=&quot;1&quot; width=&quot;128&quot; height=&quot;36&quot;>&#xd;&#xa;&lt;caption>The caption&lt;/caption>&#xd;&#xa;&lt;tbody>&lt;tr>&lt;th scope=&quot;col&quot;>head1&lt;/th>&#xd;&#xa;&lt;th scope=&quot;col&quot;>head2&lt;/th>&#xd;&#xa;&lt;th scope=&quot;col&quot;>head3&lt;/th>&#xd;&#xa;&lt;/tr>&lt;tr>&lt;th scope=&quot;row&quot;>row1&lt;/th>&#xd;&#xa;&lt;td>row2&lt;/td>&#xd;&#xa;&lt;td>row3&lt;/td>&#xd;&#xa;&lt;/tr>&lt;/tbody>&lt;/table>&#xd;&#xa;&lt;p style=&quot;text-align: left;&quot;>Left aligned&lt;/p>&#xd;&#xa;&lt;p style=&quot;text-align: center;&quot;>centered&lt;/p>&#xd;&#xa;&lt;p style=&quot;text-align: right;&quot;>right aligned&lt;/p>&#xd;&#xa;&lt;p style=&quot;text-align: justify;&quot;>This is justified.&amp;nbsp;&lt;a title=&quot;alternative text&quot; href=&quot;https://www.example.net&quot; target=&quot;_blank&quot; rel=&quot;noopener noreferrer&quot;>This is justified.&lt;/a>&amp;nbsp;This is justified.&amp;nbsp;&lt;a href=&quot;/content/wknd/us/en/magazine.html&quot;>This is justified.&lt;/a>&amp;nbsp;This is justified.&amp;nbsp;This is justified.&amp;nbsp;This is justified.&amp;nbsp;This is justified.&amp;nbsp;This is justified. &amp;nbsp;&lt;/p>&#xd;&#xa;&lt;h1>Heading 1&lt;/h1>&#xd;&#xa;&lt;h6>Heading 6&lt;/h6>&#xd;&#xa;&lt;blockquote>a quoted quote quoting quotations let us see whether that is a block quote and what happens if it has several lines because that's interesting&lt;/blockquote>&#xd;&#xa;&lt;hr>&#xd;&#xa;&#xd;&#xa;&lt;p>&amp;nbsp;&lt;/p>&#xd;&#xa;&lt;pre>&#xd;&#xa;Preformatted text&#xd;&#xa;&lt;/pre>&#xd;&#xa;&lt;p>&amp;nbsp;&lt;/p>&#xd;&#xa;&lt;p style=&quot;text-align: center;&quot;>&amp;nbsp;&lt;/p>&#xd;&#xa;"
                         textIsRich="true"/>
+                <title
+                        jcr:primaryType="nt:unstructured"
+                        sling:resourceType="wknd/components/title"
+                        jcr:title="Composum AI Testpage"/>
                 <testdialog
                         jcr:primaryType="nt:unstructured"
                         sling:resourceType="composum-ai/test/components/testdialog"
+                        text="Thisisatext"
                         textIsRich="true"
                         therichText="&lt;p>Give a little bit of &lt;b>rich&lt;/b> &lt;i>text&lt;/i>&lt;/p>&lt;ul>&lt;li>he&lt;/li>&lt;li>re&lt;/li>&lt;/ul>"
                         thetext="This is, not surprising, some plain text which we have here.&#xd;&#xa;Another line of plain text."
@@ -56,7 +61,8 @@
                 <experiencefragment
                         jcr:primaryType="nt:unstructured"
                         sling:resourceType="wknd/components/experiencefragment"
-                        fragmentVariationPath="/content/experience-fragments/wknd/us/en/adventures/adventures-2021/master"/>
+                        fragmentVariationPath="/content/experience-fragments/wknd/us/en/adventures/adventures-2021/master"
+                        text="New Adventures for 2021 Go somewhere incredible next year. This past year was challenging on a number of levels but we've got your back. We've made several changes and improvements to all the adventures to make them safer, more flexible and as stress-free as possible. All adventures offer a no-hassle cancellation, fully refundable, no questions asked. New! Bali Surf Camp Surfing in Bali is on the bucket list of every surfer - whether you're a beginner or someone who's been surfing for decades, there will be a break to cater to your ability. Bali offers warm water, tropical vibes, awesome breaks and low cost expenses. Bali Surf Camp"/>
             </container>
         </root>
     </jcr:content>

aem/ui.frontend/src/main/webpack/site/ContentCreationDialog.js

@@ -5,7 +5,7 @@ import {errorText, findSingleElement} from './common.js';
 import {DialogHistory} from './DialogHistory.js';
 import {HelpPage} from './HelpPage.js';
 
-const APPROXIMATE_MARKDOWN_SERVLET = '/bin/cpm/ai/approximated.markdown.md';
+const APPROXIMATED_MARKDOWN_SERVLET = '/bin/cpm/ai/approximated';
 
 /** Keeps dialog histories per path. */
 const historyMap = {};
@@ -70,6 +70,7 @@ class ContentCreationDialog {
     fullscreen() {
         this.$dialog.find('form').addClass('_coral-Dialog--fullscreenTakeover');
         this.$dialog.find('coral-dialog-footer').children().appendTo(this.$dialog.find('coral-dialog-header div.cq-dialog-actions'));
+        this.$dialog.find('.composum-ai-prompt-columns .u-coral-padding').removeClass('u-coral-padding');
     }
 
     removeFormAction() {
@@ -232,8 +233,9 @@ class ContentCreationDialog {
 
     retrieveValue(path, callback) {
         $.ajax({
-            url: Granite.HTTP.externalize(APPROXIMATE_MARKDOWN_SERVLET + path
-                + "?richtext=" + this.isrichtext
+            url: Granite.HTTP.externalize(APPROXIMATED_MARKDOWN_SERVLET
+                + (this.isrichtext ? '.html' : '.md')
+                + path
             ),
             type: "GET",
             dataType: "text",
@@ -246,8 +248,8 @@ class ContentCreationDialog {
             }
         });
 
-        // http://localhost:4502/bin/cpm/ai/approximated.markdown.md/content/wknd/us/en/magazine/_jcr_content
-        // http://localhost:4502/bin/cpm/ai/approximated.markdown/content/wknd/language-masters/composum-ai-testpages/jcr:content?_=1693499009746
+        // http://localhost:4502/bin/cpm/ai/approximated.md/content/wknd/us/en/magazine/_jcr_content
+        // http://localhost:4502/bin/cpm/ai/approximated.html/content/wknd/us/en/magazine/_jcr_content
     }
 
     /** The path until the /jcr:content */

aem/ui.frontend/src/main/webpack/site/SidePanelDialog.js

@@ -4,8 +4,6 @@ import {AICreate} from './AICreate.js';
 import {contentFragmentPath, errorText, findSingleElement} from './common.js';
 import {DialogHistory} from './DialogHistory.js';
 
-const APPROXIMATE_MARKDOWN_SERVLET = '/bin/cpm/ai/approximated.markdown.md';
-
 /** Keeps dialog histories per path. */
 const historyMap = {};
 

aem/ui.frontend/src/main/webpack/site/styles/composum-ai.scss

@@ -1,5 +1,3 @@
-
-
 //== Move the create dialog icon a bit to the right to avoid overlapping with the help icon.
 //== we make that horribly specific to override the default coral style there
 
@@ -10,3 +8,27 @@
 #composumAI-sidebar-panel .composum-ai-response {
     white-space: break-spaces;
 }
+
+.composum-ai-dialog .composum-ai-prompt-columns {
+
+    .coral-FixedColumn-column {
+        width: auto;
+        flex: 1;
+    }
+
+    .coral-Form-fieldset .coral-Form-fieldset-legend {
+        margin-top: 0;
+    }
+
+    .coral-Form-fieldset {
+        margin-bottom: 0;
+    }
+
+}
+
+.cq-Dialog:not([fullscreen]) .composum-ai-dialog .composum-ai-prompt-columns {
+    .cq-RichText-editable {
+        height: 11.5rem;
+    }
+
+}