From 182ce69a3b9b3d0c0ff2b8965a3671e763d5940d Mon Sep 17 00:00:00 2001 From: Curtis Man Date: Wed, 8 Jan 2025 15:41:52 -0800 Subject: [PATCH] Add support for markdown display (#537) --- ts/packages/agentSdk/src/display.ts | 2 +- .../system/handlers/displayCommandHandler.ts | 20 +++++++- ts/packages/shell/package.json | 1 + ts/packages/shell/src/main/shellSettings.ts | 8 ++-- .../shell/src/main/shellSettingsType.ts | 4 +- .../shell/src/preload/electronTypes.ts | 2 +- .../shell/src/renderer/src/setContent.ts | 46 +++++++++++++------ .../shell/src/renderer/src/settingsView.ts | 10 ++-- ts/pnpm-lock.yaml | 3 ++ 9 files changed, 66 insertions(+), 30 deletions(-) diff --git a/ts/packages/agentSdk/src/display.ts b/ts/packages/agentSdk/src/display.ts index bab91a4db..60cb49a07 100644 --- a/ts/packages/agentSdk/src/display.ts +++ b/ts/packages/agentSdk/src/display.ts @@ -8,7 +8,7 @@ // - text can supports use ANSI escape code to control additional text color and style // - depending on host support, host is expected to strip ANI escape code if not supported. // - html and iframe might not be supported by all hosts. -export type DisplayType = "html" | "iframe" | "text"; +export type DisplayType = "markdown" | "html" | "iframe" | "text"; export type DynamicDisplay = { content: DisplayContent; diff --git a/ts/packages/dispatcher/src/context/system/handlers/displayCommandHandler.ts b/ts/packages/dispatcher/src/context/system/handlers/displayCommandHandler.ts index 1ba5d5285..10360c4cc 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/displayCommandHandler.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/displayCommandHandler.ts @@ -13,6 +13,14 @@ export class DisplayCommandHandler implements CommandHandler { description: "Speak the display for the host that supports TTS", default: false, }, + type: { + description: "Display type", + default: "text", + }, + inline: { + description: "Display inline", + default: false, + }, }, args: { text: { @@ -27,14 +35,22 @@ export class DisplayCommandHandler implements CommandHandler { ) { const { flags, args } = params; + if ( + flags.type !== "text" && + flags.type !== "html" && + flags.type !== "markdown" && + flags.type !== "iframe" + ) { + throw new Error(`Invalid display type: ${flags.type}`); + } for (const content of args.text) { context.actionIO.appendDisplay( { - type: "text", + type: flags.type, content, speak: flags.speak, }, - "block", + flags.inline ? "inline" : "block", ); } } diff --git a/ts/packages/shell/package.json b/ts/packages/shell/package.json index c87bbd76e..8afd1b052 100644 --- a/ts/packages/shell/package.json +++ b/ts/packages/shell/package.json @@ -41,6 +41,7 @@ "dompurify": "^3.1.6", "dotenv": "^16.3.1", "electron-updater": "^6.3.2", + "markdown-it": "^14.1.0", "microsoft-cognitiveservices-speech-sdk": "^1.38.0", "typechat": "^0.1.1", "ws": "^8.17.1" diff --git a/ts/packages/shell/src/main/shellSettings.ts b/ts/packages/shell/src/main/shellSettings.ts index b7de1a464..9e5405a41 100644 --- a/ts/packages/shell/src/main/shellSettings.ts +++ b/ts/packages/shell/src/main/shellSettings.ts @@ -36,7 +36,7 @@ export class ShellSettings public multiModalContent: boolean; public devUI: boolean; public partialCompletion: boolean; - public allowedDisplayType: DisplayType[]; + public disallowedDisplayType: DisplayType[]; public onSettingsChanged: EmptyFunction | null; public onShowSettingsDialog: ((dialogName: string) => void) | null; public onRunDemo: ((interactive: boolean) => void) | null; @@ -81,7 +81,7 @@ export class ShellSettings this.multiModalContent = settings.multiModalContent; this.devUI = settings.devUI; this.partialCompletion = settings.partialCompletion; - this.allowedDisplayType = settings.allowedDisplayType; + this.disallowedDisplayType = settings.disallowedDisplayType; this.onSettingsChanged = null; this.onShowSettingsDialog = null; @@ -178,8 +178,8 @@ export class ShellSettings } public isDisplayTypeAllowed(displayType: DisplayType): boolean { - for (let i = 0; i < this.allowedDisplayType.length; i++) { - if (this.allowedDisplayType[i] === displayType) { + for (let i = 0; i < this.disallowedDisplayType.length; i++) { + if (this.disallowedDisplayType[i] === displayType) { return true; } } diff --git a/ts/packages/shell/src/main/shellSettingsType.ts b/ts/packages/shell/src/main/shellSettingsType.ts index 726ed6849..075b3da7d 100644 --- a/ts/packages/shell/src/main/shellSettingsType.ts +++ b/ts/packages/shell/src/main/shellSettingsType.ts @@ -21,7 +21,7 @@ export type ShellSettingsType = { multiModalContent: boolean; devUI: boolean; partialCompletion: boolean; - allowedDisplayType: DisplayType[]; + disallowedDisplayType: DisplayType[]; }; export const defaultSettings: ShellSettingsType = { @@ -35,5 +35,5 @@ export const defaultSettings: ShellSettingsType = { multiModalContent: true, devUI: false, partialCompletion: true, - allowedDisplayType: ["html", "iframe", "text"], + disallowedDisplayType: [], }; diff --git a/ts/packages/shell/src/preload/electronTypes.ts b/ts/packages/shell/src/preload/electronTypes.ts index 62ce7ea53..31a8eafc3 100644 --- a/ts/packages/shell/src/preload/electronTypes.ts +++ b/ts/packages/shell/src/preload/electronTypes.ts @@ -37,7 +37,7 @@ export interface ClientSettingsProvider { set: SetSettingFunction | null; } -export type DisplayType = "html" | "iframe" | "text"; +export type DisplayType = "html" | "iframe" | "text" | "markdown"; export type ClientActions = | "show-camera" diff --git a/ts/packages/shell/src/renderer/src/setContent.ts b/ts/packages/shell/src/renderer/src/setContent.ts index d3840d171..57e22ef36 100644 --- a/ts/packages/shell/src/renderer/src/setContent.ts +++ b/ts/packages/shell/src/renderer/src/setContent.ts @@ -11,6 +11,7 @@ import { } from "@typeagent/agent-sdk"; import DOMPurify from "dompurify"; import { SettingsView } from "./settingsView"; +import MarkdownIt from "markdown-it"; const ansi_up = new AnsiUp(); ansi_up.use_classes = true; @@ -33,16 +34,30 @@ function encodeTextToHtml(text: string): string { } const enableText2Html = true; -function processContent(content: string, type: string): string { - return type === "html" - ? DOMPurify.sanitize(content, { - ADD_ATTR: ["target", "onclick", "onerror"], - ADD_DATA_URI_TAGS: ["img"], - ADD_URI_SAFE_ATTR: ["src"], - }) - : enableText2Html - ? textToHtml(content) - : stripAnsi(encodeTextToHtml(content)); +function processContent( + content: string, + type: string, + inline: boolean = false, +): string { + switch (type) { + case "iframe": + return content; + case "html": + return DOMPurify.sanitize(content, { + ADD_ATTR: ["target", "onclick", "onerror"], + ADD_DATA_URI_TAGS: ["img"], + ADD_URI_SAFE_ATTR: ["src"], + }); + case "markdown": + const md = new MarkdownIt(); + return inline ? md.renderInline(content) : md.render(content); + case "text": + return enableText2Html + ? textToHtml(content) + : stripAnsi(encodeTextToHtml(content)); + default: + throw new Error(`Invalid content type ${type}`); + } } function matchKindStyle(elm: HTMLElement, kindStyle?: string) { @@ -60,9 +75,10 @@ function matchKindStyle(elm: HTMLElement, kindStyle?: string) { function messageContentToHTML( message: MessageContent, type: DisplayType, + inline: boolean, ): string { if (typeof message === "string") { - return processContent(message, type); + return processContent(message, type, inline); } if (message.length === 0) { @@ -71,7 +87,7 @@ function messageContentToHTML( if (typeof message[0] === "string") { return (message as string[]) - .map((s) => processContent(s, type)) + .map((s) => processContent(s, type, inline)) .join("
"); } @@ -149,7 +165,11 @@ export function setContent( } // Process content according to type - const contentHtml = messageContentToHTML(message, type); + const contentHtml = messageContentToHTML( + message, + type, + appendMode === "inline", + ); // if the agent wants to show script we need to do that in isolation so create an iframe // and put both the script and supplied HTML into it diff --git a/ts/packages/shell/src/renderer/src/settingsView.ts b/ts/packages/shell/src/renderer/src/settingsView.ts index e8c93e219..cd2a8181d 100644 --- a/ts/packages/shell/src/renderer/src/settingsView.ts +++ b/ts/packages/shell/src/renderer/src/settingsView.ts @@ -325,12 +325,8 @@ export class SettingsView { } public isDisplayTypeAllowed(displayType: DisplayType): boolean { - for (let i = 0; i < this.shellSettings.allowedDisplayType.length; i++) { - if (this.shellSettings.allowedDisplayType[i] === displayType) { - return true; - } - } - - return false; + return !this.shellSettings.disallowedDisplayType.some( + (type) => type === displayType, + ); } } diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index a973e80b8..4700d8e04 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -1865,6 +1865,9 @@ importers: electron-updater: specifier: ^6.3.2 version: 6.3.2 + markdown-it: + specifier: ^14.1.0 + version: 14.1.0 microsoft-cognitiveservices-speech-sdk: specifier: ^1.38.0 version: 1.38.0