feat: implement HTML GenUI component and register custom markdown tags

astrbot/core/astr_main_agent_resources.py

@@ -54,9 +54,17 @@ CHATUI_SPECIAL_DEFAULT_PERSONA_PROMPT = (
     "move toward structure, insight, or guidance.\n"
     "You listen more than you speak, respect uncertainty, avoid forcing quick conclusions or grand narratives, "
     "and prefer clear, restrained language over unnecessary emotional embellishment. At your core, you value "
-    "empathy, clarity, autonomy, and meaning, favoring steady, sustainable progress over judgment or dramatic leaps."
+    "empathy, clarity, autonomy, and meaning, favoring steady, sustainable progress over judgment or dramatic leaps. "
     'When you answered, you need to add a follow up question / summarization but do not add "Follow up" words. '
     "Such as, user asked you to generate codes, you can add: Do you need me to run these codes for you?"
+    "\n\n[ChatUI HTML GenUI]\n"
+    "When the user asks you to create, prototype, preview, or modify a visual HTML UI, "
+    "output the runnable HTML inside exactly one `<html-genui>...</html-genui>` block. "
+    'You may add a short optional title on the opening tag, for example `<html-genui title="Dashboard mockup">`. '
+    "Do not wrap the block in Markdown code fences. Put complete, self-contained HTML/CSS/JavaScript inside the tag, "
+    "including `<style>` and `<script>` when needed. Prefer responsive layouts that fit a chat iframe. "
+    "For revisions, output the full updated `<html-genui>` block instead of a diff. "
+    "Only use this block when an HTML UI preview is useful; otherwise answer normally."
 )
 
 LIVE_MODE_SYSTEM_PROMPT = (

dashboard/src/components/chat/Chat.vue

@@ -1720,22 +1720,6 @@ function toggleTheme() {
   padding: 0 0 18px;
 }
 
-.composer-shell::before {
-  content: "";
-  position: absolute;
-  z-index: -1;
-  left: 0;
-  right: 0;
-  top: -36px;
-  height: 36px;
-  pointer-events: none;
-  background: linear-gradient(
-    to bottom,
-    rgba(var(--v-theme-background), 0),
-    var(--chat-page-bg)
-  );
-}
-
 .composer-shell :deep(.input-area) {
   border-top: 0;
 }
@@ -1744,10 +1728,6 @@ function toggleTheme() {
   padding-bottom: 0;
 }
 
-.empty-chat .composer-shell::before {
-  display: none;
-}
-
 kbd {
   padding: 1px 5px;
   border-radius: 4px;

dashboard/src/components/chat/ChatInput.vue

@@ -901,7 +901,7 @@ defineExpose({
 
 <style scoped>
 .input-area {
-  padding: 12px 16px 0;
+  padding: 0 16px;
   background-color: transparent;
   position: relative;
   border-top: 1px solid var(--v-theme-border);

dashboard/src/components/chat/ChatMessageList.vue

@@ -389,8 +389,6 @@
 <script setup lang="ts">
 import { computed, nextTick, reactive, ref } from "vue";
 import axios from "axios";
-import { setCustomComponents } from "markstream-vue";
-import "markstream-vue/index.css";
 import RegenerateMenu, {
   type RegenerateModelSelection,
 } from "@/components/chat/RegenerateMenu.vue";
@@ -400,12 +398,13 @@ import ToolCallCard from "@/components/chat/message_list_comps/ToolCallCard.vue"
 import ToolCallItem from "@/components/chat/message_list_comps/ToolCallItem.vue";
 import IPythonToolBlock from "@/components/chat/message_list_comps/IPythonToolBlock.vue";
 import RefsSidebar from "@/components/chat/message_list_comps/RefsSidebar.vue";
-import RefNode from "@/components/chat/message_list_comps/RefNode.vue";
-import ThreadNode from "@/components/chat/message_list_comps/ThreadNode.vue";
 import ActionRef from "@/components/chat/message_list_comps/ActionRef.vue";
 import MarkdownMessagePart from "@/components/chat/message_list_comps/MarkdownMessagePart.vue";
-import ThemeAwareMarkdownCodeBlock from "@/components/shared/ThemeAwareMarkdownCodeBlock.vue";
 import StyledMenu from "@/components/shared/StyledMenu.vue";
+import {
+  CHAT_MARKDOWN_CUSTOM_TAGS,
+  registerChatMarkdownComponents,
+} from "@/components/chat/chatMarkdownComponents";
 import {
   displayParts as displayMessageParts,
   messageBlocks as buildMessageBlocks,
@@ -466,15 +465,11 @@ const emit = defineEmits<{
   openRefs: [refs: unknown];
 }>();
 
-setCustomComponents("chat-message", {
-  ref: RefNode,
-  thread: ThreadNode,
-  code_block: ThemeAwareMarkdownCodeBlock,
-});
+registerChatMarkdownComponents();
 
 const { t } = useI18n();
 const { tm } = useModuleI18n("features/chat");
-const customMarkdownTags = ["ref"];
+const customMarkdownTags = CHAT_MARKDOWN_CUSTOM_TAGS;
 const downloadingFiles = ref(new Set<string>());
 const imagePreview = reactive({ visible: false, url: "" });
 const refsSidebarOpen = ref(false);

dashboard/src/components/chat/MessageList.vue

@@ -256,17 +256,17 @@
 <script setup lang="ts">
 import { computed, nextTick, reactive, ref } from "vue";
 import axios from "axios";
-import { setCustomComponents } from "markstream-vue";
-import "markstream-vue/index.css";
+import {
+  CHAT_MARKDOWN_CUSTOM_TAGS,
+  registerChatMarkdownComponents,
+} from "@/components/chat/chatMarkdownComponents";
 import IPythonToolBlock from "@/components/chat/message_list_comps/IPythonToolBlock.vue";
 import MarkdownMessagePart from "@/components/chat/message_list_comps/MarkdownMessagePart.vue";
 import ReasoningBlock from "@/components/chat/message_list_comps/ReasoningBlock.vue";
-import RefNode from "@/components/chat/message_list_comps/RefNode.vue";
 import RefsSidebar from "@/components/chat/message_list_comps/RefsSidebar.vue";
 import ToolCallCard from "@/components/chat/message_list_comps/ToolCallCard.vue";
 import ToolCallItem from "@/components/chat/message_list_comps/ToolCallItem.vue";
 import ActionRef from "@/components/chat/message_list_comps/ActionRef.vue";
-import ThemeAwareMarkdownCodeBlock from "@/components/shared/ThemeAwareMarkdownCodeBlock.vue";
 import {
   displayParts as displayMessageParts,
   messageBlocks as buildMessageBlocks,
@@ -294,13 +294,10 @@ const props = withDefaults(
   },
 );
 
-setCustomComponents("chat-message", {
-  ref: RefNode,
-  code_block: ThemeAwareMarkdownCodeBlock,
-});
+registerChatMarkdownComponents();
 
 const { tm } = useModuleI18n("features/chat");
-const customMarkdownTags = ["ref"];
+const customMarkdownTags = CHAT_MARKDOWN_CUSTOM_TAGS;
 const downloadingFiles = ref(new Set<string>());
 const messageListRoot = ref<HTMLElement | null>(null);
 const imagePreview = reactive({ visible: false, url: "" });

dashboard/src/components/chat/StandaloneChat.vue

@@ -184,16 +184,16 @@ import {
   ref,
 } from "vue";
 import axios from "axios";
-import { setCustomComponents } from "markstream-vue";
-import "markstream-vue/index.css";
 import ChatInput from "@/components/chat/ChatInput.vue";
+import {
+  CHAT_MARKDOWN_CUSTOM_TAGS,
+  registerChatMarkdownComponents,
+} from "@/components/chat/chatMarkdownComponents";
 import IPythonToolBlock from "@/components/chat/message_list_comps/IPythonToolBlock.vue";
 import MarkdownMessagePart from "@/components/chat/message_list_comps/MarkdownMessagePart.vue";
 import ReasoningBlock from "@/components/chat/message_list_comps/ReasoningBlock.vue";
-import RefNode from "@/components/chat/message_list_comps/RefNode.vue";
 import ToolCallCard from "@/components/chat/message_list_comps/ToolCallCard.vue";
 import ToolCallItem from "@/components/chat/message_list_comps/ToolCallItem.vue";
-import ThemeAwareMarkdownCodeBlock from "@/components/shared/ThemeAwareMarkdownCodeBlock.vue";
 import { useMediaHandling } from "@/composables/useMediaHandling";
 import {
   displayParts as displayMessageParts,
@@ -213,10 +213,7 @@ const props = withDefaults(defineProps<{ configId?: string | null }>(), {
   configId: "default",
 });
 
-setCustomComponents("chat-message", {
-  ref: RefNode,
-  code_block: ThemeAwareMarkdownCodeBlock,
-});
+registerChatMarkdownComponents();
 
 const { tm } = useModuleI18n("features/chat");
 const customizer = useCustomizerStore();
@@ -231,7 +228,7 @@ const inputRef = ref<InstanceType<typeof ChatInput> | null>(null);
 const imagePreview = reactive({ visible: false, url: "" });
 
 const isDark = computed(() => customizer.uiTheme === "PurpleThemeDark");
-const customMarkdownTags = ["ref"];
+const customMarkdownTags = CHAT_MARKDOWN_CUSTOM_TAGS;
 
 const {
   stagedFiles,
@@ -616,22 +613,6 @@ function closeImage() {
   background: rgb(var(--v-theme-background));
 }
 
-.standalone-composer::before {
-  content: "";
-  position: absolute;
-  z-index: -1;
-  left: 0;
-  right: 0;
-  top: -32px;
-  height: 32px;
-  pointer-events: none;
-  background: linear-gradient(
-    to bottom,
-    rgba(var(--v-theme-background), 0),
-    rgb(var(--v-theme-background))
-  );
-}
-
 .standalone-composer :deep(.input-area) {
   border-top: 0;
 }

dashboard/src/components/chat/chatMarkdownComponents.ts

@@ -0,0 +1,17 @@
+import { setCustomComponents } from "markstream-vue";
+import "markstream-vue/index.css";
+import HtmlGenUiNode from "@/components/chat/message_list_comps/HtmlGenUiNode.vue";
+import RefNode from "@/components/chat/message_list_comps/RefNode.vue";
+import ThreadNode from "@/components/chat/message_list_comps/ThreadNode.vue";
+import ThemeAwareMarkdownCodeBlock from "@/components/shared/ThemeAwareMarkdownCodeBlock.vue";
+
+export const CHAT_MARKDOWN_CUSTOM_TAGS: string[] = ["ref", "html-genui"];
+
+export function registerChatMarkdownComponents() {
+  setCustomComponents("chat-message", {
+    ref: RefNode,
+    thread: ThreadNode,
+    "html-genui": HtmlGenUiNode,
+    code_block: ThemeAwareMarkdownCodeBlock,
+  });
+}

dashboard/src/components/chat/message_list_comps/HtmlGenUiNode.vue

@@ -0,0 +1,276 @@
+<template>
+  <div class="html-genui-node" :class="{ 'is-dark': isDark, 'is-loading': isLoading }">
+    <div class="html-genui-header">
+      <div class="html-genui-title">{{ panelTitle }}</div>
+      <div class="html-genui-toggle" role="tablist" aria-label="HTML GenUI view">
+        <button
+          class="html-genui-toggle-button"
+          :class="{ active: viewMode === 'preview' }"
+          type="button"
+          role="tab"
+          :aria-selected="viewMode === 'preview'"
+          @click="viewMode = 'preview'"
+        >
+          Preview
+        </button>
+        <button
+          class="html-genui-toggle-button"
+          :class="{ active: viewMode === 'source' }"
+          type="button"
+          role="tab"
+          :aria-selected="viewMode === 'source'"
+          @click="viewMode = 'source'"
+        >
+          Source
+        </button>
+      </div>
+    </div>
+
+    <iframe
+      v-if="viewMode === 'preview'"
+      class="html-genui-frame"
+      :srcdoc="renderedSrcdoc"
+      :sandbox="sandboxPolicy"
+      title="Generated HTML UI preview"
+      loading="lazy"
+    ></iframe>
+    <pre v-else class="html-genui-source"><code>{{ htmlContent }}</code></pre>
+  </div>
+</template>
+
+<script setup lang="ts">
+import { computed, onBeforeUnmount, ref, watch } from "vue";
+
+const RENDER_THROTTLE_MS = 500;
+const sandboxPolicy =
+  "allow-forms allow-modals allow-pointer-lock allow-popups allow-scripts";
+
+const props = defineProps<{
+  node?: {
+    attrs?: Array<[string, string]>;
+    content?: string;
+    raw?: string;
+    loading?: boolean;
+  } | null;
+  loading?: boolean;
+  isDark?: boolean;
+  title?: string;
+}>();
+
+const renderedSrcdoc = ref("");
+const viewMode = ref<"preview" | "source">("preview");
+let pendingTimer: ReturnType<typeof setTimeout> | null = null;
+let lastRenderAt = 0;
+
+const htmlContent = computed(() =>
+  stripHtmlGenUiWrapper(String(props.node?.content || props.node?.raw || "")),
+);
+const isLoading = computed(() => Boolean(props.loading || props.node?.loading));
+const isDark = computed(() => Boolean(props.isDark));
+const panelTitle = computed(
+  () => props.title?.trim() || attrValue("title") || "HTML UI",
+);
+
+watch(
+  [htmlContent, isLoading, isDark],
+  () => {
+    scheduleRender(!isLoading.value);
+  },
+  { immediate: true },
+);
+
+onBeforeUnmount(() => {
+  if (pendingTimer) {
+    clearTimeout(pendingTimer);
+    pendingTimer = null;
+  }
+});
+
+function scheduleRender(force = false) {
+  if (force) {
+    renderNow();
+    return;
+  }
+
+  const elapsed = Date.now() - lastRenderAt;
+  if (elapsed >= RENDER_THROTTLE_MS) {
+    renderNow();
+    return;
+  }
+
+  if (!pendingTimer) {
+    pendingTimer = setTimeout(renderNow, RENDER_THROTTLE_MS - elapsed);
+  }
+}
+
+function renderNow() {
+  if (pendingTimer) {
+    clearTimeout(pendingTimer);
+    pendingTimer = null;
+  }
+  lastRenderAt = Date.now();
+  renderedSrcdoc.value = buildSrcdoc(htmlContent.value, isDark.value);
+}
+
+function stripHtmlGenUiWrapper(value: string) {
+  return value
+    .replace(/^\s*<html-genui\b[^>]*>/i, "")
+    .replace(/<\/html-genui>\s*$/i, "")
+    .trim();
+}
+
+function attrValue(name: string) {
+  const attr = props.node?.attrs?.find(
+    ([key]) => key.toLowerCase() === name.toLowerCase(),
+  );
+  return attr?.[1]?.trim() || "";
+}
+
+function buildSrcdoc(content: string, dark: boolean) {
+  const headExtras = buildHeadExtras(dark);
+  if (/<html[\s>]/i.test(content)) {
+    return injectHeadExtras(content, headExtras);
+  }
+
+  return `<!doctype html>
+<html>
+<head>${headExtras}</head>
+<body>${content}</body>
+</html>`;
+}
+
+function injectHeadExtras(html: string, headExtras: string) {
+  if (/<head[\s>]/i.test(html)) {
+    return html.replace(/<head([^>]*)>/i, `<head$1>${headExtras}`);
+  }
+  if (/<html[\s>]/i.test(html)) {
+    return html.replace(/<html([^>]*)>/i, `<html$1><head>${headExtras}</head>`);
+  }
+  return `<!doctype html><html><head>${headExtras}</head><body>${html}</body></html>`;
+}
+
+function buildHeadExtras(dark: boolean) {
+  const bg = dark ? "#111827" : "#ffffff";
+  const fg = dark ? "#f9fafb" : "#111827";
+  return `<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<base target="_blank">
+<style>
+  :root { color-scheme: ${dark ? "dark" : "light"}; }
+  * { box-sizing: border-box; }
+  html, body { min-height: 100%; margin: 0; }
+  body {
+    background: ${bg};
+    color: ${fg};
+    font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+  }
+  img, video, canvas, svg { max-width: 100%; }
+</style>`;
+}
+</script>
+
+<style scoped>
+.html-genui-node {
+  width: 100%;
+  margin: 12px 0;
+  overflow: hidden;
+  border: 1px solid rgba(128, 128, 128, 0.24);
+  border-radius: 8px;
+  background: rgb(var(--v-theme-surface));
+}
+
+.html-genui-node.is-dark {
+  border-color: rgba(160, 160, 160, 0.28);
+}
+
+.html-genui-header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 12px;
+  min-height: 42px;
+  padding: 8px 10px 8px 12px;
+  border-bottom: 1px solid rgba(128, 128, 128, 0.2);
+  background: rgba(128, 128, 128, 0.04);
+}
+
+.html-genui-title {
+  min-width: 0;
+  overflow: hidden;
+  color: rgba(var(--v-theme-on-surface), 0.84);
+  font-size: 13px;
+  font-weight: 600;
+  line-height: 1.3;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.html-genui-toggle {
+  display: inline-flex;
+  flex: 0 0 auto;
+  overflow: hidden;
+  border: 1px solid rgba(128, 128, 128, 0.22);
+  border-radius: 6px;
+  background: rgba(128, 128, 128, 0.06);
+}
+
+.html-genui-toggle-button {
+  min-width: 64px;
+  border: 0;
+  border-right: 1px solid rgba(128, 128, 128, 0.2);
+  padding: 4px 10px;
+  background: transparent;
+  color: rgba(var(--v-theme-on-surface), 0.68);
+  cursor: pointer;
+  font: inherit;
+  font-size: 12px;
+  font-weight: 600;
+  line-height: 1.4;
+}
+
+.html-genui-toggle-button:focus {
+  outline: none;
+}
+
+.html-genui-toggle-button:focus-visible {
+  outline: 2px solid rgba(128, 128, 128, 0.36);
+  outline-offset: -2px;
+}
+
+.html-genui-toggle-button:last-child {
+  border-right: 0;
+}
+
+.html-genui-toggle-button.active {
+  background: rgba(128, 128, 128, 0.16);
+  color: rgba(var(--v-theme-on-surface), 0.92);
+}
+
+.html-genui-frame {
+  display: block;
+  width: 100%;
+  height: clamp(280px, 52vh, 620px);
+  border: 0;
+  background: #fff;
+}
+
+.html-genui-node.is-loading .html-genui-frame {
+  opacity: 0.96;
+}
+
+.html-genui-source {
+  height: clamp(280px, 52vh, 620px);
+  margin: 0;
+  overflow: auto;
+  padding: 14px;
+  background: rgba(var(--v-theme-on-surface), 0.035);
+  color: rgba(var(--v-theme-on-surface), 0.86);
+  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
+    "Liberation Mono", "Courier New", monospace;
+  font-size: 12px;
+  line-height: 1.55;
+  tab-size: 2;
+  white-space: pre-wrap;
+  word-break: break-word;
+}
+</style>