diff --git a/extensions/web-readability/web-content-extractor.ts b/extensions/web-readability/web-content-extractor.ts index 7126ec7253b..26330cf32f5 100644 --- a/extensions/web-readability/web-content-extractor.ts +++ b/extensions/web-readability/web-content-extractor.ts @@ -1,15 +1,12 @@ // Web Readability plugin module implements web content extractor behavior. -import { createLazyRuntimeSurface } from "openclaw/plugin-sdk/lazy-runtime"; -import type { - WebContentExtractionRequest, - WebContentExtractionResult, - WebContentExtractorPlugin, -} from "openclaw/plugin-sdk/web-content-extractor"; +import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; import { htmlToMarkdown, normalizeWhitespace, sanitizeHtml, stripInvisibleUnicode, + type WebContentExtractionRequest, + type WebContentExtractorPlugin, } from "openclaw/plugin-sdk/web-content-extractor"; const READABILITY_MAX_HTML_CHARS = 1_000_000; @@ -34,16 +31,11 @@ const HTML_VOID_TAGS = new Set([ const READABILITY_MODULE = "@mozilla/readability"; const LINKEDOM_MODULE = "linkedom"; -const loadReadabilityDeps = createLazyRuntimeSurface( - () => - Promise.all([ - import(READABILITY_MODULE) as Promise, - import(LINKEDOM_MODULE) as Promise, - ]), - ([readability, linkedom]) => ({ - Readability: readability.Readability, - parseHTML: linkedom.parseHTML, - }), +const loadReadabilityDeps = createLazyRuntimeModule(() => + Promise.all([ + import(READABILITY_MODULE) as Promise, + import(LINKEDOM_MODULE) as Promise, + ]), ); function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boolean { @@ -119,9 +111,7 @@ function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boole return false; } -async function extractWithReadability( - request: WebContentExtractionRequest, -): Promise { +async function extractWithReadability(request: WebContentExtractionRequest) { const cleanHtml = await sanitizeHtml(request.html); if ( cleanHtml.length > READABILITY_MAX_HTML_CHARS || @@ -130,19 +120,18 @@ async function extractWithReadability( return null; } try { - const { Readability, parseHTML } = await loadReadabilityDeps(); + const [{ Readability }, { parseHTML }] = await loadReadabilityDeps(); const { document } = parseHTML(cleanHtml, { location: { href: request.url } }); - const reader = new Readability(document, { charThreshold: 0 }); + const reader = new Readability(document); const parsed = reader.parse(); if (!parsed?.content) { return null; } const title = parsed.title || undefined; - if (request.extractMode === "text") { - const text = stripInvisibleUnicode(normalizeWhitespace(parsed.textContent ?? "")); - return text ? { text, title } : null; - } - const rendered = htmlToMarkdown(parsed.content); + const rendered = + request.extractMode === "text" + ? { text: normalizeWhitespace(parsed.textContent ?? ""), title } + : htmlToMarkdown(parsed.content); const text = stripInvisibleUnicode(rendered.text); return text ? { text, title: title ?? rendered.title } : null; } catch {