furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 02:06:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(web-readability): use native lazy module loading

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-14 04:47:06 -07:00
parent a2790f10af
commit 12847f26bb
1 changed files with 15 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/web-readability/web-content-extractor.ts
+15 -26
View File
@@ -1,15 +1,12 @@
// Web Readability plugin module implements web content extractor behavior.
import { createLazyRuntimeSurface } from "openclaw/plugin-sdk/lazy-runtime";
import type {
WebContentExtractionRequest,
WebContentExtractionResult,
WebContentExtractorPlugin,
} from "openclaw/plugin-sdk/web-content-extractor";
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
import {
htmlToMarkdown,
normalizeWhitespace,
sanitizeHtml,
stripInvisibleUnicode,
type WebContentExtractionRequest,
type WebContentExtractorPlugin,
} from "openclaw/plugin-sdk/web-content-extractor";
const READABILITY_MAX_HTML_CHARS = 1_000_000;
@@ -34,16 +31,11 @@ const HTML_VOID_TAGS = new Set([
const READABILITY_MODULE = "@mozilla/readability";
const LINKEDOM_MODULE = "linkedom";
const loadReadabilityDeps = createLazyRuntimeSurface(
() =>
Promise.all([
import(READABILITY_MODULE) as Promise<typeof import("@mozilla/readability")>,
import(LINKEDOM_MODULE) as Promise<typeof import("linkedom")>,
]),
([readability, linkedom]) => ({
Readability: readability.Readability,
parseHTML: linkedom.parseHTML,
}),
const loadReadabilityDeps = createLazyRuntimeModule(() =>
Promise.all([
import(READABILITY_MODULE) as Promise<typeof import("@mozilla/readability")>,
import(LINKEDOM_MODULE) as Promise<typeof import("linkedom")>,
]),
);
function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boolean {
@@ -119,9 +111,7 @@ function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boole
return false;
}
async function extractWithReadability(
request: WebContentExtractionRequest,
): Promise<WebContentExtractionResult | null> {
async function extractWithReadability(request: WebContentExtractionRequest) {
const cleanHtml = await sanitizeHtml(request.html);
if (
cleanHtml.length > READABILITY_MAX_HTML_CHARS ||
@@ -130,19 +120,18 @@ async function extractWithReadability(
return null;
}
try {
const { Readability, parseHTML } = await loadReadabilityDeps();
const [{ Readability }, { parseHTML }] = await loadReadabilityDeps();
const { document } = parseHTML(cleanHtml, { location: { href: request.url } });
const reader = new Readability(document, { charThreshold: 0 });
const reader = new Readability(document);
const parsed = reader.parse();
if (!parsed?.content) {
return null;
}
const title = parsed.title || undefined;
if (request.extractMode === "text") {
const text = stripInvisibleUnicode(normalizeWhitespace(parsed.textContent ?? ""));
return text ? { text, title } : null;
}
const rendered = htmlToMarkdown(parsed.content);
const rendered =
request.extractMode === "text"
? { text: normalizeWhitespace(parsed.textContent ?? ""), title }
: htmlToMarkdown(parsed.content);
const text = stripInvisibleUnicode(rendered.text);
return text ? { text, title: title ?? rendered.title } : null;
} catch {