mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(docs-i18n): preserve product link labels
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -37,6 +38,7 @@ type docChunkStructure struct {
|
||||
listShapes []markdownListShape
|
||||
inlineCodeSpans []string
|
||||
linkDestinations []string
|
||||
protectedLinkLabels []string
|
||||
numericValues []string
|
||||
fencedPlaceholders []string
|
||||
fencedProtocolTokens []string
|
||||
@@ -107,6 +109,9 @@ func validateDocBodyFencedLiterals(source, translated string) error {
|
||||
if !sameStringMultiset(sourceStructure.linkDestinations, translatedStructure.linkDestinations) {
|
||||
return fmt.Errorf("link destination mismatch: source=%d translated=%d", len(sourceStructure.linkDestinations), len(translatedStructure.linkDestinations))
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.protectedLinkLabels, translatedStructure.protectedLinkLabels) {
|
||||
return fmt.Errorf("protected link label mismatch: source=%d translated=%d", len(sourceStructure.protectedLinkLabels), len(translatedStructure.protectedLinkLabels))
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) {
|
||||
return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues))
|
||||
}
|
||||
@@ -296,6 +301,9 @@ func validateDocChunkTranslation(source, translated string) error {
|
||||
if !sameStringMultiset(sourceStructure.linkDestinations, translatedStructure.linkDestinations) {
|
||||
return fmt.Errorf("link destination mismatch: source=%d translated=%d", len(sourceStructure.linkDestinations), len(translatedStructure.linkDestinations))
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.protectedLinkLabels, translatedStructure.protectedLinkLabels) {
|
||||
return fmt.Errorf("protected link label mismatch: source=%d translated=%d", len(sourceStructure.protectedLinkLabels), len(translatedStructure.protectedLinkLabels))
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) {
|
||||
return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues))
|
||||
}
|
||||
@@ -460,6 +468,7 @@ func summarizeDocChunkStructure(text string) docChunkStructure {
|
||||
listShapes: extractMarkdownListShapes(text),
|
||||
inlineCodeSpans: extractMarkdownInlineCodeValues(text),
|
||||
linkDestinations: extractMarkdownLinkDestinations(text),
|
||||
protectedLinkLabels: extractProtectedMarkdownLinkLabels(text),
|
||||
numericValues: extractNumericValues(text),
|
||||
fencedPlaceholders: fencedPlaceholders,
|
||||
fencedProtocolTokens: fencedProtocolTokens,
|
||||
@@ -492,6 +501,106 @@ func extractMarkdownLinkDestinations(text string) []string {
|
||||
return destinations
|
||||
}
|
||||
|
||||
func extractProtectedMarkdownLinkLabels(text string) []string {
|
||||
source := []byte(normalizeDocComponentsForMarkdownParse(text))
|
||||
doc := parseDocsMarkdown(source)
|
||||
labels := make([]string, 0)
|
||||
_ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
kind := ""
|
||||
destination := ""
|
||||
switch link := node.(type) {
|
||||
case *ast.Link:
|
||||
kind = "link"
|
||||
destination = string(link.Destination)
|
||||
case *ast.Image:
|
||||
kind = "image"
|
||||
destination = string(link.Destination)
|
||||
default:
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
label := strings.TrimSpace(string(node.Text(source)))
|
||||
if isProtectedProductLinkLabel(label, destination) {
|
||||
labels = append(labels, kind+":"+destination+":"+label)
|
||||
}
|
||||
return ast.WalkContinue, nil
|
||||
})
|
||||
return labels
|
||||
}
|
||||
|
||||
func isProtectedProductLinkLabel(label, destination string) bool {
|
||||
if isAlwaysProtectedProductName(label) {
|
||||
return true
|
||||
}
|
||||
name, ok := contextualProtectedProductName(label)
|
||||
return ok && destinationMentionsProductName(destination, name)
|
||||
}
|
||||
|
||||
type contextualProductDestinationRule struct {
|
||||
hosts []string
|
||||
routes []string
|
||||
}
|
||||
|
||||
var contextualProductDestinations = map[string]contextualProductDestinationRule{
|
||||
"Render": {hosts: []string{"render.com"}, routes: []string{"/install/render"}},
|
||||
"Matrix": {hosts: []string{"matrix.org"}, routes: []string{"/channels/matrix"}},
|
||||
"Raft": {hosts: []string{"raft.build"}, routes: []string{"/channels/raft"}},
|
||||
"Chutes": {hosts: []string{"chutes.ai"}, routes: []string{"/providers/chutes"}},
|
||||
"fal": {hosts: []string{"fal.ai"}, routes: []string{"/providers/fal", "/plugins/reference/fal"}},
|
||||
"Fal": {hosts: []string{"fal.ai"}, routes: []string{"/providers/fal", "/plugins/reference/fal"}},
|
||||
"Fireworks": {hosts: []string{"fireworks.ai"}, routes: []string{"/providers/fireworks", "/plugins/reference/fireworks"}},
|
||||
"Inferrs": {routes: []string{"/providers/inferrs", "/ericcurtin/inferrs"}},
|
||||
"Meta": {hosts: []string{"meta.ai", "meta.com"}, routes: []string{"/providers/meta", "/plugins/reference/meta"}},
|
||||
"Runway": {hosts: []string{"runwayml.com"}, routes: []string{"/providers/runway", "/plugins/reference/runway"}},
|
||||
"Synthetic": {hosts: []string{"synthetic.new"}, routes: []string{"/providers/synthetic"}},
|
||||
"Upstash Box": {hosts: []string{"upstash.com"}, routes: []string{"/install/upstash", "/docs/box"}},
|
||||
"Lobster": {routes: []string{"/tools/lobster", "/openclaw/lobster"}},
|
||||
"Mantis": {routes: []string{"/concepts/mantis", "/openclaw/mantis"}},
|
||||
"Tokenjuice": {routes: []string{"/tools/tokenjuice", "/openclaw/tokenjuice"}},
|
||||
}
|
||||
|
||||
func destinationMentionsProductName(destination, name string) bool {
|
||||
rule, ok := contextualProductDestinations[name]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
parsed, err := url.Parse(destination)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(parsed.Hostname())
|
||||
for _, allowedHost := range rule.hosts {
|
||||
if host == allowedHost || strings.HasSuffix(host, "."+allowedHost) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
path := strings.ToLower(parsed.Path)
|
||||
for _, route := range rule.routes {
|
||||
if pathContainsRoute(path, route) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pathContainsRoute(path, route string) bool {
|
||||
pathParts := nonemptyPathParts(path)
|
||||
routeParts := nonemptyPathParts(route)
|
||||
for start := 0; start+len(routeParts) <= len(pathParts); start++ {
|
||||
if slices.Equal(pathParts[start:start+len(routeParts)], routeParts) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func nonemptyPathParts(value string) []string {
|
||||
parts := strings.Split(value, "/")
|
||||
return slices.DeleteFunc(parts, func(part string) bool { return part == "" })
|
||||
}
|
||||
|
||||
func parseDocsMarkdown(source []byte) ast.Node {
|
||||
return goldmark.New(goldmark.WithExtensions(extension.GFM, extension.Footnote)).Parser().Parse(textpkg.NewReader(source))
|
||||
}
|
||||
|
||||
@@ -2325,8 +2325,8 @@ func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
|
||||
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
|
||||
t.Fatalf("expected translated read_when entry in output:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, "prompt_version: 25") {
|
||||
t.Fatalf("expected prompt version 25 in output metadata:\n%s", text)
|
||||
if !strings.Contains(text, "prompt_version: 26") {
|
||||
t.Fatalf("expected prompt version 26 in output metadata:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2424,6 +2424,86 @@ func TestValidateDocChunkTranslationRejectsDroppedDuplicateLink(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocChunkTranslationRejectsMovedProtectedProductLinkLabel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Deploy OpenClaw on [Render](https://render.com) using the Blueprint.\n"
|
||||
translated := "Render पर Blueprint का उपयोग करके [OpenClaw](https://render.com) परिनियोजित करें।\n"
|
||||
for name, validate := range map[string]func(string, string) error{
|
||||
"chunk": validateDocChunkTranslation,
|
||||
"final": validateDocBodyFencedLiterals,
|
||||
} {
|
||||
err := validate(source, translated)
|
||||
if err == nil || !strings.Contains(err.Error(), "protected link label mismatch") {
|
||||
t.Fatalf("%s: expected protected link label mismatch, got %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocBodyAllowsTranslatedOrdinaryLinkLabel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Read the [deployment guide](/setup).\n"
|
||||
translated := "Lesen Sie den [Bereitstellungsleitfaden](/setup).\n"
|
||||
if err := validateDocBodyFencedLiterals(source, translated); err != nil {
|
||||
t.Fatalf("expected translated ordinary link label to pass: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocBodyAllowsTranslatedContextualOrdinaryLinkLabel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "[Render](/guides/pre-render) the page now.\n"
|
||||
translated := "Die Seite jetzt [darstellen](/guides/pre-render).\n"
|
||||
if err := validateDocBodyFencedLiterals(source, translated); err != nil {
|
||||
t.Fatalf("expected ordinary contextual label to translate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocBodyRejectsMovedProtectedReferenceLinkLabel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Deploy on [Render][provider].\n\n[provider]: https://render.com\n"
|
||||
translated := "Auf Render mit [OpenClaw][provider] bereitstellen.\n\n[provider]: https://render.com\n"
|
||||
err := validateDocBodyFencedLiterals(source, translated)
|
||||
if err == nil || !strings.Contains(err.Error(), "protected link label mismatch") {
|
||||
t.Fatalf("expected protected reference-link label mismatch, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualProtectedProductLinksRecognizeCanonicalDestinations(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
label string
|
||||
destination string
|
||||
}{
|
||||
{"Render", "/install/render"},
|
||||
{"Matrix", "/channels/matrix"},
|
||||
{"Raft", "/channels/raft"},
|
||||
{"Chutes", "/providers/chutes"},
|
||||
{"fal", "/providers/fal"},
|
||||
{"Fal", "/providers/fal"},
|
||||
{"Fireworks", "/providers/fireworks"},
|
||||
{"Inferrs", "/providers/inferrs"},
|
||||
{"Meta", "/providers/meta"},
|
||||
{"Runway", "/providers/runway"},
|
||||
{"Synthetic", "/providers/synthetic"},
|
||||
{"Upstash Box", "/install/upstash"},
|
||||
{"Lobster", "/tools/lobster"},
|
||||
{"Mantis", "/concepts/mantis"},
|
||||
{"Tokenjuice", "/tools/tokenjuice"},
|
||||
}
|
||||
if len(cases) != len(contextualProtectedProductNames) {
|
||||
t.Fatalf("canonical destination cases=%d contextual names=%d", len(cases), len(contextualProtectedProductNames))
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if !isProtectedProductLinkLabel(tc.label, tc.destination) {
|
||||
t.Errorf("expected %q to be protected for %q", tc.label, tc.destination)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocBodyRejectsDroppedLinkMarkup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -51,10 +51,48 @@ func translationPrompt(srcLang, tgtLang string, glossary []GlossaryEntry) string
|
||||
prettyLanguageLabel(tgtLang),
|
||||
documentationQualityRules,
|
||||
localePromptRules(tgtLang),
|
||||
protectedProductNameRule(),
|
||||
buildGlossaryPrompt(glossary),
|
||||
))
|
||||
}
|
||||
|
||||
var alwaysProtectedProductNames = []string{
|
||||
"OpenClaw", "Raspberry Pi", "WhatsApp", "Telegram", "Discord", "iMessage", "Slack", "Microsoft Teams", "Google Chat", "Signal",
|
||||
}
|
||||
|
||||
var contextualProtectedProductNames = []string{
|
||||
"Render", "Matrix", "Raft", "Chutes", "fal", "Fal", "Fireworks", "Inferrs", "Meta", "Runway", "Synthetic", "Upstash Box", "Lobster", "Mantis", "Tokenjuice",
|
||||
}
|
||||
|
||||
func protectedProductNameRule() string {
|
||||
contextualDisplay := []string{
|
||||
"Render", "Matrix", "Raft", "Chutes", "fal (title: Fal)", "Fireworks", "Inferrs", "Meta", "Runway", "Synthetic", "Upstash Box", "Lobster", "Mantis", "Tokenjuice",
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"- Keep product names in English: %s. When they name the documented product, provider, protocol, integration, runtime, or plugin, also preserve ambiguous names exactly: %s. Translate the same words normally when the source clearly uses them as ordinary prose instead of a name.",
|
||||
strings.Join(alwaysProtectedProductNames, ", "),
|
||||
strings.Join(contextualDisplay, ", "),
|
||||
)
|
||||
}
|
||||
|
||||
func isAlwaysProtectedProductName(value string) bool {
|
||||
for _, name := range alwaysProtectedProductNames {
|
||||
if value == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func contextualProtectedProductName(value string) (string, bool) {
|
||||
for _, name := range contextualProtectedProductNames {
|
||||
if value == name {
|
||||
return name, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
var localeRules = map[string]string{
|
||||
"zh-cn": `Locale rules:
|
||||
- Use Simplified Chinese, mainland technical terminology, and simplified characters. Use “你/你的”, not “您/您的”.
|
||||
@@ -137,6 +175,7 @@ Rules:
|
||||
- Do not translate or modify code spans, executable code or config blocks, config keys, CLI flags, environment variables, commands, or placeholders such as __OC_I18N_####__.
|
||||
- Fenced text, transcript, output, and documentation examples are an exception to the preceding block rule: preserve angle-bracket placeholders, square-bracket config/protocol markers, and double-bracket directive tokens exactly, but translate ordinary human prose, including prose surrounding protected directive tokens.
|
||||
- Do not alter URLs, anchors, path fragments, or identifier spelling.
|
||||
- Preserve link-label association: translate each Markdown link label in place. Never move link markup to a different word or entity, and never replace a protected product-name label with a neighboring product name.
|
||||
- Do not remove, reorder, merge, summarize, or duplicate content.
|
||||
- Use fluent, idiomatic technical language in the target language with a neutral documentation tone; avoid slang and jokes.
|
||||
%s
|
||||
@@ -145,7 +184,7 @@ Rules:
|
||||
|
||||
- Glossary terms are mandatory under the label precedence rules above. When a source term matches a glossary entry, use its target exactly, including headings, link labels, and short UI-style labels.
|
||||
- If a glossary target is identical to the source text, preserve that term exactly as written.
|
||||
- Keep product names in English: OpenClaw, Raspberry Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal. When they name the documented product, provider, protocol, integration, runtime, or plugin, also preserve ambiguous names exactly: Render, Matrix, Raft, Chutes, fal (title: Fal), Fireworks, Inferrs, Meta, Runway, Synthetic, Upstash Box, Lobster, Mantis, Tokenjuice. Translate the same words normally when the source clearly uses them as ordinary prose instead of a name.
|
||||
%s
|
||||
- Never output an empty response; if unsure, return the source text unchanged.
|
||||
|
||||
%s
|
||||
|
||||
@@ -46,6 +46,7 @@ func TestTranslationPromptUsesSharedContractAndLocaleOverlayForEverySupportedLoc
|
||||
"Preserve Markdown list nodes exactly: ordered versus unordered kind, nesting, item count, and ordered-list starting number",
|
||||
"Preserve HTML/MDX tag names, attribute names, nesting, and structural attribute values exactly",
|
||||
"Fenced text, transcript, output, and documentation examples are an exception to the preceding block rule",
|
||||
"Preserve link-label association: translate each Markdown link label in place",
|
||||
"Translate user-visible prose inside string-valued component attributes such as “title”, “label”, “description”, and “placeholder”",
|
||||
"When they name the documented product, provider, protocol, integration, runtime, or plugin, also preserve ambiguous names exactly: Render, Matrix, Raft, Chutes, fal (title: Fal), Fireworks, Inferrs, Meta, Runway, Synthetic, Upstash Box, Lobster, Mantis, Tokenjuice",
|
||||
"Translate the same words normally when the source clearly uses them as ordinary prose instead of a name",
|
||||
@@ -62,6 +63,17 @@ func TestTranslationPromptUsesSharedContractAndLocaleOverlayForEverySupportedLoc
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectedProductNameRuleCoversValidatedNames(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rule := protectedProductNameRule()
|
||||
for _, name := range append(append([]string{}, alwaysProtectedProductNames...), contextualProtectedProductNames...) {
|
||||
if !strings.Contains(rule, name) {
|
||||
t.Errorf("protected product name %q missing from prompt rule", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslationPromptDistinguishesDisplayAndStructuralAttributes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
const (
|
||||
workflowVersion = 16
|
||||
promptVersion = 25
|
||||
promptVersion = 26
|
||||
docsI18nEngineName = "codex"
|
||||
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
|
||||
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"
|
||||
|
||||
Reference in New Issue
Block a user