fix(docs-i18n): preserve product link labels

This commit is contained in:
Peter Steinberger
2026-07-14 09:48:47 -04:00
parent 95c6addca2
commit 8cd9ad023b
5 changed files with 244 additions and 4 deletions
+109
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"net/url"
"os"
"regexp"
"slices"
@@ -37,6 +38,7 @@ type docChunkStructure struct {
listShapes []markdownListShape
inlineCodeSpans []string
linkDestinations []string
protectedLinkLabels []string
numericValues []string
fencedPlaceholders []string
fencedProtocolTokens []string
@@ -107,6 +109,9 @@ func validateDocBodyFencedLiterals(source, translated string) error {
if !sameStringMultiset(sourceStructure.linkDestinations, translatedStructure.linkDestinations) {
return fmt.Errorf("link destination mismatch: source=%d translated=%d", len(sourceStructure.linkDestinations), len(translatedStructure.linkDestinations))
}
if !sameStringMultiset(sourceStructure.protectedLinkLabels, translatedStructure.protectedLinkLabels) {
return fmt.Errorf("protected link label mismatch: source=%d translated=%d", len(sourceStructure.protectedLinkLabels), len(translatedStructure.protectedLinkLabels))
}
if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) {
return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues))
}
@@ -296,6 +301,9 @@ func validateDocChunkTranslation(source, translated string) error {
if !sameStringMultiset(sourceStructure.linkDestinations, translatedStructure.linkDestinations) {
return fmt.Errorf("link destination mismatch: source=%d translated=%d", len(sourceStructure.linkDestinations), len(translatedStructure.linkDestinations))
}
if !sameStringMultiset(sourceStructure.protectedLinkLabels, translatedStructure.protectedLinkLabels) {
return fmt.Errorf("protected link label mismatch: source=%d translated=%d", len(sourceStructure.protectedLinkLabels), len(translatedStructure.protectedLinkLabels))
}
if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) {
return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues))
}
@@ -460,6 +468,7 @@ func summarizeDocChunkStructure(text string) docChunkStructure {
listShapes: extractMarkdownListShapes(text),
inlineCodeSpans: extractMarkdownInlineCodeValues(text),
linkDestinations: extractMarkdownLinkDestinations(text),
protectedLinkLabels: extractProtectedMarkdownLinkLabels(text),
numericValues: extractNumericValues(text),
fencedPlaceholders: fencedPlaceholders,
fencedProtocolTokens: fencedProtocolTokens,
@@ -492,6 +501,106 @@ func extractMarkdownLinkDestinations(text string) []string {
return destinations
}
func extractProtectedMarkdownLinkLabels(text string) []string {
source := []byte(normalizeDocComponentsForMarkdownParse(text))
doc := parseDocsMarkdown(source)
labels := make([]string, 0)
_ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
if !entering {
return ast.WalkContinue, nil
}
kind := ""
destination := ""
switch link := node.(type) {
case *ast.Link:
kind = "link"
destination = string(link.Destination)
case *ast.Image:
kind = "image"
destination = string(link.Destination)
default:
return ast.WalkContinue, nil
}
label := strings.TrimSpace(string(node.Text(source)))
if isProtectedProductLinkLabel(label, destination) {
labels = append(labels, kind+":"+destination+":"+label)
}
return ast.WalkContinue, nil
})
return labels
}
func isProtectedProductLinkLabel(label, destination string) bool {
if isAlwaysProtectedProductName(label) {
return true
}
name, ok := contextualProtectedProductName(label)
return ok && destinationMentionsProductName(destination, name)
}
type contextualProductDestinationRule struct {
hosts []string
routes []string
}
var contextualProductDestinations = map[string]contextualProductDestinationRule{
"Render": {hosts: []string{"render.com"}, routes: []string{"/install/render"}},
"Matrix": {hosts: []string{"matrix.org"}, routes: []string{"/channels/matrix"}},
"Raft": {hosts: []string{"raft.build"}, routes: []string{"/channels/raft"}},
"Chutes": {hosts: []string{"chutes.ai"}, routes: []string{"/providers/chutes"}},
"fal": {hosts: []string{"fal.ai"}, routes: []string{"/providers/fal", "/plugins/reference/fal"}},
"Fal": {hosts: []string{"fal.ai"}, routes: []string{"/providers/fal", "/plugins/reference/fal"}},
"Fireworks": {hosts: []string{"fireworks.ai"}, routes: []string{"/providers/fireworks", "/plugins/reference/fireworks"}},
"Inferrs": {routes: []string{"/providers/inferrs", "/ericcurtin/inferrs"}},
"Meta": {hosts: []string{"meta.ai", "meta.com"}, routes: []string{"/providers/meta", "/plugins/reference/meta"}},
"Runway": {hosts: []string{"runwayml.com"}, routes: []string{"/providers/runway", "/plugins/reference/runway"}},
"Synthetic": {hosts: []string{"synthetic.new"}, routes: []string{"/providers/synthetic"}},
"Upstash Box": {hosts: []string{"upstash.com"}, routes: []string{"/install/upstash", "/docs/box"}},
"Lobster": {routes: []string{"/tools/lobster", "/openclaw/lobster"}},
"Mantis": {routes: []string{"/concepts/mantis", "/openclaw/mantis"}},
"Tokenjuice": {routes: []string{"/tools/tokenjuice", "/openclaw/tokenjuice"}},
}
func destinationMentionsProductName(destination, name string) bool {
rule, ok := contextualProductDestinations[name]
if !ok {
return false
}
parsed, err := url.Parse(destination)
if err != nil {
return false
}
host := strings.ToLower(parsed.Hostname())
for _, allowedHost := range rule.hosts {
if host == allowedHost || strings.HasSuffix(host, "."+allowedHost) {
return true
}
}
path := strings.ToLower(parsed.Path)
for _, route := range rule.routes {
if pathContainsRoute(path, route) {
return true
}
}
return false
}
func pathContainsRoute(path, route string) bool {
pathParts := nonemptyPathParts(path)
routeParts := nonemptyPathParts(route)
for start := 0; start+len(routeParts) <= len(pathParts); start++ {
if slices.Equal(pathParts[start:start+len(routeParts)], routeParts) {
return true
}
}
return false
}
func nonemptyPathParts(value string) []string {
parts := strings.Split(value, "/")
return slices.DeleteFunc(parts, func(part string) bool { return part == "" })
}
func parseDocsMarkdown(source []byte) ast.Node {
return goldmark.New(goldmark.WithExtensions(extension.GFM, extension.Footnote)).Parser().Parse(textpkg.NewReader(source))
}
+82 -2
View File
@@ -2325,8 +2325,8 @@ func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
t.Fatalf("expected translated read_when entry in output:\n%s", text)
}
if !strings.Contains(text, "prompt_version: 25") {
t.Fatalf("expected prompt version 25 in output metadata:\n%s", text)
if !strings.Contains(text, "prompt_version: 26") {
t.Fatalf("expected prompt version 26 in output metadata:\n%s", text)
}
}
@@ -2424,6 +2424,86 @@ func TestValidateDocChunkTranslationRejectsDroppedDuplicateLink(t *testing.T) {
}
}
func TestValidateDocChunkTranslationRejectsMovedProtectedProductLinkLabel(t *testing.T) {
t.Parallel()
source := "Deploy OpenClaw on [Render](https://render.com) using the Blueprint.\n"
translated := "Render पर Blueprint का उपयोग करके [OpenClaw](https://render.com) परिनियोजित करें।\n"
for name, validate := range map[string]func(string, string) error{
"chunk": validateDocChunkTranslation,
"final": validateDocBodyFencedLiterals,
} {
err := validate(source, translated)
if err == nil || !strings.Contains(err.Error(), "protected link label mismatch") {
t.Fatalf("%s: expected protected link label mismatch, got %v", name, err)
}
}
}
func TestValidateDocBodyAllowsTranslatedOrdinaryLinkLabel(t *testing.T) {
t.Parallel()
source := "Read the [deployment guide](/setup).\n"
translated := "Lesen Sie den [Bereitstellungsleitfaden](/setup).\n"
if err := validateDocBodyFencedLiterals(source, translated); err != nil {
t.Fatalf("expected translated ordinary link label to pass: %v", err)
}
}
func TestValidateDocBodyAllowsTranslatedContextualOrdinaryLinkLabel(t *testing.T) {
t.Parallel()
source := "[Render](/guides/pre-render) the page now.\n"
translated := "Die Seite jetzt [darstellen](/guides/pre-render).\n"
if err := validateDocBodyFencedLiterals(source, translated); err != nil {
t.Fatalf("expected ordinary contextual label to translate: %v", err)
}
}
func TestValidateDocBodyRejectsMovedProtectedReferenceLinkLabel(t *testing.T) {
t.Parallel()
source := "Deploy on [Render][provider].\n\n[provider]: https://render.com\n"
translated := "Auf Render mit [OpenClaw][provider] bereitstellen.\n\n[provider]: https://render.com\n"
err := validateDocBodyFencedLiterals(source, translated)
if err == nil || !strings.Contains(err.Error(), "protected link label mismatch") {
t.Fatalf("expected protected reference-link label mismatch, got %v", err)
}
}
func TestContextualProtectedProductLinksRecognizeCanonicalDestinations(t *testing.T) {
t.Parallel()
cases := []struct {
label string
destination string
}{
{"Render", "/install/render"},
{"Matrix", "/channels/matrix"},
{"Raft", "/channels/raft"},
{"Chutes", "/providers/chutes"},
{"fal", "/providers/fal"},
{"Fal", "/providers/fal"},
{"Fireworks", "/providers/fireworks"},
{"Inferrs", "/providers/inferrs"},
{"Meta", "/providers/meta"},
{"Runway", "/providers/runway"},
{"Synthetic", "/providers/synthetic"},
{"Upstash Box", "/install/upstash"},
{"Lobster", "/tools/lobster"},
{"Mantis", "/concepts/mantis"},
{"Tokenjuice", "/tools/tokenjuice"},
}
if len(cases) != len(contextualProtectedProductNames) {
t.Fatalf("canonical destination cases=%d contextual names=%d", len(cases), len(contextualProtectedProductNames))
}
for _, tc := range cases {
if !isProtectedProductLinkLabel(tc.label, tc.destination) {
t.Errorf("expected %q to be protected for %q", tc.label, tc.destination)
}
}
}
func TestValidateDocBodyRejectsDroppedLinkMarkup(t *testing.T) {
t.Parallel()
+40 -1
View File
@@ -51,10 +51,48 @@ func translationPrompt(srcLang, tgtLang string, glossary []GlossaryEntry) string
prettyLanguageLabel(tgtLang),
documentationQualityRules,
localePromptRules(tgtLang),
protectedProductNameRule(),
buildGlossaryPrompt(glossary),
))
}
var alwaysProtectedProductNames = []string{
"OpenClaw", "Raspberry Pi", "WhatsApp", "Telegram", "Discord", "iMessage", "Slack", "Microsoft Teams", "Google Chat", "Signal",
}
var contextualProtectedProductNames = []string{
"Render", "Matrix", "Raft", "Chutes", "fal", "Fal", "Fireworks", "Inferrs", "Meta", "Runway", "Synthetic", "Upstash Box", "Lobster", "Mantis", "Tokenjuice",
}
func protectedProductNameRule() string {
contextualDisplay := []string{
"Render", "Matrix", "Raft", "Chutes", "fal (title: Fal)", "Fireworks", "Inferrs", "Meta", "Runway", "Synthetic", "Upstash Box", "Lobster", "Mantis", "Tokenjuice",
}
return fmt.Sprintf(
"- Keep product names in English: %s. When they name the documented product, provider, protocol, integration, runtime, or plugin, also preserve ambiguous names exactly: %s. Translate the same words normally when the source clearly uses them as ordinary prose instead of a name.",
strings.Join(alwaysProtectedProductNames, ", "),
strings.Join(contextualDisplay, ", "),
)
}
func isAlwaysProtectedProductName(value string) bool {
for _, name := range alwaysProtectedProductNames {
if value == name {
return true
}
}
return false
}
func contextualProtectedProductName(value string) (string, bool) {
for _, name := range contextualProtectedProductNames {
if value == name {
return name, true
}
}
return "", false
}
var localeRules = map[string]string{
"zh-cn": `Locale rules:
- Use Simplified Chinese, mainland technical terminology, and simplified characters. Use “你/你的”, not “您/您的”.
@@ -137,6 +175,7 @@ Rules:
- Do not translate or modify code spans, executable code or config blocks, config keys, CLI flags, environment variables, commands, or placeholders such as __OC_I18N_####__.
- Fenced text, transcript, output, and documentation examples are an exception to the preceding block rule: preserve angle-bracket placeholders, square-bracket config/protocol markers, and double-bracket directive tokens exactly, but translate ordinary human prose, including prose surrounding protected directive tokens.
- Do not alter URLs, anchors, path fragments, or identifier spelling.
- Preserve link-label association: translate each Markdown link label in place. Never move link markup to a different word or entity, and never replace a protected product-name label with a neighboring product name.
- Do not remove, reorder, merge, summarize, or duplicate content.
- Use fluent, idiomatic technical language in the target language with a neutral documentation tone; avoid slang and jokes.
%s
@@ -145,7 +184,7 @@ Rules:
- Glossary terms are mandatory under the label precedence rules above. When a source term matches a glossary entry, use its target exactly, including headings, link labels, and short UI-style labels.
- If a glossary target is identical to the source text, preserve that term exactly as written.
- Keep product names in English: OpenClaw, Raspberry Pi, WhatsApp, Telegram, Discord, iMessage, Slack, Microsoft Teams, Google Chat, Signal. When they name the documented product, provider, protocol, integration, runtime, or plugin, also preserve ambiguous names exactly: Render, Matrix, Raft, Chutes, fal (title: Fal), Fireworks, Inferrs, Meta, Runway, Synthetic, Upstash Box, Lobster, Mantis, Tokenjuice. Translate the same words normally when the source clearly uses them as ordinary prose instead of a name.
%s
- Never output an empty response; if unsure, return the source text unchanged.
%s
+12
View File
@@ -46,6 +46,7 @@ func TestTranslationPromptUsesSharedContractAndLocaleOverlayForEverySupportedLoc
"Preserve Markdown list nodes exactly: ordered versus unordered kind, nesting, item count, and ordered-list starting number",
"Preserve HTML/MDX tag names, attribute names, nesting, and structural attribute values exactly",
"Fenced text, transcript, output, and documentation examples are an exception to the preceding block rule",
"Preserve link-label association: translate each Markdown link label in place",
"Translate user-visible prose inside string-valued component attributes such as “title”, “label”, “description”, and “placeholder”",
"When they name the documented product, provider, protocol, integration, runtime, or plugin, also preserve ambiguous names exactly: Render, Matrix, Raft, Chutes, fal (title: Fal), Fireworks, Inferrs, Meta, Runway, Synthetic, Upstash Box, Lobster, Mantis, Tokenjuice",
"Translate the same words normally when the source clearly uses them as ordinary prose instead of a name",
@@ -62,6 +63,17 @@ func TestTranslationPromptUsesSharedContractAndLocaleOverlayForEverySupportedLoc
}
}
func TestProtectedProductNameRuleCoversValidatedNames(t *testing.T) {
t.Parallel()
rule := protectedProductNameRule()
for _, name := range append(append([]string{}, alwaysProtectedProductNames...), contextualProtectedProductNames...) {
if !strings.Contains(rule, name) {
t.Errorf("protected product name %q missing from prompt rule", name)
}
}
}
func TestTranslationPromptDistinguishesDisplayAndStructuralAttributes(t *testing.T) {
t.Parallel()
+1 -1
View File
@@ -12,7 +12,7 @@ import (
const (
workflowVersion = 16
promptVersion = 25
promptVersion = 26
docsI18nEngineName = "codex"
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"