340 lines
12 KiB
Python
340 lines
12 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import re
|
|
from collections import defaultdict
|
|
|
|
from fluent.syntax import ast as ftl
|
|
from fluent.syntax.serializer import serialize_variant_key
|
|
from fluent.syntax.visitor import Visitor
|
|
|
|
from .base import Checker, CSSCheckMixin
|
|
from compare_locales import plurals
|
|
|
|
|
|
MSGS = {
|
|
"missing-msg-ref": "Missing message reference: {ref}",
|
|
"missing-term-ref": "Missing term reference: {ref}",
|
|
"obsolete-msg-ref": "Obsolete message reference: {ref}",
|
|
"obsolete-term-ref": "Obsolete term reference: {ref}",
|
|
"duplicate-attribute": 'Attribute "{name}" is duplicated',
|
|
"missing-value": "Missing value",
|
|
"obsolete-value": "Obsolete value",
|
|
"missing-attribute": "Missing attribute: {name}",
|
|
"obsolete-attribute": "Obsolete attribute: {name}",
|
|
"duplicate-variant": 'Variant key "{name}" is duplicated',
|
|
"missing-plural": "Plural categories missing: {categories}",
|
|
"plain-message": "{message}",
|
|
}
|
|
|
|
|
|
def pattern_variants(pattern):
|
|
"""Get variants of plain text of a pattern.
|
|
|
|
For now, just return simple text patterns.
|
|
This can be improved to allow for SelectExpressions
|
|
of simple text patterns, or even nested expressions, and Literals.
|
|
Variants with Variable-, Message-, or TermReferences should be ignored.
|
|
"""
|
|
elements = pattern.elements
|
|
if len(elements) == 1:
|
|
if isinstance(elements[0], ftl.TextElement):
|
|
return [elements[0].value]
|
|
return []
|
|
|
|
|
|
class ReferenceMessageVisitor(Visitor, CSSCheckMixin):
|
|
def __init__(self):
|
|
# References to Messages, their Attributes, and Terms
|
|
# Store reference name and type
|
|
self.entry_refs = defaultdict(dict)
|
|
# The currently active references
|
|
self.refs = {}
|
|
# Start with the Entry value (associated with None)
|
|
self.entry_refs[None] = self.refs
|
|
# If we're a messsage, store if there was a value
|
|
self.message_has_value = False
|
|
# Map attribute names to positions
|
|
self.attribute_positions = {}
|
|
# Map of CSS style attribute properties and units
|
|
self.css_styles = None
|
|
self.css_errors = None
|
|
|
|
def generic_visit(self, node):
|
|
if isinstance(node, (ftl.Span, ftl.Annotation, ftl.BaseComment)):
|
|
return
|
|
super().generic_visit(node)
|
|
|
|
def visit_Message(self, node):
|
|
if node.value is not None:
|
|
self.message_has_value = True
|
|
super().generic_visit(node)
|
|
|
|
def visit_Attribute(self, node):
|
|
self.attribute_positions[node.id.name] = node.span.start
|
|
old_refs = self.refs
|
|
self.refs = self.entry_refs[node.id.name]
|
|
super().generic_visit(node)
|
|
self.refs = old_refs
|
|
if node.id.name != "style":
|
|
return
|
|
text_values = pattern_variants(node.value)
|
|
if not text_values:
|
|
self.css_styles = "skip"
|
|
return
|
|
# right now, there's just one possible text value
|
|
self.css_styles, self.css_errors = self.parse_css_spec(text_values[0])
|
|
|
|
def visit_SelectExpression(self, node):
|
|
# optimize select expressions to only go through the variants
|
|
self.visit(node.variants)
|
|
|
|
def visit_MessageReference(self, node):
|
|
ref = node.id.name
|
|
if node.attribute:
|
|
ref += "." + node.attribute.name
|
|
self.refs[ref] = "msg-ref"
|
|
|
|
def visit_TermReference(self, node):
|
|
# only collect term references, but not attributes of terms
|
|
if node.attribute:
|
|
return
|
|
self.refs["-" + node.id.name] = "term-ref"
|
|
|
|
|
|
class GenericL10nChecks:
|
|
"""Helper Mixin for checks shared between Terms and Messages."""
|
|
|
|
def check_duplicate_attributes(self, node):
|
|
warned = set()
|
|
for left in range(len(node.attributes) - 1):
|
|
if left in warned:
|
|
continue
|
|
left_attr = node.attributes[left]
|
|
warned_left = False
|
|
for right in range(left + 1, len(node.attributes)):
|
|
right_attr = node.attributes[right]
|
|
if left_attr.id.name == right_attr.id.name:
|
|
if not warned_left:
|
|
warned_left = True
|
|
self.messages.append(
|
|
(
|
|
"warning",
|
|
left_attr.span.start,
|
|
MSGS["duplicate-attribute"].format(
|
|
name=left_attr.id.name
|
|
),
|
|
)
|
|
)
|
|
warned.add(right)
|
|
self.messages.append(
|
|
(
|
|
"warning",
|
|
right_attr.span.start,
|
|
MSGS["duplicate-attribute"].format(name=left_attr.id.name),
|
|
)
|
|
)
|
|
|
|
def check_variants(self, variants):
|
|
# Check for duplicate variants
|
|
warned = set()
|
|
for left in range(len(variants) - 1):
|
|
if left in warned:
|
|
continue
|
|
left_key = variants[left].key
|
|
key_string = None
|
|
for right in range(left + 1, len(variants)):
|
|
if left_key.equals(variants[right].key):
|
|
if key_string is None:
|
|
key_string = serialize_variant_key(left_key)
|
|
self.messages.append(
|
|
(
|
|
"warning",
|
|
left_key.span.start,
|
|
MSGS["duplicate-variant"].format(name=key_string),
|
|
)
|
|
)
|
|
warned.add(right)
|
|
self.messages.append(
|
|
(
|
|
"warning",
|
|
variants[right].key.span.start,
|
|
MSGS["duplicate-variant"].format(name=key_string),
|
|
)
|
|
)
|
|
# Check for plural categories
|
|
known_plurals = plurals.get_plural(self.locale)
|
|
if known_plurals:
|
|
known_plurals = set(known_plurals)
|
|
# Ask for known plurals, but check for plurals w/out `other`.
|
|
# `other` is used for all kinds of things.
|
|
check_plurals = known_plurals.copy()
|
|
check_plurals.discard("other")
|
|
given_plurals = {serialize_variant_key(v.key) for v in variants}
|
|
if given_plurals & check_plurals:
|
|
missing_plurals = sorted(known_plurals - given_plurals)
|
|
if missing_plurals:
|
|
self.messages.append(
|
|
(
|
|
"warning",
|
|
variants[0].key.span.start,
|
|
MSGS["missing-plural"].format(
|
|
categories=", ".join(missing_plurals)
|
|
),
|
|
)
|
|
)
|
|
|
|
|
|
class L10nMessageVisitor(GenericL10nChecks, ReferenceMessageVisitor):
|
|
def __init__(self, locale, reference):
|
|
super().__init__()
|
|
self.locale = locale
|
|
# Overload refs to map to sets, just store what we found
|
|
# References to Messages, their Attributes, and Terms
|
|
# Store reference name and type
|
|
self.entry_refs = defaultdict(set)
|
|
# The currently active references
|
|
self.refs = set()
|
|
# Start with the Entry value (associated with None)
|
|
self.entry_refs[None] = self.refs
|
|
self.reference = reference
|
|
self.reference_refs = reference.entry_refs[None]
|
|
self.messages = []
|
|
|
|
def visit_Message(self, node):
|
|
self.check_duplicate_attributes(node)
|
|
super().visit_Message(node)
|
|
if self.message_has_value and not self.reference.message_has_value:
|
|
self.messages.append(
|
|
("error", node.value.span.start, MSGS["obsolete-value"])
|
|
)
|
|
if not self.message_has_value and self.reference.message_has_value:
|
|
self.messages.append(("error", 0, MSGS["missing-value"]))
|
|
ref_attrs = set(self.reference.attribute_positions)
|
|
l10n_attrs = set(self.attribute_positions)
|
|
for missing_attr in ref_attrs - l10n_attrs:
|
|
self.messages.append(
|
|
("error", 0, MSGS["missing-attribute"].format(name=missing_attr))
|
|
)
|
|
for obs_attr in l10n_attrs - ref_attrs:
|
|
self.messages.append(
|
|
(
|
|
"error",
|
|
self.attribute_positions[obs_attr],
|
|
MSGS["obsolete-attribute"].format(name=obs_attr),
|
|
)
|
|
)
|
|
|
|
def visit_Term(self, node):
|
|
raise RuntimeError("Should not use L10nMessageVisitor for Terms")
|
|
|
|
def visit_Attribute(self, node):
|
|
old_reference_refs = self.reference_refs
|
|
self.reference_refs = self.reference.entry_refs[node.id.name]
|
|
super().visit_Attribute(node)
|
|
self.reference_refs = old_reference_refs
|
|
if node.id.name != "style" or self.css_styles == "skip":
|
|
return
|
|
ref_styles = self.reference.css_styles
|
|
if ref_styles in ("skip", None):
|
|
# Reference is complex, l10n isn't.
|
|
# Let's still validate the css spec.
|
|
ref_styles = {}
|
|
for cat, msg, pos, _ in self.check_style(
|
|
ref_styles, self.css_styles, self.css_errors
|
|
):
|
|
self.messages.append((cat, msg, pos))
|
|
|
|
def visit_SelectExpression(self, node):
|
|
super().visit_SelectExpression(node)
|
|
self.check_variants(node.variants)
|
|
|
|
def visit_MessageReference(self, node):
|
|
ref = node.id.name
|
|
if node.attribute:
|
|
ref += "." + node.attribute.name
|
|
self.refs.add(ref)
|
|
self.check_obsolete_ref(node, ref, "msg-ref")
|
|
|
|
def visit_TermReference(self, node):
|
|
if node.attribute:
|
|
return
|
|
ref = "-" + node.id.name
|
|
self.refs.add(ref)
|
|
self.check_obsolete_ref(node, ref, "term-ref")
|
|
|
|
def check_obsolete_ref(self, node, ref, ref_type):
|
|
if ref not in self.reference_refs:
|
|
self.messages.append(
|
|
(
|
|
"warning",
|
|
node.span.start,
|
|
MSGS["obsolete-" + ref_type].format(ref=ref),
|
|
)
|
|
)
|
|
|
|
|
|
class TermVisitor(GenericL10nChecks, Visitor):
|
|
def __init__(self, locale):
|
|
super().__init__()
|
|
self.locale = locale
|
|
self.messages = []
|
|
|
|
def generic_visit(self, node):
|
|
if isinstance(node, (ftl.Span, ftl.Annotation, ftl.BaseComment)):
|
|
return
|
|
super().generic_visit(node)
|
|
|
|
def visit_Message(self, node):
|
|
raise RuntimeError("Should not use TermVisitor for Messages")
|
|
|
|
def visit_Term(self, node):
|
|
self.check_duplicate_attributes(node)
|
|
super().generic_visit(node)
|
|
|
|
def visit_SelectExpression(self, node):
|
|
super().generic_visit(node)
|
|
self.check_variants(node.variants)
|
|
|
|
|
|
class FluentChecker(Checker):
|
|
"""Tests to run on Fluent (FTL) files."""
|
|
|
|
pattern = re.compile(r".*\.ftl")
|
|
|
|
def check_message(self, ref_entry, l10n_entry):
|
|
"""Run checks on localized messages against reference message."""
|
|
ref_data = ReferenceMessageVisitor()
|
|
ref_data.visit(ref_entry)
|
|
l10n_data = L10nMessageVisitor(self.locale, ref_data)
|
|
l10n_data.visit(l10n_entry)
|
|
|
|
messages = l10n_data.messages
|
|
for attr_or_val, refs in ref_data.entry_refs.items():
|
|
for ref, ref_type in refs.items():
|
|
if ref not in l10n_data.entry_refs[attr_or_val]:
|
|
msg = MSGS["missing-" + ref_type].format(ref=ref)
|
|
messages.append(("warning", 0, msg))
|
|
return messages
|
|
|
|
def check_term(self, l10n_entry):
|
|
"""Check localized terms."""
|
|
l10n_data = TermVisitor(self.locale)
|
|
l10n_data.visit(l10n_entry)
|
|
return l10n_data.messages
|
|
|
|
def check(self, refEnt, l10nEnt):
|
|
yield from super().check(refEnt, l10nEnt)
|
|
l10n_entry = l10nEnt.entry
|
|
if isinstance(l10n_entry, ftl.Message):
|
|
ref_entry = refEnt.entry
|
|
messages = self.check_message(ref_entry, l10n_entry)
|
|
elif isinstance(l10n_entry, ftl.Term):
|
|
messages = self.check_term(l10n_entry)
|
|
|
|
messages.sort(key=lambda t: t[1])
|
|
for cat, pos, msg in messages:
|
|
if pos:
|
|
pos = pos - l10n_entry.span.start
|
|
yield (cat, pos, msg, "fluent")
|