# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import re from collections import defaultdict from fluent.syntax import ast as ftl from fluent.syntax.serializer import serialize_variant_key from fluent.syntax.visitor import Visitor from .base import Checker, CSSCheckMixin from compare_locales import plurals MSGS = { "missing-msg-ref": "Missing message reference: {ref}", "missing-term-ref": "Missing term reference: {ref}", "obsolete-msg-ref": "Obsolete message reference: {ref}", "obsolete-term-ref": "Obsolete term reference: {ref}", "duplicate-attribute": 'Attribute "{name}" is duplicated', "missing-value": "Missing value", "obsolete-value": "Obsolete value", "missing-attribute": "Missing attribute: {name}", "obsolete-attribute": "Obsolete attribute: {name}", "duplicate-variant": 'Variant key "{name}" is duplicated', "missing-plural": "Plural categories missing: {categories}", "plain-message": "{message}", } def pattern_variants(pattern): """Get variants of plain text of a pattern. For now, just return simple text patterns. This can be improved to allow for SelectExpressions of simple text patterns, or even nested expressions, and Literals. Variants with Variable-, Message-, or TermReferences should be ignored. """ elements = pattern.elements if len(elements) == 1: if isinstance(elements[0], ftl.TextElement): return [elements[0].value] return [] class ReferenceMessageVisitor(Visitor, CSSCheckMixin): def __init__(self): # References to Messages, their Attributes, and Terms # Store reference name and type self.entry_refs = defaultdict(dict) # The currently active references self.refs = {} # Start with the Entry value (associated with None) self.entry_refs[None] = self.refs # If we're a messsage, store if there was a value self.message_has_value = False # Map attribute names to positions self.attribute_positions = {} # Map of CSS style attribute properties and units self.css_styles = None self.css_errors = None def generic_visit(self, node): if isinstance(node, (ftl.Span, ftl.Annotation, ftl.BaseComment)): return super().generic_visit(node) def visit_Message(self, node): if node.value is not None: self.message_has_value = True super().generic_visit(node) def visit_Attribute(self, node): self.attribute_positions[node.id.name] = node.span.start old_refs = self.refs self.refs = self.entry_refs[node.id.name] super().generic_visit(node) self.refs = old_refs if node.id.name != "style": return text_values = pattern_variants(node.value) if not text_values: self.css_styles = "skip" return # right now, there's just one possible text value self.css_styles, self.css_errors = self.parse_css_spec(text_values[0]) def visit_SelectExpression(self, node): # optimize select expressions to only go through the variants self.visit(node.variants) def visit_MessageReference(self, node): ref = node.id.name if node.attribute: ref += "." + node.attribute.name self.refs[ref] = "msg-ref" def visit_TermReference(self, node): # only collect term references, but not attributes of terms if node.attribute: return self.refs["-" + node.id.name] = "term-ref" class GenericL10nChecks: """Helper Mixin for checks shared between Terms and Messages.""" def check_duplicate_attributes(self, node): warned = set() for left in range(len(node.attributes) - 1): if left in warned: continue left_attr = node.attributes[left] warned_left = False for right in range(left + 1, len(node.attributes)): right_attr = node.attributes[right] if left_attr.id.name == right_attr.id.name: if not warned_left: warned_left = True self.messages.append( ( "warning", left_attr.span.start, MSGS["duplicate-attribute"].format( name=left_attr.id.name ), ) ) warned.add(right) self.messages.append( ( "warning", right_attr.span.start, MSGS["duplicate-attribute"].format(name=left_attr.id.name), ) ) def check_variants(self, variants): # Check for duplicate variants warned = set() for left in range(len(variants) - 1): if left in warned: continue left_key = variants[left].key key_string = None for right in range(left + 1, len(variants)): if left_key.equals(variants[right].key): if key_string is None: key_string = serialize_variant_key(left_key) self.messages.append( ( "warning", left_key.span.start, MSGS["duplicate-variant"].format(name=key_string), ) ) warned.add(right) self.messages.append( ( "warning", variants[right].key.span.start, MSGS["duplicate-variant"].format(name=key_string), ) ) # Check for plural categories known_plurals = plurals.get_plural(self.locale) if known_plurals: known_plurals = set(known_plurals) # Ask for known plurals, but check for plurals w/out `other`. # `other` is used for all kinds of things. check_plurals = known_plurals.copy() check_plurals.discard("other") given_plurals = {serialize_variant_key(v.key) for v in variants} if given_plurals & check_plurals: missing_plurals = sorted(known_plurals - given_plurals) if missing_plurals: self.messages.append( ( "warning", variants[0].key.span.start, MSGS["missing-plural"].format( categories=", ".join(missing_plurals) ), ) ) class L10nMessageVisitor(GenericL10nChecks, ReferenceMessageVisitor): def __init__(self, locale, reference): super().__init__() self.locale = locale # Overload refs to map to sets, just store what we found # References to Messages, their Attributes, and Terms # Store reference name and type self.entry_refs = defaultdict(set) # The currently active references self.refs = set() # Start with the Entry value (associated with None) self.entry_refs[None] = self.refs self.reference = reference self.reference_refs = reference.entry_refs[None] self.messages = [] def visit_Message(self, node): self.check_duplicate_attributes(node) super().visit_Message(node) if self.message_has_value and not self.reference.message_has_value: self.messages.append( ("error", node.value.span.start, MSGS["obsolete-value"]) ) if not self.message_has_value and self.reference.message_has_value: self.messages.append(("error", 0, MSGS["missing-value"])) ref_attrs = set(self.reference.attribute_positions) l10n_attrs = set(self.attribute_positions) for missing_attr in ref_attrs - l10n_attrs: self.messages.append( ("error", 0, MSGS["missing-attribute"].format(name=missing_attr)) ) for obs_attr in l10n_attrs - ref_attrs: self.messages.append( ( "error", self.attribute_positions[obs_attr], MSGS["obsolete-attribute"].format(name=obs_attr), ) ) def visit_Term(self, node): raise RuntimeError("Should not use L10nMessageVisitor for Terms") def visit_Attribute(self, node): old_reference_refs = self.reference_refs self.reference_refs = self.reference.entry_refs[node.id.name] super().visit_Attribute(node) self.reference_refs = old_reference_refs if node.id.name != "style" or self.css_styles == "skip": return ref_styles = self.reference.css_styles if ref_styles in ("skip", None): # Reference is complex, l10n isn't. # Let's still validate the css spec. ref_styles = {} for cat, msg, pos, _ in self.check_style( ref_styles, self.css_styles, self.css_errors ): self.messages.append((cat, msg, pos)) def visit_SelectExpression(self, node): super().visit_SelectExpression(node) self.check_variants(node.variants) def visit_MessageReference(self, node): ref = node.id.name if node.attribute: ref += "." + node.attribute.name self.refs.add(ref) self.check_obsolete_ref(node, ref, "msg-ref") def visit_TermReference(self, node): if node.attribute: return ref = "-" + node.id.name self.refs.add(ref) self.check_obsolete_ref(node, ref, "term-ref") def check_obsolete_ref(self, node, ref, ref_type): if ref not in self.reference_refs: self.messages.append( ( "warning", node.span.start, MSGS["obsolete-" + ref_type].format(ref=ref), ) ) class TermVisitor(GenericL10nChecks, Visitor): def __init__(self, locale): super().__init__() self.locale = locale self.messages = [] def generic_visit(self, node): if isinstance(node, (ftl.Span, ftl.Annotation, ftl.BaseComment)): return super().generic_visit(node) def visit_Message(self, node): raise RuntimeError("Should not use TermVisitor for Messages") def visit_Term(self, node): self.check_duplicate_attributes(node) super().generic_visit(node) def visit_SelectExpression(self, node): super().generic_visit(node) self.check_variants(node.variants) class FluentChecker(Checker): """Tests to run on Fluent (FTL) files.""" pattern = re.compile(r".*\.ftl") def check_message(self, ref_entry, l10n_entry): """Run checks on localized messages against reference message.""" ref_data = ReferenceMessageVisitor() ref_data.visit(ref_entry) l10n_data = L10nMessageVisitor(self.locale, ref_data) l10n_data.visit(l10n_entry) messages = l10n_data.messages for attr_or_val, refs in ref_data.entry_refs.items(): for ref, ref_type in refs.items(): if ref not in l10n_data.entry_refs[attr_or_val]: msg = MSGS["missing-" + ref_type].format(ref=ref) messages.append(("warning", 0, msg)) return messages def check_term(self, l10n_entry): """Check localized terms.""" l10n_data = TermVisitor(self.locale) l10n_data.visit(l10n_entry) return l10n_data.messages def check(self, refEnt, l10nEnt): yield from super().check(refEnt, l10nEnt) l10n_entry = l10nEnt.entry if isinstance(l10n_entry, ftl.Message): ref_entry = refEnt.entry messages = self.check_message(ref_entry, l10n_entry) elif isinstance(l10n_entry, ftl.Term): messages = self.check_term(l10n_entry) messages.sort(key=lambda t: t[1]) for cat, pos, msg in messages: if pos: pos = pos - l10n_entry.span.start yield (cat, pos, msg, "fluent")