# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os from compare_locales import mozpath REFERENCE_LOCALE = "en-x-moz-reference" class ConfigList(list): def maybe_extend(self, other): """Add configs from other list if this list doesn't have this path yet.""" for config in other: if any(mine.path == config.path for mine in self): continue self.append(config) class ProjectFiles: """Iterable object to get all files and tests for a locale and a list of ProjectConfigs. If the given locale is None, iterate over reference files as both reference and locale for a reference self-test. """ def __init__(self, locale, projects, mergebase=None): self.locale = locale self.matchers = [] self.exclude = None self.mergebase = mergebase configs = ConfigList() excludes = ConfigList() for project in projects: # Only add this project if we're not in validation mode, # and the given locale is enabled for the project. if locale is not None and locale not in project.all_locales: continue configs.maybe_extend(project.configs) excludes.maybe_extend(project.excludes) # If an excluded config is explicitly included, drop if from the # excludes. excludes = [ exclude for exclude in excludes if not any(c.path == exclude.path for c in configs) ] if excludes: self.exclude = ProjectFiles(locale, excludes) for pc in configs: if locale and pc.locales is not None and locale not in pc.locales: continue for paths in pc.paths: if locale and "locales" in paths and locale not in paths["locales"]: continue m = { "l10n": paths["l10n"].with_env( {"locale": locale or REFERENCE_LOCALE} ), "module": paths.get("module"), } if "reference" in paths: m["reference"] = paths["reference"] if self.mergebase is not None: m["merge"] = paths["l10n"].with_env( {"locale": locale, "l10n_base": self.mergebase} ) m["test"] = set(paths.get("test", [])) if "locales" in paths: m["locales"] = paths["locales"][:] self.matchers.append(m) self.matchers.reverse() # we always iterate last first # Remove duplicate patterns, comparing each matcher # against all other matchers. # Avoid n^2 comparisons by only scanning the upper triangle # of a n x n matrix of all possible combinations. # Using enumerate and keeping track of indexes, as we can't # modify the list while iterating over it. drops = set() # duplicate matchers to remove for i, m in enumerate(self.matchers[:-1]): if i in drops: continue # we're dropping this anyway, don't search again for i_, m_ in enumerate(self.matchers[(i + 1) :]): if mozpath.realpath(m["l10n"].prefix) != mozpath.realpath( m_["l10n"].prefix ): # ok, not the same thing, continue continue if m["l10n"].pattern != m_["l10n"].pattern: # We cannot guess whether same entry until the pattern is # resolved, continue continue # check that we're comparing the same thing if "reference" in m: if mozpath.realpath(m["reference"].prefix) != mozpath.realpath( m_.get("reference").prefix ): raise RuntimeError( "Mismatch in reference for " + mozpath.realpath(m["l10n"].prefix) ) drops.add(i_ + i + 1) m["test"] |= m_["test"] drops = sorted(drops, reverse=True) for i in drops: del self.matchers[i] def __iter__(self): # The iteration is pretty different when we iterate over # a localization vs over the reference. We do that latter # when running in validation mode. inner = self.iter_locale() if self.locale else self.iter_reference() yield from inner def iter_locale(self): """Iterate over locale files.""" known = {} for matchers in self.matchers: matcher = matchers["l10n"] for path in self._files(matcher): if path not in known: known[path] = {"test": matchers.get("test")} if "reference" in matchers: known[path]["reference"] = matcher.sub( matchers["reference"], path ) if "merge" in matchers: known[path]["merge"] = matcher.sub(matchers["merge"], path) if "reference" not in matchers: continue matcher = matchers["reference"] for path in self._files(matcher): l10npath = matcher.sub(matchers["l10n"], path) if l10npath not in known: known[l10npath] = {"reference": path, "test": matchers.get("test")} if "merge" in matchers: known[l10npath]["merge"] = matcher.sub(matchers["merge"], path) for path, d in sorted(known.items()): yield (path, d.get("reference"), d.get("merge"), d["test"]) def iter_reference(self): """Iterate over reference files.""" # unset self.exclude, as we don't want that for our reference files exclude = self.exclude self.exclude = None known = {} for matchers in self.matchers: if "reference" not in matchers: continue matcher = matchers["reference"] for path in self._files(matcher): refpath = matcher.sub(matchers["reference"], path) if refpath not in known: known[refpath] = {"reference": path, "test": matchers.get("test")} for path, d in sorted(known.items()): yield (path, d.get("reference"), None, d["test"]) self.exclude = exclude def _files(self, matcher): """Base implementation of getting all files in a hierarchy using the file system. Subclasses might replace this method to support different IO patterns. """ base = matcher.prefix if self._isfile(base): if self.exclude and self.exclude.match(base) is not None: return if matcher.match(base) is not None: yield base return for d, dirs, files in self._walk(base): for f in files: p = mozpath.join(d, f) if self.exclude and self.exclude.match(p) is not None: continue if matcher.match(p) is not None: yield p def _isfile(self, path): return os.path.isfile(path) def _walk(self, base): yield from os.walk(base) def match(self, path): """Return the tuple of l10n_path, reference, mergepath, tests if the given path matches any config, otherwise None. This routine doesn't check that the files actually exist. """ if ( self.locale is not None and self.exclude and self.exclude.match(path) is not None ): return for matchers in self.matchers: matcher = matchers["l10n"] if self.locale is not None and matcher.match(path) is not None: ref = merge = None if "reference" in matchers: ref = matcher.sub(matchers["reference"], path) if "merge" in matchers: merge = matcher.sub(matchers["merge"], path) return path, ref, merge, matchers.get("test") if "reference" not in matchers: continue matcher = matchers["reference"] if matcher.match(path) is not None: merge = None l10n = matcher.sub(matchers["l10n"], path) if "merge" in matchers: merge = matcher.sub(matchers["merge"], path) return l10n, path, merge, matchers.get("test")