Abrowser: updates for v91, new process-json-files.py
This commit is contained in:
Ruben Rodriguez
parent
8dab7d172b
commit
b2a945fe39
3 changed files with 241 additions and 55 deletions
238
helpers/DATA/firefox/process-json-files.py
Normal file
238
helpers/DATA/firefox/process-json-files.py
Normal file
|
|
@ -0,0 +1,238 @@
|
||||||
|
#! /usr/bin/python3
|
||||||
|
|
||||||
|
# Copyright (C) 2020, 2021 grizzlyuser <grizzlyuser@protonmail.com>
|
||||||
|
# Copyright (C) 2020, 2021 Ruben Rodriguez <ruben@trisquel.info>
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation; either version 2 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import copy
|
||||||
|
import argparse
|
||||||
|
import pathlib
|
||||||
|
from collections import namedtuple
|
||||||
|
from jsonschema import validate
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
'MAIN_PATH',
|
||||||
|
type=pathlib.Path,
|
||||||
|
help='path to main application source code directory')
|
||||||
|
parser.add_argument(
|
||||||
|
'BRANDING_PATH',
|
||||||
|
type=pathlib.Path,
|
||||||
|
help='path to branding source code directory')
|
||||||
|
parser.add_argument(
|
||||||
|
'-i',
|
||||||
|
'--indent',
|
||||||
|
type=int,
|
||||||
|
default=2,
|
||||||
|
help='indent for pretty printing of output files')
|
||||||
|
arguments = parser.parse_args()
|
||||||
|
|
||||||
|
File = namedtuple('File', ['path', 'content'])
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteSettings:
|
||||||
|
DUMPS_PATH_RELATIVE = 'services/settings/dumps'
|
||||||
|
DUMPS_PATH_ABSOLUTE = arguments.MAIN_PATH / DUMPS_PATH_RELATIVE
|
||||||
|
|
||||||
|
_WRAPPER_NAME = 'data'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def wrap(cls, processed):
|
||||||
|
return File(processed.path, {cls._WRAPPER_NAME: processed.content})
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def unwrap(cls, parsed_jsons):
|
||||||
|
return [File(json.path, json.content[cls._WRAPPER_NAME])
|
||||||
|
for json in parsed_jsons]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def should_modify_collection(cls, collection):
|
||||||
|
return True
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def now(cls):
|
||||||
|
return int(round(time.time() / 10 ** 6))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def process_raw(cls, unwrapped_jsons, parsed_schema):
|
||||||
|
timestamps, result = [], []
|
||||||
|
for collection in unwrapped_jsons:
|
||||||
|
should_modify_collection = cls.should_modify_collection(collection)
|
||||||
|
for record in collection.content:
|
||||||
|
if should_modify_collection:
|
||||||
|
if cls.should_drop_record(record):
|
||||||
|
continue
|
||||||
|
|
||||||
|
clone = copy.deepcopy(record)
|
||||||
|
|
||||||
|
record = cls.process_record(record)
|
||||||
|
|
||||||
|
if clone != record:
|
||||||
|
timestamp = cls.now()
|
||||||
|
while timestamp in timestamps:
|
||||||
|
timestamp += 1
|
||||||
|
timestamps.append(timestamp)
|
||||||
|
record['last_modified'] = timestamp
|
||||||
|
|
||||||
|
if parsed_schema is not None:
|
||||||
|
validate(record, schema=parsed_schema)
|
||||||
|
|
||||||
|
result.append(record)
|
||||||
|
|
||||||
|
cls.OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
return File(cls.OUTPUT_PATH, result)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def process(cls, parsed_jsons, parsed_schema):
|
||||||
|
return cls.wrap(
|
||||||
|
cls.process_raw(
|
||||||
|
cls.unwrap(parsed_jsons),
|
||||||
|
parsed_schema))
|
||||||
|
|
||||||
|
|
||||||
|
class Changes(RemoteSettings):
|
||||||
|
JSON_PATHS = tuple(RemoteSettings.DUMPS_PATH_ABSOLUTE.glob('*/*.json'))
|
||||||
|
OUTPUT_PATH = RemoteSettings.DUMPS_PATH_ABSOLUTE / 'monitor/changes.json'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def wrap(cls, processed):
|
||||||
|
return File(
|
||||||
|
processed.path, {
|
||||||
|
'changes': processed.content, 'timestamp': cls.now()})
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def process_raw(cls, unwrapped_jsons, parsed_schema):
|
||||||
|
changes = []
|
||||||
|
|
||||||
|
for collection in unwrapped_jsons:
|
||||||
|
if collection.path != RemoteSettings.DUMPS_PATH_ABSOLUTE / 'main/example.json':
|
||||||
|
latest_change = {}
|
||||||
|
latest_change['last_modified'] = max(
|
||||||
|
(record['last_modified'] for record in collection.content), default=0)
|
||||||
|
latest_change['bucket'] = collection.path.parent.name
|
||||||
|
latest_change['collection'] = collection.path.stem
|
||||||
|
changes.append(latest_change)
|
||||||
|
|
||||||
|
cls.OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
return File(cls.OUTPUT_PATH, changes)
|
||||||
|
|
||||||
|
|
||||||
|
class SearchConfig(RemoteSettings):
|
||||||
|
JSON_PATHS = (
|
||||||
|
RemoteSettings.DUMPS_PATH_ABSOLUTE /
|
||||||
|
'main/search-config.json',
|
||||||
|
)
|
||||||
|
SCHEMA_PATH = arguments.MAIN_PATH / \
|
||||||
|
'toolkit/components/search/schema/search-engine-config-schema.json'
|
||||||
|
OUTPUT_PATH = JSON_PATHS[0]
|
||||||
|
|
||||||
|
_DUCKDUCKGO_SEARCH_ENGINE_ID = 'ddg@search.mozilla.org'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def should_drop_record(cls, search_engine):
|
||||||
|
return search_engine['webExtension']['id'] not in (
|
||||||
|
cls._DUCKDUCKGO_SEARCH_ENGINE_ID, 'wikipedia@search.mozilla.org')
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def process_record(cls, search_engine):
|
||||||
|
[search_engine.pop(key, None)
|
||||||
|
for key in ['extraParams', 'telemetryId']]
|
||||||
|
|
||||||
|
general_specifier = {}
|
||||||
|
for specifier in search_engine['appliesTo'].copy():
|
||||||
|
if 'application' in specifier:
|
||||||
|
if 'distributions' in specifier['application']:
|
||||||
|
search_engine['appliesTo'].remove(specifier)
|
||||||
|
continue
|
||||||
|
specifier['application'].pop('extraParams', None)
|
||||||
|
|
||||||
|
if 'included' in specifier and 'everywhere' in specifier[
|
||||||
|
'included'] and specifier['included']['everywhere']:
|
||||||
|
general_specifier = specifier
|
||||||
|
|
||||||
|
if not general_specifier:
|
||||||
|
general_specifier = {'included': {'everywhere': True}}
|
||||||
|
search_engine['appliesTo'].insert(0, general_specifier)
|
||||||
|
if search_engine['webExtension']['id'] == cls._DUCKDUCKGO_SEARCH_ENGINE_ID:
|
||||||
|
general_specifier['default'] = 'yes'
|
||||||
|
|
||||||
|
return search_engine
|
||||||
|
|
||||||
|
|
||||||
|
class TippyTopSites:
|
||||||
|
JSON_PATHS = (
|
||||||
|
arguments.MAIN_PATH /
|
||||||
|
'browser/components/newtab/data/content/tippytop/top_sites.json',
|
||||||
|
arguments.BRANDING_PATH /
|
||||||
|
'tippytop/top_sites.json')
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def process(cls, parsed_jsons, parsed_schema):
|
||||||
|
tippy_top_sites_main = parsed_jsons[0]
|
||||||
|
tippy_top_sites_branding = parsed_jsons[1]
|
||||||
|
result = tippy_top_sites_branding.content + \
|
||||||
|
[site for site in tippy_top_sites_main.content if 'wikipedia.org' in site['domains']]
|
||||||
|
return File(tippy_top_sites_main.path, result)
|
||||||
|
|
||||||
|
|
||||||
|
class TopSites(RemoteSettings):
|
||||||
|
_TOP_SITES_JSON_PATH = 'main/top-sites.json'
|
||||||
|
_TOP_SITES_PATH_MAIN = RemoteSettings.DUMPS_PATH_ABSOLUTE / _TOP_SITES_JSON_PATH
|
||||||
|
|
||||||
|
JSON_PATHS = (
|
||||||
|
arguments.BRANDING_PATH /
|
||||||
|
RemoteSettings.DUMPS_PATH_RELATIVE /
|
||||||
|
_TOP_SITES_JSON_PATH,
|
||||||
|
_TOP_SITES_PATH_MAIN)
|
||||||
|
OUTPUT_PATH = _TOP_SITES_PATH_MAIN
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def should_modify_collection(cls, collection):
|
||||||
|
return cls._TOP_SITES_PATH_MAIN == collection.path
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def should_drop_record(cls, site):
|
||||||
|
return site['url'] != 'https://www.wikipedia.org/'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def process_record(cls, site):
|
||||||
|
site.pop('exclude_regions', None)
|
||||||
|
return site
|
||||||
|
|
||||||
|
|
||||||
|
# To reflect the latest timestamps, Changes class should always come after
|
||||||
|
# all other RemoteSettings subclasses
|
||||||
|
processors = (SearchConfig, Changes)
|
||||||
|
|
||||||
|
for processor in processors:
|
||||||
|
parsed_jsons = []
|
||||||
|
for json_path in processor.JSON_PATHS:
|
||||||
|
with json_path.open(encoding='utf-8') as file:
|
||||||
|
parsed_jsons.append(File(json_path, json.load(file)))
|
||||||
|
|
||||||
|
parsed_schema = None
|
||||||
|
if hasattr(processor, "SCHEMA_PATH"):
|
||||||
|
with processor.SCHEMA_PATH.open() as file:
|
||||||
|
parsed_schema = json.load(file)
|
||||||
|
|
||||||
|
processed = processor.process(parsed_jsons, parsed_schema)
|
||||||
|
with processed.path.open('w') as file:
|
||||||
|
json.dump(processed.content, file, indent=arguments.indent)
|
||||||
|
|
@ -1,52 +0,0 @@
|
||||||
#! /usr/bin/python3
|
|
||||||
|
|
||||||
# Copyright (C) 2020 Ruben Rodriguez <ruben@trisquel.info>
|
|
||||||
#
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation; either version 2 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program; if not, write to the Free Software
|
|
||||||
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|
||||||
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
|
|
||||||
data={}
|
|
||||||
|
|
||||||
with open(sys.argv[1]) as f:
|
|
||||||
data=json.load(f)
|
|
||||||
|
|
||||||
whitelist=['ddg@search.mozilla.org','google@search.mozilla.org','wikipedia@search.mozilla.org','bing@search.mozilla.org']
|
|
||||||
|
|
||||||
i=0
|
|
||||||
newdata={"data":[]}
|
|
||||||
trisquel={}
|
|
||||||
|
|
||||||
for item in data["data"]:
|
|
||||||
if item["webExtension"]["id"] in whitelist:
|
|
||||||
item["appliesTo"][0]["included"]["everywhere"]=True
|
|
||||||
item["appliesTo"][0]["default"]="no"
|
|
||||||
if item["webExtension"]["id"] == 'ddg@search.mozilla.org':
|
|
||||||
item["appliesTo"][0]["default"]="yes"
|
|
||||||
item["appliesTo"][0]["orderHint"]=5000
|
|
||||||
item["appliesTo"][0]["override"]=True
|
|
||||||
# del item["appliesTo"][1]["application"]["distributions"]
|
|
||||||
# del item["appliesTo"][1]["extraParams"]
|
|
||||||
# del item["extraParams"]
|
|
||||||
newdata["data"].append(item)
|
|
||||||
i+=1
|
|
||||||
trisquel={u'webExtension': {u'id': u'trisquel@search.mozilla.org'}, u'appliesTo': [{u'included': {u'everywhere': True}, 'default': 'no'}, {u'override': True, u'application': {'override': True, 'orderHint': 4000}}], u'id': u'4341e834-7290-4d33-beb0-377c04a49566', u'last_modified': 1595254832054, u'telemetryId': u'trisquel', u'schema': 1594312388241}
|
|
||||||
ddghtml={u'webExtension': {u'id': u'ddg-html@search.mozilla.org'}, u'appliesTo': [{u'included': {u'everywhere': True}, 'default': 'no'}, {u'override': True, u'application': {'override': True, 'orderHint': 3000}}], u'id': u'55bf6437-3b82-41a6-98be-09c3b53b5b4d', u'last_modified': 1595254832054, u'telemetryId': u'ddg-html', u'schema': 1594312388241}
|
|
||||||
newdata["data"].append(trisquel)
|
|
||||||
newdata["data"].append(ddghtml)
|
|
||||||
|
|
||||||
with open(sys.argv[1], 'w') as outfile:
|
|
||||||
json.dump(newdata, outfile, indent=4)
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
#
|
#
|
||||||
# Copyright (C) 2008-2020 Ruben Rodriguez <ruben@trisquel.info>
|
# Copyright (C) 2008-2021 Ruben Rodriguez <ruben@trisquel.info>
|
||||||
# Copyright (C) 2015 Santiago Rodriguez <santi@trisquel.info>
|
# Copyright (C) 2015 Santiago Rodriguez <santi@trisquel.info>
|
||||||
#
|
#
|
||||||
# This program is free software; you can redistribute it and/or modify
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||||
#
|
#
|
||||||
|
|
||||||
VERSION=80
|
VERSION=81
|
||||||
|
|
||||||
. ./config
|
. ./config
|
||||||
|
|
||||||
|
|
@ -206,7 +206,7 @@ sed 's|ddg@|ddg-html@|' -i browser/components/search/extensions/ddg-html/manifes
|
||||||
#sed '/search/s|q=|k1=-1\&kd=-1\&ko=1\&q=|' -i browser/components/search/extensions/ddg/manifest.json
|
#sed '/search/s|q=|k1=-1\&kd=-1\&ko=1\&q=|' -i browser/components/search/extensions/ddg/manifest.json
|
||||||
|
|
||||||
# Reprocess search preconfiguration dump
|
# Reprocess search preconfiguration dump
|
||||||
python $DATA/reprocess-search-config.py ./services/settings/dumps/main/search-config.json
|
python3 $DATA/process-json-files.py . browser/components/extensions/schemas/
|
||||||
|
|
||||||
cat << EOF > debian/distribution.ini
|
cat << EOF > debian/distribution.ini
|
||||||
[Global]
|
[Global]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue