519 lines
18 KiB
YAML
519 lines
18 KiB
YAML
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
---
|
|
task-defaults:
|
|
max-run-time: 3600
|
|
suite: raptor
|
|
variants:
|
|
- geckoview-no-fission
|
|
run-on-projects: []
|
|
tier: 3
|
|
test-manifest-loader: null # don't load tests in the taskgraph
|
|
target:
|
|
by-app:
|
|
fenix:
|
|
by-test-platform:
|
|
android-hw.*-aarch64.*/.*:
|
|
upstream-task: signing-apk
|
|
name: public/build/target.arm64-v8a.apk
|
|
default:
|
|
upstream-task: signing-apk
|
|
name: public/build/target.armeabi-v7a.apk
|
|
refbrow:
|
|
by-test-platform:
|
|
android-hw.*-aarch64.*/.*:
|
|
index: mobile.v2.reference-browser.raptor.latest.arm64-v8a
|
|
name: target.arm64-v8a.apk
|
|
default:
|
|
index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a
|
|
name: target.armeabi-v7a.apk
|
|
default: geckoview_example.apk
|
|
virtualization: hardware
|
|
raptor:
|
|
activity:
|
|
by-app:
|
|
fenix: org.mozilla.fenix.IntentReceiverActivity
|
|
geckoview: org.mozilla.geckoview_example.GeckoViewActivity
|
|
refbrow: org.mozilla.reference.browser.BrowserTestActivity
|
|
default: None
|
|
binary-path:
|
|
by-app:
|
|
fenix: org.mozilla.fenix
|
|
geckoview: org.mozilla.geckoview_example
|
|
refbrow: org.mozilla.reference.browser.raptor
|
|
default: None
|
|
mozharness:
|
|
script: raptor_script.py
|
|
config:
|
|
- raptor/android_hw_config.py
|
|
extra-options:
|
|
# Bug 1635749, disable window recorder temporarily
|
|
- --browsertime-no-ffwindowrecorder
|
|
optimization:
|
|
skip-unless-backstop: null
|
|
limit-platforms:
|
|
by-app:
|
|
chrome-m:
|
|
- .*shippable-qr.*
|
|
cstm-car-m:
|
|
- .*shippable-qr.*
|
|
default: []
|
|
fetches:
|
|
toolchain:
|
|
by-app:
|
|
cstm-car-m:
|
|
- android-custom-car
|
|
- linux64-minidump-stackwalk
|
|
- linux64-fix-stacks
|
|
default:
|
|
- linux64-minidump-stackwalk
|
|
- linux64-fix-stacks
|
|
dependencies:
|
|
by-app:
|
|
fenix:
|
|
signing-apk: signing-apk-fenix-nightly-simulation
|
|
default: {}
|
|
|
|
browsertime-tp6m:
|
|
<<: &tp6m-defaults
|
|
description: Raptor (browsertime) tp6 page-load tests on android
|
|
max-run-time:
|
|
by-subtest:
|
|
reddit:
|
|
by-app:
|
|
geckoview: 7200
|
|
default: 3600
|
|
imdb:
|
|
by-app:
|
|
geckoview: 7200
|
|
default: 3600
|
|
default: 3600
|
|
mozharness:
|
|
extra-options:
|
|
- --chimera
|
|
# Bug 1884667 - Fresh profiles cause high frequency intermittents
|
|
- --conditioned-profile=settled
|
|
tier:
|
|
by-app:
|
|
geckoview: 1
|
|
fenix: 2
|
|
refbrow: 2
|
|
default: 3
|
|
treeherder-symbol: Btime(tp6m)
|
|
raptor:
|
|
apps: [geckoview, fenix, chrome-m, refbrow]
|
|
run-visual-metrics: true
|
|
test: tp6m
|
|
subtests:
|
|
- amazon
|
|
- bing
|
|
- [bing-search-restaurants, bing-s-r]
|
|
- booking
|
|
- cnn
|
|
- [cnn-ampstories, cnn-amp]
|
|
- dailymail
|
|
- [ebay-kleinanzeigen, ebay-k]
|
|
- [ebay-kleinanzeigen-search, ebay-k-s]
|
|
- [facebook-cristiano, fb-cris]
|
|
- [google-maps, gmaps]
|
|
- [google-search-restaurants, gsearch-r]
|
|
- instagram
|
|
- imdb
|
|
- reddit
|
|
- sina
|
|
- [stackoverflow, stacko]
|
|
- bild-de
|
|
- wikipedia
|
|
- youtube
|
|
run-on-projects:
|
|
by-app:
|
|
geckoview:
|
|
by-variant:
|
|
geckoview-no-fission:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: [trunk-only]
|
|
default: []
|
|
default:
|
|
by-subtest:
|
|
wikipedia:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: []
|
|
default: []
|
|
google-search-restaurants:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: []
|
|
default: []
|
|
default:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: []
|
|
default: []
|
|
fenix:
|
|
by-variant:
|
|
geckoview-no-fission:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: [trunk-only]
|
|
default: []
|
|
default: []
|
|
default: []
|
|
|
|
browsertime-tp6m-essential:
|
|
<<: *tp6m-defaults
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [geckoview, fenix, chrome-m, refbrow]
|
|
run-visual-metrics: true
|
|
test: tp6m
|
|
subtests:
|
|
- allrecipes
|
|
- [amazon-search, amazon-s]
|
|
- espn
|
|
# Bug 1682027 - Disabled due to high failure rate
|
|
# - facebook
|
|
- google
|
|
- [microsoft-support, micros-sup]
|
|
- [youtube-watch, youtube-w]
|
|
run-on-projects:
|
|
by-app:
|
|
geckoview:
|
|
by-subtest:
|
|
allrecipes:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable.*: [trunk]
|
|
default: []
|
|
default:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable.*: [trunk-only]
|
|
default: []
|
|
fenix:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable.*: [trunk-only]
|
|
default: []
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
geckoview: 1
|
|
refbrow: 2
|
|
default: 3
|
|
|
|
browsertime-tp6m-webextensions:
|
|
<<: *tp6m-defaults
|
|
description: Raptor (browsertime) tp6 page-load tests with webextensions installed on android
|
|
treeherder-symbol: Btime-webext(tp6m)
|
|
run-without-variant: false
|
|
mozharness:
|
|
extra-options:
|
|
- --chimera
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --conditioned-profile=settled-webext
|
|
- --browser-cycles=5
|
|
raptor:
|
|
apps: [fenix]
|
|
run-visual-metrics: true
|
|
test: tp6m
|
|
subtests:
|
|
- [amazon-search, amazon-s]
|
|
fetches:
|
|
fetch:
|
|
- icecat-addons
|
|
run-on-projects:
|
|
by-app:
|
|
fenix:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
# TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3.
|
|
tier: 3
|
|
|
|
browsertime-youtube-playback-mobile:
|
|
description: Browsertime YouTube Playback on Android
|
|
treeherder-symbol: Btime(ytp)
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [fenix, geckoview, refbrow]
|
|
subtests:
|
|
by-app:
|
|
fenix:
|
|
by-test-platform:
|
|
android-hw-a51.*:
|
|
- [youtube-playback-av1-sfr, ytp-av1-sfr]
|
|
- [youtube-playback-hfr, ytp-hfr]
|
|
- [youtube-playback-vp9-sfr, ytp-vp9-sfr]
|
|
# Bug 1699469 - Disabled until a crash is fixed.
|
|
# - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
|
|
# - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
|
|
default:
|
|
by-test-platform:
|
|
android-hw-a51.*:
|
|
- [youtube-playback-h264-sfr, ytp-h264-sfr]
|
|
- [youtube-playback-hfr, ytp-hfr]
|
|
# Bug 1699469 - Disabled until a crash is fixed.
|
|
# - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
|
|
# - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
|
|
test-url-param:
|
|
by-subtest:
|
|
youtube-playback-h264-sfr: # remove high resolution tests
|
|
exclude=1,2,
|
|
23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38
|
|
youtube-playback-hfr:
|
|
by-test-platform:
|
|
android-hw-a51.*: # remove VP9(1-34) and AV1(51-74) tests
|
|
exclude=1,2,
|
|
3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,
|
|
51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74
|
|
default: ""
|
|
youtube-playback-widevine-hfr:
|
|
by-test-platform:
|
|
android-hw-a51.*: # remove VP9(1-34) tests
|
|
exclude=1,2,
|
|
3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34
|
|
default: ""
|
|
default: ""
|
|
run-on-projects:
|
|
by-app:
|
|
fenix:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable.*: [mozilla-central]
|
|
default: []
|
|
geckoview:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
max-run-time: 3600
|
|
|
|
browsertime-tp6m-live:
|
|
<<: *tp6m-defaults
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [chrome-m, fenix, geckoview]
|
|
run-visual-metrics: true
|
|
test: tp6m
|
|
subtests:
|
|
- allrecipes
|
|
- amazon
|
|
- [amazon-search, amazon-s]
|
|
- bing
|
|
- [bing-search-restaurants, bing-s-r]
|
|
- booking
|
|
- booking-sf
|
|
- cnn
|
|
- [cnn-ampstories, cnn-amp]
|
|
- dailymail
|
|
- discord
|
|
- [ebay-kleinanzeigen, ebay-k]
|
|
- [ebay-kleinanzeigen-search, ebay-k-s]
|
|
- espn
|
|
- expedia
|
|
- facebook
|
|
- [facebook-cristiano, fb-cris]
|
|
- [fashionbeans, fashbea]
|
|
- google
|
|
- [google-accounts, gacc]
|
|
- [google-maps, gmaps]
|
|
- [google-search-restaurants, gsearch-r]
|
|
- instagram
|
|
- imdb
|
|
- [imdb-icecat, imdb-f]
|
|
- [medium-article, medium]
|
|
- [microsoft-support, micros-sup]
|
|
- nytimes
|
|
- [people-article, people]
|
|
- reddit
|
|
- [reddit-thread, reddit-thr]
|
|
- rumble-fox
|
|
- [stackoverflow, stacko]
|
|
- [stackoverflow-question, stack-q]
|
|
- [urbandictionary-define, urbict-def]
|
|
- bild-de
|
|
- [wikia-marvel, wikia-m]
|
|
- wikipedia
|
|
- youtube
|
|
- [youtube-watch, youtube-w]
|
|
mozharness:
|
|
extra-options:
|
|
- --live-sites
|
|
- --chimera
|
|
run-on-projects:
|
|
by-subtest:
|
|
cnn-ampstories:
|
|
by-app:
|
|
geckoview:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: [autoland]
|
|
default: []
|
|
default: []
|
|
default: []
|
|
tier: 3
|
|
treeherder-symbol: Btime-live(tp6m)
|
|
|
|
browsertime-tp6m-profiling:
|
|
description: Raptor (browsertime) tp6 page-load tests on android with Gecko profiling
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [geckoview]
|
|
test: tp6m
|
|
subtests: [amazon, youtube]
|
|
treeherder-symbol: Btime-Prof(tp6m)
|
|
mozharness:
|
|
script: raptor_script.py
|
|
config:
|
|
- raptor/android_hw_config.py
|
|
extra-options:
|
|
- --chimera
|
|
- --gecko-profile
|
|
run-on-projects:
|
|
by-test-platform:
|
|
android-hw-(a51).*aarch64-shippable-qr/opt: [mozilla-central]
|
|
default: []
|
|
|
|
browsertime-benchmark-speedometer-mobile:
|
|
description: Browsertime Speedometer on Android
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [geckoview, fenix, chrome-m, cstm-car-m]
|
|
tier:
|
|
by-app:
|
|
fenix: 1
|
|
default: 3
|
|
run-on-projects:
|
|
by-app:
|
|
fenix:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: [trunk]
|
|
default: []
|
|
geckoview:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: [trunk]
|
|
default: []
|
|
cstm-car-m:
|
|
by-test-platform:
|
|
android-hw-p6.*shippable-qr.*: []
|
|
android-hw-s21.*shippable-qr.*: []
|
|
default: [mozilla-central]
|
|
default: []
|
|
treeherder-symbol: Btime(sp)
|
|
mozharness:
|
|
extra-options:
|
|
- --test=speedometer
|
|
|
|
browsertime-benchmark-speedometer3-mobile:
|
|
description: Browsertime Speedometer 3 on Android
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [geckoview, fenix, chrome-m, cstm-car-m]
|
|
tier:
|
|
by-app:
|
|
fenix: 2
|
|
geckoview: 2
|
|
default: 3
|
|
run-on-projects:
|
|
by-app:
|
|
fenix:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: [trunk-only]
|
|
android-hw-p6.*shippable-qr.*: [mozilla-central]
|
|
android-hw-s21.*shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
geckoview:
|
|
by-test-platform:
|
|
android-hw-a51.*shippable-qr.*: [trunk-only]
|
|
android-hw-p6.*shippable-qr.*: [mozilla-central]
|
|
android-hw-s21.*shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
cstm-car-m: [mozilla-central]
|
|
chrome-m:
|
|
by-test-platform:
|
|
android-hw-p6.*shippable-qr.*: [mozilla-central]
|
|
android-hw-s21.*shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
treeherder-symbol: Btime(sp3)
|
|
mozharness:
|
|
extra-options:
|
|
- --test=speedometer3
|
|
|
|
browsertime-benchmark-motionmark:
|
|
description: Raptor (browsertime) Motionmark benchmark tests
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [fenix, chrome-m, geckoview]
|
|
subtests:
|
|
- [motionmark-animometer, mm-a]
|
|
- [motionmark-animometer-ramp, mm-ar]
|
|
- [motionmark-htmlsuite, mm-h]
|
|
- [motionmark-htmlsuite-ramp, mm-hr]
|
|
run-on-projects:
|
|
by-app:
|
|
chrome-m: []
|
|
default:
|
|
by-test-platform:
|
|
android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
treeherder-symbol: Btime()
|
|
tier: 2
|
|
|
|
browsertime-benchmark-motionmark-1-3:
|
|
description: Raptor (browsertime) Motionmark 1.3 benchmark tests
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [geckoview, fenix, chrome-m, cstm-car-m]
|
|
subtests:
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
run-on-projects:
|
|
by-app:
|
|
chrome-m: []
|
|
default:
|
|
by-test-platform:
|
|
android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
treeherder-symbol: Btime()
|
|
tier: 2
|
|
|
|
browsertime-benchmark-unity-webgl-mobile:
|
|
description: Browsertime Unity WebGL on Android
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [geckoview, refbrow, fenix, chrome-m]
|
|
tier:
|
|
by-app:
|
|
geckoview: 2
|
|
default: 3
|
|
treeherder-symbol: Btime(ugl)
|
|
run-on-projects:
|
|
by-app:
|
|
refbrow: []
|
|
geckoview:
|
|
by-test-platform:
|
|
# Disabled on autoland due to bug 1720063, and bug 1795826
|
|
android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
default:
|
|
by-test-platform:
|
|
android-hw-a51-.*aarch64-shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
max-run-time:
|
|
by-test-platform:
|
|
anrdroid-hw-a51.*: 1500
|
|
default: 900
|
|
mozharness:
|
|
extra-options:
|
|
- --test=unity-webgl
|
|
|
|
browsertime-power:
|
|
description: Browsertime Power Usage Tests on Android
|
|
run-without-variant: false
|
|
raptor:
|
|
apps: [geckoview, fenix, refbrow]
|
|
subtests:
|
|
- idle
|
|
- idle-bg
|
|
- [speedometer, sp]
|
|
tier: 2
|
|
run-on-projects: []
|
|
treeherder-symbol: Btime-P(power)
|
|
mozharness:
|
|
extra-options:
|
|
- --power-test
|