1305 lines
44 KiB
YAML
1305 lines
44 KiB
YAML
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
---
|
|
task-defaults:
|
|
max-run-time:
|
|
by-test-platform:
|
|
.*-qr/.*: 2400
|
|
default: 1800
|
|
suite: raptor
|
|
run-on-projects:
|
|
by-test-platform:
|
|
linux.*shippable.*: [mozilla-central]
|
|
default: []
|
|
limit-platforms:
|
|
by-app:
|
|
chrome:
|
|
- .*64.*shippable.*
|
|
safari:
|
|
- .*macosx.*shippable.*
|
|
safari-tp:
|
|
- macosx1400.*
|
|
- macosx1500.*
|
|
custom-car:
|
|
- macosx1400.*shippable.*
|
|
- macosx1470.*shippable.*
|
|
- macosx1500.*shippable.*
|
|
- linux.*-64.*shippable.*
|
|
- windows.*-64.*shippable.*
|
|
default: []
|
|
test-manifest-loader: null # don't load tests in the taskgraph
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
virtualization: hardware
|
|
mozharness:
|
|
script: raptor_script.py
|
|
config:
|
|
by-test-platform:
|
|
macosx.*:
|
|
- raptor/mac_config.py
|
|
windows.*:
|
|
- raptor/windows_config.py
|
|
default:
|
|
- raptor/linux_config.py
|
|
extra-options:
|
|
- --conditioned-profile=settled
|
|
use-caches:
|
|
by-test-platform:
|
|
windows.*: ["checkout"]
|
|
default: ["checkout", "pip", "uv"]
|
|
fetches:
|
|
toolchain:
|
|
by-app:
|
|
custom-car:
|
|
by-test-platform:
|
|
# this satisfied taskgraph for now, but once mac/win is implemented
|
|
# then linux will be the default
|
|
linux.*:
|
|
- linux64-custom-car
|
|
- linux64-minidump-stackwalk
|
|
- linux64-fix-stacks
|
|
windows.*-64.*:
|
|
- win64-custom-car
|
|
- win64-minidump-stackwalk
|
|
- win64-fix-stacks
|
|
macosx.*-aarch64.*:
|
|
- macosx-arm64-custom-car
|
|
- macosx64-aarch64-minidump-stackwalk
|
|
- macosx64-aarch64-fix-stacks
|
|
macosx.*-64.*:
|
|
- macosx-custom-car
|
|
- macosx64-minidump-stackwalk
|
|
- macosx64-fix-stacks
|
|
default: []
|
|
safari-tp:
|
|
by-test-platform:
|
|
macosx.*-aarch64.*:
|
|
- macosx64-aarch64-minidump-stackwalk
|
|
- macosx64-aarch64-fix-stacks
|
|
default: []
|
|
default:
|
|
by-test-platform:
|
|
windows.*-32.*.:
|
|
- win32-minidump-stackwalk
|
|
- win32-fix-stacks
|
|
windows.*-64.*:
|
|
- win64-minidump-stackwalk
|
|
- win64-fix-stacks
|
|
macosx.*-aarch64.*:
|
|
- macosx64-aarch64-minidump-stackwalk
|
|
- macosx64-aarch64-fix-stacks
|
|
default:
|
|
- linux64-minidump-stackwalk
|
|
- linux64-fix-stacks
|
|
|
|
browsertime-tp6:
|
|
<<: &tp6-defaults
|
|
description: Raptor (browsertime) tp6 page-load tests
|
|
limit-platforms:
|
|
by-app:
|
|
chrome:
|
|
- .*shippable.*
|
|
safari:
|
|
- .*macos.*shippable.*
|
|
custom-car:
|
|
- macosx1400.*shippable.*
|
|
- macosx1470.*shippable.*
|
|
- macosx1500.*shippable.*
|
|
- linux.*shippable.*
|
|
- windows.*-64.*shippable.*
|
|
default: []
|
|
max-run-time:
|
|
by-subtest:
|
|
imgur: 4800
|
|
default: 4000
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*64.*shippable.*:
|
|
- --chimera
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --conditioned-profile=settled
|
|
- --extra-profiler-run
|
|
default:
|
|
- --chimera
|
|
- --conditioned-profile=settled
|
|
- --extra-profiler-run
|
|
run-on-projects:
|
|
by-subtest:
|
|
netflix:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
linux.*shippable.*: [trunk]
|
|
windows.*-64.*shippable.*: [trunk]
|
|
macosx1470.*shippable.*: [trunk]
|
|
default: []
|
|
default: []
|
|
default:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [trunk]
|
|
default: []
|
|
default: []
|
|
treeherder-symbol: Btime(tp6)
|
|
raptor:
|
|
apps: [icecat, chrome, custom-car]
|
|
test: tp6
|
|
run-visual-metrics: true
|
|
subtests:
|
|
- buzzfeed
|
|
- ebay
|
|
- espn
|
|
- expedia
|
|
- [facebook, fb]
|
|
- [google-docs, gdocs]
|
|
- [google-mail, gmail]
|
|
- [google-search, gsearch]
|
|
- imdb
|
|
- imgur
|
|
- linkedin
|
|
- microsoft
|
|
- netflix
|
|
- nytimes
|
|
- office
|
|
- outlook
|
|
- paypal
|
|
- pinterest
|
|
- reddit
|
|
- tumblr
|
|
- twitch
|
|
- wikia
|
|
- youtube
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
|
|
browsertime-responsiveness:
|
|
<<: *tp6-defaults
|
|
raptor:
|
|
apps: [icecat, chrome]
|
|
test: tp6
|
|
run-visual-metrics: true
|
|
subtests:
|
|
- cnn-nav
|
|
- [facebook-nav, fbnav]
|
|
- [reddit-billgates-ama, rdt-ama]
|
|
- [reddit-billgates-post-1, rdt-post-1]
|
|
- [reddit-billgates-post-2, rdt-post-2]
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*:
|
|
- --cold
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --conditioned-profile=settled
|
|
- --extra-profiler-run
|
|
default:
|
|
- --cold
|
|
- --conditioned-profile=settled
|
|
- --extra-profiler-run
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [trunk]
|
|
default: []
|
|
default: []
|
|
|
|
browsertime-tp6-essential:
|
|
<<: *tp6-defaults
|
|
<<: &tp6-essential-defaults
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*-64.*shippable.*:
|
|
- --chimera
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --conditioned-profile=settled
|
|
- --collect-perfstats
|
|
- --extra-profiler-run
|
|
default:
|
|
- --chimera
|
|
- --conditioned-profile=settled
|
|
- --extra-profiler-run
|
|
raptor:
|
|
apps: [icecat, chrome, custom-car]
|
|
test: tp6
|
|
run-visual-metrics:
|
|
by-test-platform:
|
|
macosx1400.*shippable.*: false
|
|
macosx1500.*shippable.*: false
|
|
# Bug 1845014 Resolve osx 1300 vismet permafails
|
|
default: true
|
|
subtests:
|
|
- amazon
|
|
- [bing-search, bing]
|
|
- cnn
|
|
- fandom
|
|
- [google-slides, gslides]
|
|
- instagram
|
|
- twitter
|
|
- wikipedia
|
|
- yahoo-mail
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [trunk]
|
|
default: []
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
|
|
browsertime-tp6-bytecode:
|
|
<<: *tp6-defaults
|
|
<<: *tp6-essential-defaults
|
|
raptor:
|
|
apps: [icecat]
|
|
test: tp6
|
|
run-visual-metrics: true
|
|
subtests:
|
|
- amazon
|
|
- [bing-search, bing]
|
|
- cnn
|
|
- fandom
|
|
- [google-slides, gslides]
|
|
- instagram
|
|
- twitter
|
|
- wikipedia
|
|
- yahoo-mail
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [trunk]
|
|
default: []
|
|
default: []
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*-64.*shippable.*:
|
|
- --chimera
|
|
- --browsertime
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --conditioned-profile=settled
|
|
- --test-bytecode-cache
|
|
- --extra-profiler-run
|
|
default:
|
|
- --chimera
|
|
- --browsertime
|
|
- --conditioned-profile=settled
|
|
- --test-bytecode-cache
|
|
- --extra-profiler-run
|
|
treeherder-symbol: Btime-cache(tp6)
|
|
|
|
browsertime-tp7:
|
|
<<: &tp7-defaults
|
|
description: Raptor (browsertime) tp7 page-load tests
|
|
limit-platforms:
|
|
by-app:
|
|
chrome:
|
|
- .*shippable.*
|
|
safari:
|
|
- .*macos.*shippable.*
|
|
custom-car:
|
|
- macosx1400.*shippable.*
|
|
- macosx1470.*shippable.*
|
|
- macosx1500.*shippable.*
|
|
- linux.*shippable.*
|
|
- windows.*-64.*shippable.*
|
|
default: []
|
|
max-run-time:
|
|
by-subtest:
|
|
imgur: 4800
|
|
default: 4000
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*-64.*shippable.*:
|
|
- --chimera
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --conditioned-profile=settled
|
|
- --extra-profiler-run
|
|
default:
|
|
- --chimera
|
|
- --conditioned-profile=settled
|
|
- --extra-profiler-run
|
|
run-on-projects: []
|
|
treeherder-symbol: Btime(tp7)
|
|
raptor:
|
|
apps: [icecat, chrome]
|
|
test: tp7
|
|
run-visual-metrics: true
|
|
subtests:
|
|
- tiktok
|
|
- openai
|
|
- samsung
|
|
- weather
|
|
- docomo
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
|
|
browsertime-benchmark:
|
|
description: Raptor (browsertime) Benchmark tests
|
|
raptor:
|
|
apps: [icecat, chrome, safari, safari-tp, custom-car]
|
|
subtests:
|
|
by-test-platform:
|
|
(macosx1400|macosx1500).*shippable.*:
|
|
by-app:
|
|
safari-tp:
|
|
- [speedometer3, sp3]
|
|
# Bug 1931699, jetstream permafailing
|
|
# - [jetstream2, js2]
|
|
- [jetstream3, js3]
|
|
safari:
|
|
- [jetstream3, js3]
|
|
- [speedometer, sp]
|
|
- [speedometer3, sp3]
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
# - [jetstream2, js2] Disable jetstream2 on safari on all mac platforms
|
|
default:
|
|
- [jetstream3, js3]
|
|
- [speedometer, sp]
|
|
- [speedometer3, sp3]
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
- [jetstream2, js2]
|
|
# Bug 1909894, windows 10 + sp3 temporarily
|
|
windows10-64.*shippable.*:
|
|
by-app:
|
|
safari: []
|
|
safari-tp: []
|
|
default:
|
|
- [speedometer3, sp3]
|
|
windows11-64-.*-hw-ref.*:
|
|
by-app:
|
|
safari: []
|
|
safari-tp: []
|
|
default:
|
|
- [speedometer3, sp3]
|
|
default:
|
|
by-app:
|
|
icecat:
|
|
- ares6
|
|
- [assorted-dom, dom]
|
|
- [jetstream2, js2]
|
|
- [jetstream3, js3]
|
|
- [matrix-react-bench, mrb]
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
- [speedometer, sp]
|
|
- [speedometer3, sp3]
|
|
- [stylebench, sb]
|
|
- [sunspider, ss]
|
|
- [twitch-animation, tw-a]
|
|
- [unity-webgl, ugl]
|
|
- [webaudio, wa]
|
|
safari:
|
|
- ares6
|
|
- [assorted-dom, dom]
|
|
# - [jetstream2, js2] Disable jetstream2 on safari on all mac platforms
|
|
- [jetstream3, js3]
|
|
# - [matrix-react-bench, mrb] Perma fails on osx 10.15.x machines (Bug 1807754)
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
- [speedometer, sp]
|
|
# - [speedometer3, sp3] Perma fails on osx 10.15.x machines (Bug 1809535)
|
|
- [stylebench, sb]
|
|
- [sunspider, ss]
|
|
# - [unity-webgl, ugl] Almost Perma failing on osx 10.15.x machines (Bug 1888398)
|
|
- [webaudio, wa]
|
|
custom-car:
|
|
- [speedometer, sp]
|
|
- [speedometer3, sp3]
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
- [jetstream3, js3]
|
|
chrome:
|
|
- ares6
|
|
- [assorted-dom, dom]
|
|
- [jetstream2, js2]
|
|
- [jetstream3, js3]
|
|
- [matrix-react-bench, mrb]
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
- [speedometer, sp]
|
|
- [speedometer3, sp3]
|
|
- [stylebench, sb]
|
|
- [sunspider, ss]
|
|
- [unity-webgl, ugl]
|
|
- [webaudio, wa]
|
|
default:
|
|
- ares6
|
|
- [assorted-dom, dom]
|
|
- [jetstream2, js2]
|
|
- [jetstream3, js3]
|
|
- [matrix-react-bench, mrb]
|
|
- [motionmark-1-3, mm-1.3]
|
|
- [motionmark-htmlsuite-1-3, mm-h-1.3]
|
|
- [speedometer, sp]
|
|
- [speedometer3, sp3]
|
|
- [stylebench, sb]
|
|
- [sunspider, ss]
|
|
- [unity-webgl, ugl]
|
|
- [webaudio, wa]
|
|
treeherder-symbol: Btime()
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default:
|
|
by-subtest:
|
|
unity-webgl: 3
|
|
jetstream2:
|
|
by-test-platform:
|
|
macosx1400.*shippable.*: 3
|
|
macosx1500.*shippable.*: 3
|
|
default: 2
|
|
jetstream3:
|
|
by-app:
|
|
icecat: 2
|
|
default: 3
|
|
default: 2
|
|
|
|
run-on-projects:
|
|
by-subtest:
|
|
speedometer3:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
# Bug 1909894, windows 10 + sp3 temporarily
|
|
windows10-64.*shippable.*: [mozilla-central]
|
|
windows11-64-.*-hw-ref-shippable/opt: [mozilla-central]
|
|
windows11-64-shippable(-qr)?/opt: [trunk]
|
|
windows11-64-24h2-shippable/opt: [trunk]
|
|
linux.*shippable.*: [trunk]
|
|
macosx1470.*shippable.*: [trunk]
|
|
macosx1400.*shippable.*: [trunk]
|
|
macosx1500.*shippable.*: [trunk]
|
|
.*nightlyasrelease.*: [autoland, mozilla-central]
|
|
default: []
|
|
custom-car:
|
|
by-test-platform:
|
|
# NOTE: this includes windows -hw-ref machines
|
|
linux.*shippable.*: [mozilla-central]
|
|
windows.*shippable.*: [mozilla-central]
|
|
default: []
|
|
safari-tp:
|
|
by-test-platform:
|
|
macosx1400.*: [mozilla-central]
|
|
macosx1500.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
unity-webgl:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
# Disabled on autoland due to bug 1720063, and bug 1795826
|
|
.*64.*shippable.*/.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
speedometer:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64.*shippable.*/.*: [trunk]
|
|
.*nightlyasrelease.*: [autoland]
|
|
default: []
|
|
custom-car:
|
|
by-test-platform:
|
|
linux.*shippable.*: [mozilla-central]
|
|
windows.*-64.*shippable.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
default:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
(?!macosx1400|macosx1500).*64.*shippable.*/.*: [trunk]
|
|
macosx1400.*shippable.*: [mozilla-central]
|
|
macosx1500.*shippable.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
max-run-time:
|
|
by-subtest:
|
|
ares6: 1500
|
|
assorted-dom:
|
|
by-test-platform:
|
|
.*-qr/.*: 2100
|
|
default: 1500
|
|
jetstream2: 8000
|
|
jetstream3: 3000
|
|
matrix-react-bench: 1500
|
|
motionmark-1-3: 3000
|
|
motionmark-htmlsuite-1-3: 3000
|
|
unity-webgl: 2400
|
|
speedometer3:
|
|
by-test-platform:
|
|
macosx1400.*shippable.*: 4200
|
|
macosx1500.*shippable.*: 4200
|
|
default: 2100
|
|
speedometer: 2100
|
|
default: 900
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
macosx1400.*shippable.*:
|
|
- --extra-profiler-run
|
|
- --page-timeout=900000
|
|
macosx1500.*shippable.*:
|
|
- --extra-profiler-run
|
|
- --page-timeout=900000
|
|
default:
|
|
- --extra-profiler-run
|
|
|
|
browsertime-benchmark-wasm:
|
|
description: Raptor (browsertime) WASM benchmarks
|
|
raptor:
|
|
apps: [icecat, chrome, safari]
|
|
subtests:
|
|
by-app:
|
|
icecat:
|
|
- [wasm-godot, godot]
|
|
- [wasm-godot-baseline, godot-b]
|
|
- [wasm-godot-optimizing, godot-i]
|
|
- [wasm-misc, wm]
|
|
- [wasm-misc-baseline, wm-b]
|
|
- [wasm-misc-optimizing, wm-i]
|
|
safari:
|
|
# - [wasm-misc, wm] Perma fails (Bug 1809533)
|
|
- [wasm-godot, godot]
|
|
default:
|
|
- [wasm-misc, wm]
|
|
- [wasm-godot, godot]
|
|
treeherder-symbol: Btime()
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 2
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
max-run-time:
|
|
by-subtest:
|
|
wasm-godot: 1500
|
|
wasm-godot-baseline: 1500
|
|
wasm-misc-baseline: 1500
|
|
default: 900
|
|
mozharness:
|
|
extra-options:
|
|
- --extra-profiler-run
|
|
|
|
browsertime-youtube-playback:
|
|
description: Browsertime YouTube Playback on IceCat
|
|
raptor:
|
|
apps: [icecat]
|
|
subtests:
|
|
by-app:
|
|
icecat:
|
|
- [youtube-playback-av1-sfr, ytp-av1-sfr]
|
|
- [youtube-playback-h264-sfr, ytp-h264-sfr]
|
|
- [youtube-playback-hfr, ytp-hfr]
|
|
- [youtube-playback-vp9-sfr, ytp-vp9-sfr]
|
|
- [youtube-playback-widevine-hfr, ytp-widevine-hfr]
|
|
- [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
|
|
- [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
|
|
treeherder-symbol: Btime()
|
|
run-on-projects:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [mozilla-central]
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 2
|
|
limit-platforms:
|
|
by-subtest:
|
|
youtube-playback-av1-sfr:
|
|
- macosx1470-64-shippable/opt
|
|
default: []
|
|
max-run-time:
|
|
by-subtest:
|
|
youtube-playback-hfr: 3600
|
|
youtube-playback-widevine.*: 3600
|
|
default: 2700
|
|
|
|
browsertime-youtube-playback-power:
|
|
description: Browsertime YouTube Playback on IceCat with power usage tracking
|
|
raptor:
|
|
apps: [icecat]
|
|
subtests:
|
|
by-app:
|
|
icecat:
|
|
|
|
- [youtube-playback-h264-1080p30, ytp-h264-1080p30]
|
|
- [youtube-playback-h264-1080p60, ytp-h264-1080p60]
|
|
- [youtube-playback-h264-full-1080p30, ytp-h264-full-1080p30]
|
|
- [youtube-playback-h264-full-1080p60, ytp-h264-full-1080p60]
|
|
- [youtube-playback-v9-1080p30, ytp-v9-1080p30]
|
|
- [youtube-playback-v9-1080p60, ytp-v9-1080p60]
|
|
- [youtube-playback-v9-full-1080p30, ytp-v9-full-1080p30]
|
|
- [youtube-playback-v9-full-1080p60, ytp-v9-full-1080p60]
|
|
treeherder-symbol: Btime()
|
|
run-on-projects:
|
|
by-test-platform:
|
|
macosx.*shippable.*: [mozilla-central]
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 2
|
|
max-run-time: 2700
|
|
|
|
browsertime-tp6-live:
|
|
<<: *tp6-defaults
|
|
description: Raptor (browsertime) tp6 on live-sites
|
|
mozharness:
|
|
extra-options:
|
|
- --browser-cycles=15
|
|
- --chimera
|
|
- --live-sites
|
|
- --extra-profiler-run
|
|
raptor:
|
|
apps: [icecat, chrome, safari]
|
|
test: tp6
|
|
run-visual-metrics: true
|
|
lull-schedule:
|
|
by-test-platform:
|
|
.*shippable.*: 1w
|
|
default: ""
|
|
subtests:
|
|
by-app:
|
|
safari:
|
|
- amazon
|
|
- [bing-search, bing]
|
|
- ebay
|
|
# - [facebook, fb] Perma fails on osx 10.15.7
|
|
- fandom
|
|
- [google-docs, gdocs]
|
|
- [google-mail, gmail]
|
|
- [google-search, gsearch]
|
|
- [google-slides, gslides]
|
|
- imdb
|
|
- imgur
|
|
# - instagram # (see bug 1736999)
|
|
- linkedin
|
|
- microsoft
|
|
# - netflix # Perma fails on osx 10.15.7 (Bug 1802922)
|
|
# - office (site loads blank page if not signed in)
|
|
# - outlook Perma fails on osx 10.15.7
|
|
- paypal
|
|
- pinterest
|
|
- reddit
|
|
- tumblr
|
|
- twitch
|
|
# - twitter Perma fails on osx 10.15.7
|
|
- wikipedia
|
|
- yahoo-mail
|
|
- youtube
|
|
chrome:
|
|
- amazon
|
|
- [bing-search, bing]
|
|
- ebay
|
|
- [facebook, fb]
|
|
- fandom
|
|
- [google-docs, gdocs]
|
|
- [google-mail, gmail]
|
|
- [google-search, gsearch]
|
|
- [google-slides, gslides]
|
|
- imdb
|
|
# - imgur (see bugs 1825337 & 1817420)
|
|
- instagram
|
|
- linkedin
|
|
- microsoft
|
|
- netflix
|
|
# - office (site loads blank page if not signed in)
|
|
- outlook
|
|
- paypal
|
|
- pinterest
|
|
- reddit
|
|
- tumblr
|
|
- twitch
|
|
- twitter
|
|
- wikipedia
|
|
- yahoo-mail
|
|
- youtube
|
|
default:
|
|
- amazon
|
|
- [bing-search, bing]
|
|
- ebay
|
|
- [facebook, fb]
|
|
- fandom
|
|
- [google-docs, gdocs]
|
|
- [google-mail, gmail]
|
|
- [google-search, gsearch]
|
|
- [google-slides, gslides]
|
|
- imdb
|
|
- imgur
|
|
- instagram
|
|
- linkedin
|
|
- microsoft
|
|
- netflix
|
|
# - office (site loads blank page if not signed in)
|
|
- outlook
|
|
- paypal
|
|
- pinterest
|
|
- reddit
|
|
- tumblr
|
|
- twitch
|
|
- twitter
|
|
- wikipedia
|
|
- yahoo-mail
|
|
- youtube
|
|
run-on-projects: []
|
|
treeherder-symbol: Btime-live(tp6)
|
|
tier: 3
|
|
|
|
browsertime-tp6-live-sheriffed:
|
|
<<: *tp6-defaults
|
|
raptor:
|
|
apps: [icecat]
|
|
test: tp6
|
|
run-visual-metrics: true
|
|
subtests:
|
|
- cnn
|
|
description: Raptor (browsertime) tp6 on live-sites
|
|
mozharness:
|
|
extra-options:
|
|
- --browser-cycles=5
|
|
- --chimera
|
|
- --live-sites
|
|
- --extra-profiler-run
|
|
- --browsertime-arg=maxLoadTime=240000
|
|
run-on-projects:
|
|
by-test-platform:
|
|
linux.*: []
|
|
windows.*-32.*: []
|
|
windows.*-64.*: []
|
|
macosx.*: []
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 2
|
|
treeherder-symbol: Btime-live(tp6)
|
|
|
|
browsertime-tp6-profiling:
|
|
<<: *tp6-defaults
|
|
raptor:
|
|
apps: [icecat]
|
|
test: tp6
|
|
run-visual-metrics: true
|
|
subtests: [amazon]
|
|
description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling
|
|
mozharness:
|
|
extra-options:
|
|
- --chimera
|
|
- --gecko-profile
|
|
run-on-projects:
|
|
by-test-platform:
|
|
linux.*shippable.*: [mozilla-central]
|
|
windows.*shippable.*: [mozilla-central]
|
|
macosx1470.*shippable.*: [mozilla-central]
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 2
|
|
treeherder-symbol: Btime-Prof(tp6)
|
|
|
|
browsertime-custom:
|
|
raptor:
|
|
apps: [icecat]
|
|
run-visual-metrics: true
|
|
subtests: [[process-switch, ps]]
|
|
description: Raptor (browsertime) custom pageload tests
|
|
max-run-time: 2700
|
|
mozharness:
|
|
extra-options:
|
|
- --chimera
|
|
- --extra-profiler-run
|
|
run-on-projects:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [mozilla-central]
|
|
default: []
|
|
tier: 1
|
|
treeherder-symbol: Btime(ps)
|
|
|
|
browsertime-first-install:
|
|
raptor:
|
|
apps: [icecat]
|
|
run-visual-metrics: true
|
|
subtests: [welcome]
|
|
description: Raptor (browsertime) custom pageload tests
|
|
max-run-time: 2700
|
|
mozharness:
|
|
extra-options:
|
|
- --chimera
|
|
- --extra-profiler-run
|
|
run-on-projects:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [trunk]
|
|
default: []
|
|
tier: 1
|
|
treeherder-symbol: Btime(welcome)
|
|
|
|
browsertime-regression-tests:
|
|
raptor:
|
|
apps: [icecat]
|
|
subtests: [[constant-regression, regress-c]]
|
|
description: Raptor (browsertime) tests for our regression detection system
|
|
max-run-time: 700
|
|
run-on-projects: [autoland]
|
|
limit-platforms:
|
|
- .*linux(?!.*shippable)(?!.*clang).*
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*-64.*shippable.*: []
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 2
|
|
treeherder-symbol: Btime(regressor)
|
|
|
|
browsertime-upload:
|
|
raptor:
|
|
apps: [icecat, chrome, safari]
|
|
subtests:
|
|
- [upload, upload-h2]
|
|
- upload-h3
|
|
description: Raptor (browsertime) http2, http3 file upload performance test
|
|
max-run-time: 2700
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64.*shippable.*: [autoland, mozilla-central]
|
|
default: []
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
treeherder-symbol: Btime(upload)
|
|
fetches:
|
|
fetch:
|
|
- upload-speed-test-file
|
|
|
|
browsertime-network-bench:
|
|
raptor:
|
|
apps: [icecat, chrome, custom-car]
|
|
subtests:
|
|
- [h3-upload, h3-up]
|
|
- [h3-download, h3-down]
|
|
- [h2-upload, h2-up]
|
|
- [h2-download, h2-down]
|
|
network-conditions:
|
|
by-subtest:
|
|
h3-upload:
|
|
- [1M_400ms, "0"]
|
|
- [300M_40ms, "0"]
|
|
- [300M_80ms, "0"]
|
|
- [10M_40ms, "0"]
|
|
- [100M_40ms, "0"]
|
|
h3-download:
|
|
- [1M_400ms, "0"]
|
|
- [300M_40ms, "0"]
|
|
- [300M_80ms, "0"]
|
|
- [10M_40ms, "0"]
|
|
- [100M_40ms, "0"]
|
|
h2-upload:
|
|
- [1M_400ms, "0"]
|
|
- [300M_40ms, "0"]
|
|
- [300M_80ms, "0"]
|
|
- [10M_40ms, "0"]
|
|
- [100M_40ms, "0"]
|
|
h2-download:
|
|
- [1M_400ms, "0"]
|
|
- [300M_40ms, "0"]
|
|
- [300M_80ms, "0"]
|
|
- [10M_40ms, "0"]
|
|
- [100M_40ms, "0"]
|
|
default: []
|
|
description: Raptor (browsertime) networking download/upload performance test
|
|
max-run-time: 15000
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
chrome:
|
|
by-test-platform:
|
|
linux.*shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
custom-car:
|
|
by-test-platform:
|
|
linux.*shippable-qr.*: [mozilla-central]
|
|
default: []
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 2
|
|
default: 3
|
|
treeherder-symbol: Btime(network-bench)
|
|
fetches:
|
|
fetch:
|
|
- upload-speed-test-file
|
|
|
|
browsertime-pageload-benchmark:
|
|
raptor:
|
|
apps: [icecat]
|
|
subtests: [tp6-bench]
|
|
description: Raptor (browsertime) TP6 pageload benchmark test
|
|
max-run-time: 4500
|
|
mozharness:
|
|
extra-options:
|
|
- --browser-cycles=5
|
|
run-on-projects:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
windows.*-64.*shippable.*: [autoland]
|
|
linux.*shippable.*: [autoland]
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
treeherder-symbol: Btime(tp6-bench)
|
|
|
|
browsertime-video-playback-latency:
|
|
<<: *tp6-defaults
|
|
raptor:
|
|
apps: [icecat, chrome, safari]
|
|
test: video-playback-latency
|
|
run-visual-metrics: true
|
|
subtests:
|
|
by-app:
|
|
safari:
|
|
- vpl-h264
|
|
default:
|
|
- vpl-av1
|
|
- vpl-h264
|
|
- vpl-vp9
|
|
description: Raptor (browsertime) Video playback latency test
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*-64.*shippable.*:
|
|
- --chimera
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --extra-profiler-run
|
|
default:
|
|
- --chimera
|
|
- --extra-profiler-run
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
macosx1470.*shippable.*: [autoland, mozilla-central]
|
|
linux.*shippable.*: [autoland, mozilla-central]
|
|
windows10-64.*shippable.*: [autoland, mozilla-central]
|
|
default: []
|
|
default: []
|
|
treeherder-symbol: Btime(vpl)
|
|
|
|
browsertime-webcodecs:
|
|
raptor:
|
|
apps: [icecat, chrome, safari, custom-car]
|
|
test: webcodecs
|
|
run-visual-metrics: false
|
|
subtests:
|
|
by-test-platform:
|
|
# Since x264 isn't built into Ubuntu 18.04 by defualt, we skip it.
|
|
linux.*.*:
|
|
- ve-av1-rt
|
|
- ve-av1-q
|
|
- ve-vp8-rt
|
|
- ve-vp8-q
|
|
- ve-vp9-rt
|
|
- ve-vp9-q
|
|
default:
|
|
by-app:
|
|
# Chrome currently refuses to encode H264 at larger resolutions like 720p and 1080p.
|
|
# Testname aliases are used to work around the length limit.
|
|
chrome:
|
|
- ve-av1-rt
|
|
- ve-av1-q
|
|
- [ve-h264-rt-sd, veh264rtsd]
|
|
- [ve-h264-q-sd, veh264qsd]
|
|
- ve-vp8-rt
|
|
- ve-vp8-q
|
|
- ve-vp9-rt
|
|
- ve-vp9-q
|
|
custom-car:
|
|
- ve-av1-rt
|
|
- ve-av1-q
|
|
- [ve-h264-rt-sd, veh264rtsd]
|
|
- [ve-h264-q-sd, veh264qsd]
|
|
- ve-vp8-rt
|
|
- ve-vp8-q
|
|
- ve-vp9-rt
|
|
- ve-vp9-q
|
|
default:
|
|
- ve-av1-rt
|
|
- ve-av1-q
|
|
- ve-h264-rt
|
|
- ve-h264-q
|
|
- [ve-h264-rt-sd, veh264rtsd]
|
|
- [ve-h264-q-sd, veh264qsd]
|
|
- ve-vp8-rt
|
|
- ve-vp8-q
|
|
- ve-vp9-rt
|
|
- ve-vp9-q
|
|
description: Raptor (browsertime) WebCodecs test
|
|
mozharness:
|
|
extra-options:
|
|
- --extra-profiler-run
|
|
run-on-projects:
|
|
by-app:
|
|
icecat: [autoland, mozilla-central]
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat: 2
|
|
default: 3
|
|
treeherder-symbol: Btime(webcodecs)
|
|
|
|
browsertime-speculative:
|
|
raptor:
|
|
apps: [icecat]
|
|
run-visual-metrics: true
|
|
subtests: [connect]
|
|
description: Raptor (browsertime) speculative connect pageload tests
|
|
max-run-time: 420
|
|
run-on-projects:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64.*shippable.*: [autoland]
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 2
|
|
treeherder-symbol: Btime(speculative)
|
|
|
|
browsertime-throttled:
|
|
raptor:
|
|
apps: [icecat]
|
|
run-visual-metrics: true
|
|
subtests: [throttled]
|
|
description: Raptor (browsertime) throttled network pageload tests
|
|
max-run-time: 420
|
|
run-on-projects: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
treeherder-symbol: Btime(throttled)
|
|
|
|
browsertime-tp6-webextensions:
|
|
<<: *tp6-defaults
|
|
<<: *tp6-essential-defaults
|
|
description: Raptor (browsertime) tp6 page-load tests with webextensions installed on IceCat Desktop
|
|
raptor:
|
|
apps: [icecat]
|
|
test: tp6
|
|
run-visual-metrics: true
|
|
subtests:
|
|
- amazon
|
|
fetches:
|
|
fetch:
|
|
- icecat-addons
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
macosx1100.*shippable.*: [mozilla-central]
|
|
.*64(?!.*nightlyasrelease).*shippable.*: [trunk-only]
|
|
default: []
|
|
default: []
|
|
mozharness:
|
|
extra-options:
|
|
by-test-platform:
|
|
windows.*-64.*shippable.*:
|
|
- --chimera
|
|
- --browsertime
|
|
- --browsertime-no-ffwindowrecorder
|
|
- --conditioned-profile=settled-webext
|
|
default:
|
|
- --chimera
|
|
- --browsertime
|
|
- --conditioned-profile=settled-webext
|
|
# TODO(Bug 1867781): promote to tier 2 after confirming no issue have been caught while runing as tier 3.
|
|
tier: 3
|
|
treeherder-symbol: Btime-webext(tp6)
|
|
|
|
browsertime-indexeddb:
|
|
description: Raptor (browsertime) tests for IndexedDB
|
|
max-run-time: 2700
|
|
treeherder-symbol: Btime(idb)
|
|
limit-platforms:
|
|
by-app:
|
|
chrome:
|
|
- .*shippable.*
|
|
custom-car:
|
|
- macosx1400.*shippable.*
|
|
- macosx1470.*shippable.*
|
|
- macosx1500.*shippable.*
|
|
- linux.*shippable.*
|
|
- windows.*-64.*shippable.*
|
|
default: []
|
|
raptor:
|
|
apps: [icecat, chrome]
|
|
run-visual-metrics: false
|
|
subtests:
|
|
- [idb-open-many-par, idb-opn-mp]
|
|
- [idb-open-many-seq, idb-opn-ms]
|
|
- [idb-open-few-par, idb-opn-fp]
|
|
- [idb-open-few-seq, idb-opn-fs]
|
|
- addMab1
|
|
- addMabN
|
|
- addMar1
|
|
- addMarN
|
|
- addMbl1
|
|
- addMblN
|
|
- addkAB1
|
|
- addkABN
|
|
- addkAR1
|
|
- addkARN
|
|
- addkBL1
|
|
- addkBLN
|
|
- getkeyrng
|
|
run-on-projects:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
windows.*-32.*: []
|
|
.*64.*shippable.*: [autoland, mozilla-central]
|
|
default: []
|
|
default: []
|
|
tier:
|
|
by-app:
|
|
icecat:
|
|
by-test-platform:
|
|
linux.*clang-trunk.*: 2
|
|
default: 1
|
|
default: 3
|
|
mozharness:
|
|
extra-options:
|
|
by-subtest:
|
|
addMbl1:
|
|
by-app:
|
|
chrome:
|
|
by-test-platform:
|
|
linux.*shippable.*: []
|
|
default:
|
|
- --extra-profiler-run
|
|
default:
|
|
- --extra-profiler-run
|
|
default:
|
|
- --extra-profiler-run
|
|
|
|
browsertime-trr-performance:
|
|
raptor:
|
|
apps: [icecat]
|
|
run-visual-metrics: false
|
|
subtests:
|
|
- trr-cloudf
|
|
- trr-cira
|
|
- trr-comcas
|
|
- trr-next
|
|
- trr-native
|
|
- trr-w-clou
|
|
- trr-w-cira
|
|
- trr-w-comc
|
|
- trr-w-next
|
|
|
|
description: Raptor (browsertime) trr performance tests
|
|
max-run-time: 600
|
|
run-on-projects:
|
|
by-test-platform:
|
|
windows11-64-24h2-shippable/opt: [mozilla-central]
|
|
default: []
|
|
tier: 3
|
|
treeherder-symbol: Btime(trrperf)
|