Merge upstream.

This commit is contained in:
Rob Sayre 2011-06-23 09:57:36 -07:00
commit 9da2b9c0ea
4 changed files with 141 additions and 56 deletions

View file

@ -11,6 +11,7 @@ DOC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__),".."))
ANAL = True
DEFAULT_MANIFEST_FILE = 'test_manifest.json'
DEFAULT_BROWSER_MANIFEST_FILE = 'browser_manifest.json'
EQLOG_FILE = 'eq.log'
REFDIR = 'ref'
TMPDIR = 'tmp'
VERBOSE = False
@ -42,7 +43,7 @@ def prompt(question):
MIMEs = {
'.css': 'text/css',
'.html': 'text/html',
'.js': 'application/json',
'.js': 'application/javascript',
'.json': 'application/json',
'.pdf': 'application/pdf',
'.xhtml': 'application/xhtml+xml',
@ -61,6 +62,7 @@ class State:
numEqNoSnapshot = 0
numFBFFailures = 0
numLoadFailures = 0
eqLog = None
class Result:
def __init__(self, snapshot, failure):
@ -191,7 +193,7 @@ def setUp(options):
taskResults.append([ ])
State.taskResults[b.name][id] = taskResults
State.remaining = len(manifestList)
State.remaining = len(testBrowsers) * len(manifestList)
for b in testBrowsers:
print 'Launching', b.name
@ -231,6 +233,7 @@ def check(task, results, browser):
def checkEq(task, results, browser):
pfx = os.path.join(REFDIR, sys.platform, browser, task['id'])
results = results[0]
taskId = task['id']
passed = True
for page in xrange(len(results)):
@ -249,7 +252,21 @@ def checkEq(task, results, browser):
eq = (ref == snapshot)
if not eq:
print 'TEST-UNEXPECTED-FAIL | eq', task['id'], '| in', browser, '| rendering of page', page + 1, '!= reference rendering'
print 'TEST-UNEXPECTED-FAIL | eq', taskId, '| in', browser, '| rendering of page', page + 1, '!= reference rendering'
# XXX need to dump this always, somehow, when we have
# the reference repository
if State.masterMode:
if not State.eqLog:
State.eqLog = open(EQLOG_FILE, 'w')
eqLog = State.eqLog
# NB: this follows the format of Mozilla reftest
# output so that we can reuse its reftest-analyzer
# script
print >>eqLog, 'REFTEST TEST-UNEXPECTED-FAIL |', browser +'-'+ taskId +'-page'+ str(page + 1), '| image comparison (==)'
print >>eqLog, 'REFTEST IMAGE 1 (TEST):', snapshot
print >>eqLog, 'REFTEST IMAGE 2 (REFERENCE):', ref
passed = False
State.numEqFailures += 1