# HG changeset patch # User Philipp von Weitershausen # Date 1276106233 25200 # Node ID ebd5ae433cc9f018895548ff7065659eeda76682 # Parent f5a64d0ecc66f4699c608fc5dc9947e2ecdbed6d Bug 569228 - Record test coverage diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -44,6 +44,8 @@ export stage_dir=$(objdir)/stage xpi_dir=$(objdir)/xpi +coveragefilter=resource://weave/* + weave_version := 1.4a1pre storage_version := 2 weave_id := {340c2bbc-ce74-4362-90b5-7c26312808ef} @@ -116,6 +118,12 @@ test: build $(MAKE) -k -C services/sync/tests/unit +testcoverage: build + mkdir -p unittest/coverage + rm -f unittest/coverage/* + COVERAGE_OUTPUT=$(TOPSRCDIR)/unittest/coverage/coverage.json COVERAGE_FILTER=$(coveragefilter) $(MAKE) -k -C services/sync/tests/unit + cd unittest/coverage && python $(TOPSRCDIR)/tests/coverage/report.py coverage.json + setup: $(MKDIR) $(objdir) $(MKDIR) $(stage_dir) diff --git a/services/sync/tests/harness/app/chrome/content/main.js b/services/sync/tests/harness/app/chrome/content/main.js --- a/services/sync/tests/harness/app/chrome/content/main.js +++ b/services/sync/tests/harness/app/chrome/content/main.js @@ -34,6 +34,19 @@ * * ***** END LICENSE BLOCK ***** */ +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; + +// Constants for file I/O +const MODE_RDONLY = 0x01; +const MODE_WRONLY = 0x02; +const MODE_CREATE = 0x08; +const MODE_APPEND = 0x10; +const MODE_TRUNCATE = 0x20; + + let cmdLine = window.arguments[0]. QueryInterface(Components.interfaces.nsICommandLine); @@ -43,6 +56,10 @@ let ioService = Components.classes["@mozilla.org/network/io-service;1"]. getService(Components.interfaces.nsIIOService); +let jsd = Components.classes["@mozilla.org/js/jsd/debugger-service;1"] + .getService(Components.interfaces.jsdIDebuggerService); + + // xpcshell provides this method, which some tests use, so we emulate it. // We also use it ourselves to load the files specified on our command line. function load(path) { @@ -64,6 +81,103 @@ let environment = Components.classes["@mozilla.org/process/environment;1"]. getService(Components.interfaces.nsIEnvironment); + +// Code coverage analysis. We use the debugger and trigger on every +// statement. Statistics are collected in this object which is then +// written to disk. +var coverageStats = {}; + +function enableCoverage(resourcePath) { + let RETURN_CONTINUE = Ci.jsdIExecutionHook.RETURN_CONTINUE; + + jsd.interruptHook = { + onExecute: function (frame, type, rv) { + let filename = frame.script.fileName; + let filestats = coverageStats[filename]; + if (!filestats) { + filestats = coverageStats[filename] = {}; + } + if (filestats[frame.line] == undefined) { + filestats[frame.line] = 0; + } + filestats[frame.line]++; + return RETURN_CONTINUE; + } + }; + + function createFilter(pattern, pass) { + return { + globalObject: null, + flags: pass ? (Ci.jsdIFilter.FLAG_ENABLED | Ci.jsdIFilter.FLAG_PASS) : Ci.jsdIFilter.FLAG_ENABLED, + urlPattern: pattern, + startLine: 0, + endLine: 0 + }; + }; + + if (resourcePath) { + // Resolve resource:// path to filesystem path + let uri = ioService.newURI(resourcePath, null, null); + let path = uri.QueryInterface(Ci.nsIFileURL).file.path; + + // Ignore all files except files corresponding to 'resourcePath' + jsd.appendFilter(createFilter('file:' + path, true)); + jsd.appendFilter(createFilter('*')); + } + + jsd.on(); +} + +function absolutePath(filename) { + let uri = ioService.newURI(filename, null, + ioService.newFileURI(cmdLine.workingDirectory)); + return uri.QueryInterface(Ci.nsIFileURL).file.path; +} + +function openFile(filename) { + let file = Cc["@mozilla.org/file/local;1"] + .createInstance(Ci.nsILocalFile); + file.initWithPath(absolutePath(filename)); + return file; +} + +function writeStatsToFile(filename) { + let file = openFile(filename); + let stream = Cc["@mozilla.org/network/file-output-stream;1"] + .createInstance(Ci.nsIFileOutputStream); + stream.init(file, MODE_WRONLY | MODE_CREATE | MODE_TRUNCATE, 0644, 0); + + let data = JSON.stringify(coverageStats); + stream.write(data, data.length); +} + +function loadStatsFromFile(filename) { + let file = openFile(filename); + if (!file.exists()) { + return; + } + + let stream = Cc["@mozilla.org/network/file-input-stream;1"] + .createInstance(Ci.nsIFileInputStream); + stream.init(file, MODE_RDONLY, 0644, 0); + + let BinaryInputStream = Components.Constructor( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream"); + let data = new BinaryInputStream(stream).readBytes(stream.available()); + coverageStats = JSON.parse(data); +} + +let coverage_output = environment.get("COVERAGE_OUTPUT"); +let coverage_filter = environment.get("COVERAGE_FILTER"); + +if (coverage_output) { + loadStatsFromFile(coverage_output); + enableCoverage(coverage_filter); +} + + while (cmdLine.findFlag("f", false) != -1) { try { // This throws if the param is missing. @@ -81,4 +195,7 @@ } } +if (coverage_output) { + writeStatsToFile(coverage_output); +} goQuitApplication(); diff --git a/services/sync/tests/unit/head.js b/services/sync/tests/unit/head.js --- a/services/sync/tests/unit/head.js +++ b/services/sync/tests/unit/head.js @@ -1,8 +1,3 @@ -const Cc = Components.classes; -const Ci = Components.interfaces; -const Cr = Components.results; -const Cu = Components.utils; - let versSvc = Cc["@mozilla.org/xpcom/version-comparator;1"]. getService(Ci.nsIVersionComparator); let appinfo = Cc["@mozilla.org/xre/app-info;1"]. diff --git a/tests/coverage/coverage.css b/tests/coverage/coverage.css new file mode 100644 --- /dev/null +++ b/tests/coverage/coverage.css @@ -0,0 +1,76 @@ +body { +} + +table { + border-collapse: collapse; +} + +.filelink { + font-family: monospace; + color: black; + text-decoration: none; +} +.filelink:hover { + text-decoration: underline; +} + +.source-line { + font-size: 12px; +} + +.source-line pre { + font-family: monospace; + margin: 0; +} + +.coverage-count { + font-family: Helvetica,sans-serif; + padding-right: 5px; +} + +.line-number { + border-left: 1px solid #aaa; + padding: 0 3px 0 5px; +} + +.line-number a { + color: #aaa; + font-family: monospace; + text-decoration: none; +} +.line-number a:hover { + color: black; +} + +.line-content { + border-left: 1px solid #aaa; + padding-left: 5px; +} + +.uncovered { + background-color: #fdd; +} + +.covered { + background-color: #dfd; +} + +.number-cell { + text-align: right; +} + +.summary-header { + text-align: left; +} + +.summary-header th { + border-bottom: 1px solid black; +} + +.summary-total { + font-weight: bold; +} + +.summary-total td { + border-top: 1px solid black; +} diff --git a/tests/coverage/file.html.in b/tests/coverage/file.html.in new file mode 100644 --- /dev/null +++ b/tests/coverage/file.html.in @@ -0,0 +1,23 @@ + + + + + + +

{{stats.filename}}

+ +

{{stats.lines}} lines, {{stats.loc}} LOC, {{stats.covered}} covered ({{stats.percentage}}%)

+ + + {% for line in data %} + + + + + + {% endfor %} + +
{{line.count}}{{line.number}}
{{line.line}}
+ + + diff --git a/tests/coverage/index.html.in b/tests/coverage/index.html.in new file mode 100644 --- /dev/null +++ b/tests/coverage/index.html.in @@ -0,0 +1,36 @@ + + + + + + +

Coverage report

+ + + + + + + + + + {% for stats in filestats %} + + + + + + + {% endfor %} + + + + + + + + +
FilenameLOCcoveredratio
{{stats.filename}}{{stats.loc}}{{stats.covered}}{{stats.percentage}}%
Total{{totalstats.loc}}{{totalstats.covered}}{{totalstats.percentage}}%
+ + + diff --git a/tests/coverage/report.py b/tests/coverage/report.py new file mode 100644 --- /dev/null +++ b/tests/coverage/report.py @@ -0,0 +1,144 @@ +import os +import re +import sys +import cgi +try: + import json +except ImportError: + import simplejson as json +import shutil +from StringIO import StringIO +from templite import Templite + + +def multilineCommentMatcher(code): + """Return a function that finds out whether a given portion of a + piece of code lies within a multiline comment. + """ + match_comment = re.compile(r'\s*/\*(.*?)\*/\s*', re.DOTALL).finditer + def findMatches(begin, end): + for match in match_comment(code): + if (begin >= match.start()) and (end <= match.end()): + return True + return False + return findMatches + +def isIgnoredLine(line): + """Figure out whether 'line' can be ignored in coverage statistics. + + These include empty lines and lines containing nothing but closing braces. + """ + try: + pos = line.index('//') + line = line[:pos] + except ValueError: + pass + # This is a bit pathetic but effective + return line.strip() in ['', '}', '},', '};', '});'] + +def readfile(name): + """Read a file relative to the current working directory.""" + filename = os.path.join(os.path.dirname(__file__), name) + return open(filename).read() + +def generateReport(datafile): + data = open(datafile).read() + coverage = json.loads(data) + + totalstats = {"loc": 0, "covered": 0, "ratio": 0, "percentage": ""} + stats = {} + + # Find common base directory + chunks = [] + for filename in coverage: + if filename.startswith('file:'): + filename = filename[5:] + if not chunks: + chunks = filename.split(os.path.sep) + continue + for i, part in enumerate(filename.split(os.path.sep)): + if i < len(chunks) and chunks[i] != part: + chunks = chunks[:i] + break + basedir = os.path.sep.join(chunks) + + for filename, linecounts in coverage.iteritems(): + if filename.startswith('file:'): + filename = filename[5:] + + displayname = filename[len(basedir)+1:] + htmlname = displayname.replace('/', '_') + '.html' + + filestats = stats[filename] = {"filename": displayname, + "htmlname": htmlname, + "lines": 0, + "loc": 0, + "covered": 0, + "ratio": 0, + "percentage": ""} + + filedata = [] + code = open(filename).read() + matchMultilineComment = multilineCommentMatcher(code) + + pos = 0 + for i, line in enumerate(StringIO(code)): + covered = str(i+1) in linecounts + ignored = (isIgnoredLine(line) or + matchMultilineComment(pos, pos+len(line))) + pos += len(line) + + cssClass = 'uncovered' + if ignored: + cssClass = 'ignored' + + # Ignored lines don't count -- unless they have coverage + if not ignored or (ignored and covered): + filestats["loc"] += 1 + if covered: + filestats["covered"] += 1 + cssClass = 'covered' + + filedata.append({'number': i+1, + 'count': linecounts.get(str(i+1), ""), + 'line': cgi.escape(line[:-1]), + 'cssClass': cssClass}) + + # Compute coverage for this file and update total + filestats['lines'] = i + 1 + filestats['ratio'] = float(filestats['covered'])/filestats['loc'] + filestats['percentage'] = "%d" % (filestats['ratio'] * 100) + + totalstats['loc'] += filestats['loc'] + totalstats['covered'] += filestats['covered'] + + # Render HTML view for this file + file_templ = Templite(readfile("file.html.in"), globals()) + file_html = file_templ.render({'stats': filestats, + 'data': filedata}) + htmlfile = open(htmlname, 'w') + htmlfile.write(file_html) + htmlfile.close() + + + # Compute total coverage + totalstats['ratio'] = float(totalstats['covered'])/totalstats['loc'] + totalstats['percentage'] = "%d" % (totalstats['ratio'] * 100) + + # Write HTML overview file + index_templ = Templite(readfile("index.html.in"), globals()) + filestats = [value for (key, value) in sorted(stats.iteritems())] + index_html = index_templ.render({'filestats': filestats, + 'totalstats': totalstats}) + indexfile = open('index.html', 'w') + indexfile.write(index_html) + indexfile.close() + + shutil.copy(os.path.join(os.path.dirname(__file__), 'coverage.css'), + os.getcwd()) + + print "Generated HTML report: %(loc)s LOC, %(covered)s covered (%(percentage)s%%)" % totalstats + + +if __name__ == "__main__": + generateReport(sys.argv[1]) diff --git a/tests/coverage/templite.py b/tests/coverage/templite.py new file mode 100644 --- /dev/null +++ b/tests/coverage/templite.py @@ -0,0 +1,169 @@ +"""A simple Python template renderer, for a nano-subset of Django syntax.""" + +# Written by Ned Batchelder as part of coverage.py, released under the +# BSD license. See http://nedbatchelder.com/code/coverage/ + +# Coincidentally named the same as http://code.activestate.com/recipes/496702/ + +import re, sys + +class Templite(object): + """A simple template renderer, for a nano-subset of Django syntax. + + Supported constructs are extended variable access:: + + {{var.modifer.modifier|filter|filter}} + + loops:: + + {% for var in list %}...{% endfor %} + + and ifs:: + + {% if var %}...{% endif %} + + Comments are within curly-hash markers:: + + {# This will be ignored #} + + Construct a Templite with the template text, then use `render` against a + dictionary context to create a finished string. + + """ + def __init__(self, text, *contexts): + """Construct a Templite with the given `text`. + + `contexts` are dictionaries of values to use for future renderings. + These are good for filters and global values. + + """ + self.text = text + self.context = {} + for context in contexts: + self.context.update(context) + + # Split the text to form a list of tokens. + toks = re.split(r"(?s)({{.*?}}|{%.*?%}|{#.*?#})", text) + + # Parse the tokens into a nested list of operations. Each item in the + # list is a tuple with an opcode, and arguments. They'll be + # interpreted by TempliteEngine. + # + # When parsing an action tag with nested content (if, for), the current + # ops list is pushed onto ops_stack, and the parsing continues in a new + # ops list that is part of the arguments to the if or for op. + ops = [] + ops_stack = [] + for tok in toks: + if tok.startswith('{{'): + # Expression: ('exp', expr) + ops.append(('exp', tok[2:-2].strip())) + elif tok.startswith('{#'): + # Comment: ignore it and move on. + continue + elif tok.startswith('{%'): + # Action tag: split into words and parse further. + words = tok[2:-2].strip().split() + if words[0] == 'if': + # If: ('if', (expr, body_ops)) + if_ops = [] + assert len(words) == 2 + ops.append(('if', (words[1], if_ops))) + ops_stack.append(ops) + ops = if_ops + elif words[0] == 'for': + # For: ('for', (varname, listexpr, body_ops)) + assert len(words) == 4 and words[2] == 'in' + for_ops = [] + ops.append(('for', (words[1], words[3], for_ops))) + ops_stack.append(ops) + ops = for_ops + elif words[0].startswith('end'): + # Endsomething. Pop the ops stack + ops = ops_stack.pop() + assert ops[-1][0] == words[0][3:] + else: + raise SyntaxError("Don't understand tag %r" % words) + else: + ops.append(('lit', tok)) + + assert not ops_stack, "Unmatched action tag: %r" % ops_stack[-1][0] + self.ops = ops + + def render(self, context=None): + """Render this template by applying it to `context`. + + `context` is a dictionary of values to use in this rendering. + + """ + # Make the complete context we'll use. + ctx = dict(self.context) + if context: + ctx.update(context) + + # Run it through an engine, and return the result. + engine = _TempliteEngine(ctx) + engine.execute(self.ops) + return "".join(engine.result) + + +class _TempliteEngine(object): + """Executes Templite objects to produce strings.""" + def __init__(self, context): + self.context = context + self.result = [] + + def execute(self, ops): + """Execute `ops` in the engine. + + Called recursively for the bodies of if's and loops. + + """ + for op, args in ops: + if op == 'lit': + self.result.append(args) + elif op == 'exp': + try: + self.result.append(str(self.evaluate(args))) + except: + exc_class, exc, _ = sys.exc_info() + new_exc = exc_class("Couldn't evaluate {{ %s }}: %s" + % (args, exc)) + raise new_exc + elif op == 'if': + expr, body = args + if self.evaluate(expr): + self.execute(body) + elif op == 'for': + var, lis, body = args + vals = self.evaluate(lis) + for val in vals: + self.context[var] = val + self.execute(body) + else: + raise AssertionError("TempliteEngine doesn't grok op %r" % op) + + def evaluate(self, expr): + """Evaluate an expression. + + `expr` can have pipes and dots to indicate data access and filtering. + + """ + if "|" in expr: + pipes = expr.split("|") + value = self.evaluate(pipes[0]) + for func in pipes[1:]: + value = self.evaluate(func)(value) + elif "." in expr: + dots = expr.split('.') + value = self.evaluate(dots[0]) + for dot in dots[1:]: + try: + value = getattr(value, dot) + except AttributeError: + value = value[dot] + if hasattr(value, '__call__'): + value = value() + else: + value = self.context[expr] + return value