#!/usr/bin/env python # # Copyright (C) 2011 Igalia S.L. # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Library General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Library General Public License for more details. # # You should have received a copy of the GNU Library General Public License # along with this library; see the file COPYING.LIB. If not, write to # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, # Boston, MA 02110-1301, USA. import subprocess import os import sys import time import optparse from gi.repository import Gio, GLib TIMEOUT=180 # seconds class SkippedTest: def __init__(self, test, reason, bug=None, test_cases=[]): self.test = test self.reason = reason self.bug = bug self.test_cases = test_cases def __str__(self): skipped_test_str = "%s" % self.test if self.test_cases: skipped_test_str += " [%s]" % ", ".join(self.test_cases) skipped_test_str += ": %s " % self.reason if self.bug is not None: skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug return skipped_test_str class TestRunner: TEST_DIRS = [ "unittests", "WebKit2APITests" ] SKIPPED = [ SkippedTest("unittests/testdownload", "Test fails in GTK Linux 64-bit Release bot", 82329, ["/webkit/download/not-found"]), SkippedTest("unittests/testwebview", "Test times out in GTK Linux 64-bit Release bot", 82328, ["/webkit/webview/icon-uri"]), SkippedTest("unittests/testwebresource", "Test fails in GTK Linux 64-bit Release bot", 82330, ["/webkit/webresource/sub_resource_loading"]), SkippedTest("unittests/testwebinspector", "Test is flaky in GTK Linux 32-bit Release bot", 82869, ["/webkit/webinspector/close-and-inspect"]), SkippedTest("WebKit2APITests/TestWebKitWebView", "Test is flaky in GTK Linux 32-bit Release bot", 82866, ["/webkit2/WebKitWebView/mouse-target"]), SkippedTest("WebKit2APITests/TestResources", "Test is flaky in GTK Linux 32-bit Release bot", 82868, ["/webkit2/WebKitWebView/resources"]), ] def __init__(self, options, tests=[]): # FIXME: webkit-build-directory --configuration always returns # Release because we never call set-webkit-configuration. #build_directory_script = os.path.join(os.path.dirname(__file__), "webkit-build-directory") #build_directory = self._executive.run_command([build_directory_script, "--configuration"]).rstrip() self._options = options self._gtk_tools_directory = os.path.join(self._get_top_level_directory(), "Tools", "gtk") self._programs_path = os.path.join(self._get_build_directory(), "Programs") self._tests = self._get_tests(tests) self._skipped_tests = TestRunner.SKIPPED # These SPI daemons need to be active for the accessibility tests to work. self._spi_registryd = None self._spi_bus_launcher = None # run-gtk-tests may be run during make distcheck, which doesn't include jhbuild. self._jhbuild_path = os.path.join(self._gtk_tools_directory, "run-with-jhbuild") if not os.path.exists(self._jhbuild_path): self._jhbuild_path = None def _get_top_level_directory(self): return os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) def _get_build_directory(self): top_level = self._get_top_level_directory() if self._options.release: return os.path.join(top_level, 'WebKitBuild', 'Release') if self._options.debug: return os.path.join(top_level, 'WebKitBuild', 'Debug') build_directory = os.path.join(top_level, 'WebKitBuild', 'Release') if os.path.exists(os.path.join(build_directory, '.libs')): return build_directory build_directory = os.path.join(top_level, 'WebKitBuild', 'Debug') if os.path.exists(os.path.join(build_directory, '.libs')): return build_directory build_directory = os.path.join(top_level, '_build') if os.path.exists(os.path.join(build_directory, '.libs')): return build_directory return os.path.join(top_level, 'WebKitBuild') def _get_tests(self, tests): if tests: return tests tests = [] for test_dir in self.TEST_DIRS: absolute_test_dir = os.path.join(self._programs_path, test_dir) if not os.path.isdir(absolute_test_dir): continue for test_file in os.listdir(absolute_test_dir): if not test_file.lower().startswith("test"): continue test_path = os.path.join(self._programs_path, test_dir, test_file) if os.path.isfile(test_path) and os.access(test_path, os.X_OK): tests.append(test_path) return tests def _create_process(self, command, stdout=None, stderr=None, env=os.environ): if self._jhbuild_path: command.insert(0, self._jhbuild_path) return subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env) def _lookup_atspi2_binary(self, filename): process = self._create_process(['pkg-config', '--variable=exec_prefix', 'atspi-2'], stdout=subprocess.PIPE) stdout = process.communicate()[0] exec_prefix = stdout.rstrip('\r\n') for path in [ 'libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core' ]: filepath = os.path.join(exec_prefix, path, filename) if os.path.isfile(filepath): return filepath return None def _start_accessibility_daemons(self): if not self._jhbuild_path: return False spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher') spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd') if not spi_bus_launcher_path or not spi_registryd_path: return False try: self._ally_bus_launcher = self._create_process([spi_bus_launcher_path], env=self._test_env) except: sys.stderr.write("Failed to launch the accessibility bus\n") sys.stderr.flush() return False # We need to wait until the SPI bus is launched before trying to start the SPI # registry, so we spin a main loop until the bus name appears on DBus. loop = GLib.MainLoop() Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE, lambda *args: loop.quit(), None) loop.run() try: self._spi_registryd = self._create_process([spi_registryd_path], env=self._test_env) except: sys.stderr.write("Failed to launch the accessibility registry\n") sys.stderr.flush() return False return True def _setup_testing_environment(self): self._test_env = os.environ self._test_env["DISPLAY"] = self._options.display self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector')) self._test_env['GSETTINGS_BACKEND'] = 'memory' try: self._xvfb = self._create_process(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) except Exception as e: sys.stderr.write("Failed to run Xvfb: %s\n", e) sys.stderr.flush() return False # If we cannot start the accessibility daemons, we can just skip the accessibility tests. if not self._start_accessibility_daemons(): print "Could not start accessibility bus, so skipping TestWebKitAccessibility" self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", "Could not start accessibility bus")) return True def _tear_down_testing_environment(self): if self._spi_registryd: self._spi_registryd.terminate() if self._spi_bus_launcher: self._spi_bus_launcher.terminate() self._xvfb.kill(); def _find_skipped_test(self, test): for skipped in self._skipped_tests: if test.endswith(skipped.test): return skipped return None def _test_cases_to_skip(self, test): if self._options.skipped_action != 'skip': return [] skipped = self._find_skipped_test(test) if skipped is not None: return skipped.test_cases return [] def _should_run_test(self, test): # Skipped test are ignored, run all tests. if self._options.skipped_action == 'ignore': return True skipped = self._find_skipped_test(test) # By default skipped test are skipped, run them only when there are specific test cases failing. if self._options.skipped_action == 'skip': return skipped is None or skipped.test_cases # Run only skipped tests. return skipped is not None def _run_test(self, test): tester_command = ['gtester'] if self._options.verbose: tester_command.append('--verbose') for test_case in self._test_cases_to_skip(test): tester_command.extend(['-s', test_case]) tester_command.append(test) return not self._create_process(tester_command, env=self._test_env).wait() def run_tests(self): if not self._tests: sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path)) sys.stderr.flush() return 1 if not self._setup_testing_environment(): return 1 # Remove skipped tests now instead of when we find them, because # some tests might be skipped while setting up the test environment. self._tests = [test for test in self._tests if self._should_run_test(test)] failed_tests = [] try: start_time = time.time() for test in self._tests: if not self._run_test(test): failed_tests.append(test) if time.time() - start_time >= TIMEOUT: sys.stdout.write("Tests timed out after %d seconds\n" % TIMEOUT) sys.stdout.flush() return 1 finally: self._tear_down_testing_environment() if failed_tests: names = [test.replace(self._programs_path, '', 1) for test in failed_tests] sys.stdout.write("Tests failed: %s\n" % ", ".join(names)) sys.stdout.flush() if self._skipped_tests and self._options.skipped_action == 'skip': sys.stdout.write("Tests skipped:\n%s\n" % "\n".join([str(skipped) for skipped in self._skipped_tests])) sys.stdout.flush() return len(failed_tests) if __name__ == "__main__": option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]') option_parser.add_option('-r', '--release', action='store_true', dest='release', help='Run in Release') option_parser.add_option('-d', '--debug', action='store_true', dest='debug', help='Run in Debug') option_parser.add_option('-v', '--verbose', action='store_true', dest='verbose', help='Run gtester in verbose mode') option_parser.add_option('--display', action='store', dest='display', default=':55', help='Display to run Xvfb') option_parser.add_option('--skipped', action='store', dest='skipped_action', choices=['skip', 'ignore', 'only'], default='skip', metavar='skip|ignore|only', help='Specifies how to treat the skipped tests') options, args = option_parser.parse_args() sys.exit(TestRunner(options, args).run_tests())