""" Custom pytest plugin to count the number of tests

collected and executed over multiple pytest runs

tests_collected is set of nodeids for collected tests

tests_executed is set of nodeids for executed tests

"""

def __init__(self):

self.tests_collected = set()

self.tests_executed = set()

# pytest hook to handle test collection when xdist is used (parallel tests)

# https://github.com/pytest-dev/pytest-xdist/pull/35/commits (No official documentation available)

def pytest_xdist_node_collection_finished(self, node, ids):

self.tests_collected.update(set(ids))

# link to pytest_collection_modifyitems

# https://docs.pytest.org/en/2.9.2/writing_plugins.html#_pytest.hookspec.pytest_collection_modifyitems

def pytest_collection_modifyitems(self, items):

for item in items:

self.tests_collected.add(item.nodeid)

# link to pytest_runtest_logreport

# https://docs.pytest.org/en/2.9.2/_modules/_pytest/hookspec.html#pytest_runtest_logreport

def pytest_runtest_logreport(self, report):

if report.passed:

def __init__(self, exit_on_error=True):

self._exit_on_error = exit_on_error

self.tests_failed = False

self.total_executed = 0

def run_tests(self, args):

testcounterplugin = TestCounterPlugin()

try:

pytest_exit_code = pytest.main(args, plugins=[testcounterplugin])

except:

sys.stderr.write("Unexpected exception with pytest {0}".format(args))

raise

if '--collect-only' in args:

for test in testcounterplugin.tests_collected:

print(test)

self.total_executed += len(testcounterplugin.tests_executed)

if 0 < pytest_exit_code < EXIT_NOTESTSCOLLECTED and self._exit_on_error:

sys.exit(pytest_exit_code)

"""

Prepare the list of arguments that will be passed to pytest.main().

Args:

base_name: the base name for the log file to write

valid_dirs: a white list of sub-directories with desired tests (i.e, those

that will not get flagged with --ignore before py.test is called.)

Return:

a list of command line arguments

For most test stages (e.g., serial, parallel), we augment the given command

line arguments with a list of directories to ignore. However, when running the

metric verification tests at the end of the test run:

- verifiers.test_verify_metrics.TestValidateMetrics.test_metrics_are_zero

- verifiers.test_verify_metrics.TestValidateMetrics.test_num_unused_buffers

then we instead need to filter out args that specifiy other tests (otherwise,

they will be run again), but still retain the basic config args.

"""

# When building the list of command line args, in order to correctly filter

# them as needed (see issue IMPALA-4510) we should account for the fact that

# '--foo bar' and '--foo=bar' might be supplied by the user. We also need to

# be able identify any other arbitrary options. E.g., if the user specified

# the following on the command line:

#

# 'run-tests.py --arg1 value1 --random_opt --arg2=value2'

#

# we want an iterable that, if unpacked as a list, would look like:

#

# [arg1, value1, random_opt, arg2, value2]

#

commandline_args = itertools.chain(*[arg.split('=') for arg in sys.argv[1:]])

ignored_dirs = build_ignore_dir_arg_list(valid_dirs=valid_dirs)

logging_args = []

for arg, log in LOGGING_ARGS.items():

logging_args.extend([arg, os.path.join(RESULT_DIR, log.format(base_name))])

if valid_dirs != ['verifiers']:

# This isn't the metrics verification stage yet, so we don't need to filter.

test_args = ignored_dirs + logging_args + list(commandline_args)

else:

# For metrics verification, we only want to run the verifier tests, so we need

# to filter out any command line args that specify other test modules, classes,

# and functions. The list of these can be found by calling

#

# pytest.config.getoption(FILE_OR_DIR)

#

# For example, with the following command line invocation:

#

# $ ./run-tests.py query_test/test_limit.py::TestLimit::test_limit \

# query_test/test_queries.py::TestHdfsQueries --verbose -n 4 \

# --table_formats=parquet/none --exploration_strategy core

#

# then pytest.config.getoption(FILE_OR_DIR) will return a list of two elements:

#

# ['query_test/test_limit.py::TestLimit::test_limit',

# 'query_test/test_queries.py::TestHdfsQueries']

#

explicit_tests = pytest.config.getoption(FILE_OR_DIR)

config_options = [arg for arg in commandline_args if arg not in explicit_tests]

# We also want to strip out any --shard_tests option and its corresponding value.

while "--shard_tests" in config_options:

i = config_options.index("--shard_tests")

del config_options[i:i+2]

test_args = ignored_dirs + logging_args + config_options

""" Builds a list of directories to ignore

Return:

a list ['--ignore', 'dir1', '--ignore', 'dir2', etc...]

Because we have several non-test directories and files in our tests/ path, pytest

can have auto-discovery problems -- i.e., pytest may try to execute some non-test

code as though it contained tests, resulting in misleading warnings or failures.

(There is a JIRA filed to restructure this: IMPALA-4417.)

"""

subdirs = [subdir for subdir in os.listdir(TEST_DIR)

if os.path.isdir(subdir) and not subdir.startswith(".")]

for subdir in subdirs:

assert subdir in VALID_TEST_DIRS or subdir in TEST_HELPER_DIRS,\

"Unexpected test dir '%s' is not in the list of valid or helper test dirs"\

% subdir

ignored_dir_list = []

for subdir in (set(subdirs) - set(valid_dirs)):

ignored_dir_list += ['--ignore', subdir]

"""Prints metrics with the given substring in the name"""

for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:

print(">" * 80)

port = impalad.get_webserver_port()

cert = impalad._get_webserver_certificate_file()

print("connections metrics for impalad at port {0}:".format(port))

debug_info = json.loads(ImpaladService(impalad.hostname, webserver_port=port,

webserver_certificate_file=cert).read_debug_webpage('metrics?json'))

for metric in debug_info['metric_group']['metrics']:

if substring in metric['name']:

print(json.dumps(metric, indent=1))

"""Find any usage of 'flag' in sys.argv and remove them. Return true if the

flag is found. Return false otherwise."""

flag_exists = False

# Handle multiple occurrences of the same flag

while flag in sys.argv:

flag_exists = True

sys.argv.remove(flag)