diff --git a/runtime-light/stdlib/diagnostics/functions-call-stats.h b/runtime-light/stdlib/diagnostics/functions-call-stats.h index c1914f1495..31eb5d6d53 100644 --- a/runtime-light/stdlib/diagnostics/functions-call-stats.h +++ b/runtime-light/stdlib/diagnostics/functions-call-stats.h @@ -6,4 +6,4 @@ #include "runtime-light/stdlib/diagnostics/logs.h" -#define DUMP_BUILTIN_CALL_STATS(builtin_name, builtin_call) (kphp::log::debug("built-in called: " builtin_name), builtin_call) +#define DUMP_BUILTIN_CALL_STATS(builtin_name, builtin_call) (kphp::log::debug("std function called: " builtin_name), builtin_call) diff --git a/tests/aggregate_std_function_invocations.py b/tests/aggregate_std_function_invocations.py new file mode 100644 index 0000000000..87bdf665f1 --- /dev/null +++ b/tests/aggregate_std_function_invocations.py @@ -0,0 +1,63 @@ +import json +import csv +import argparse +import pathlib +import collections + +SCRIPT_DIR = pathlib.Path(__file__).parent +DEFAULT_OUTPUT = SCRIPT_DIR / 'tmp' / 'std_function_invocations.csv' + +def main(): + parser = argparse.ArgumentParser( + description="Aggregates std function invocations from multiple JSON files into a single CSV report." + ) + + parser.add_argument( + 'inputs', + nargs='+', + type=pathlib.Path, + help='Paths to the JSON files generated by test groups', + ) + + parser.add_argument( + '--output', '-o', + type=pathlib.Path, + default=DEFAULT_OUTPUT, + help=f'Path to the output CSV file (default: {DEFAULT_OUTPUT})', + ) + + args = parser.parse_args() + total_counts = collections.defaultdict(int) + + for file_path in args.inputs: + if file_path.exists(): + try: + data = json.loads(file_path.read_text(encoding='utf-8')) + if isinstance(data, dict): + for func_name, count in data.items(): + total_counts[func_name] += count + else: + print(f"Warning: {file_path} content is not a dictionary. Skipping.") + except json.JSONDecodeError: + print(f"Error: {file_path} is not a valid JSON file. Skipping.") + else: + print(f"Info: {file_path} not found. It might have been disabled.") + + args.output.parent.mkdir(parents=True, exist_ok=True) + + try: + with args.output.open('w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow(['Built-in', 'Call count']) + + sorted_results = sorted(total_counts.items(), key=lambda item: (item[1], item[0])) + + for func, count in sorted_results: + writer.writerow([func, count]) + + print(f"Success! Aggregated report saved to: {args.output.absolute()}") + except Exception as e: + print(f"Error saving report: {e}") + +if __name__ == "__main__": + main() diff --git a/tests/python/lib/k2_kphp_tracked_builtins_list.txt b/tests/k2_kphp_tracked_builtins_list.txt similarity index 80% rename from tests/python/lib/k2_kphp_tracked_builtins_list.txt rename to tests/k2_kphp_tracked_builtins_list.txt index 137d468b5e..d72ec258bb 100644 --- a/tests/python/lib/k2_kphp_tracked_builtins_list.txt +++ b/tests/k2_kphp_tracked_builtins_list.txt @@ -353,15 +353,11 @@ rpc_queue_next rpc_server_fetch_request rpc_server_store_response rpc_tl_pending_queries_count -rpc_tl_query -rpc_tl_query_result -rpc_tl_query_result_synchronously rsort rtrim sched_yield sched_yield_sleep serialize -set_detect_incorrect_encoding_names_warning set_fail_rpc_on_int32_overflow set_json_log_on_timeout_mode set_migration_php8_warning @@ -413,9 +409,6 @@ tan time to_array_debug trim -typed_rpc_tl_query -typed_rpc_tl_query_result -typed_rpc_tl_query_result_synchronously uasort UberH3$$geoToH3 UberH3$$h3ToParent @@ -454,3 +447,81 @@ warning wordwrap zstd_compress zstd_uncompress +flush +getopt +is_readable +kphp_extended_instance_cache_metrics_init +md5_file +fgetcsv +fgets +file_exists +file_put_contents +prepare_search_query +set_json_log_demangle_stacktrace +strftime +ini_set +get_webserver_stats +thread_pool_test_load +chmod +getimagesize +copy +dirname +feof +filemtime +fputcsv +fseek +ftell +is_dir +mkdir +rename +rewind +scandir +tempnam +stream_context_create +kphp_job_worker_fetch_request +kphp_job_worker_start +kphp_job_worker_start_multi +kphp_job_worker_start_no_reply +kphp_job_worker_store_response +kphp_backtrace +kphp_turn_on_host_tag_in_inner_statshouse_metrics_toggle +memory_get_static_usage +openssl_x509_checkpurpose +openssl_x509_verify +preg_last_error +new_rpc_connection +store_finish +fetch_lookup_int +gethostbynamel +localtime +system +kphp_tracing_func_enter_branch +kphp_tracing_get_current_active_span +kphp_tracing_get_level +kphp_tracing_get_root_span +kphp_tracing_init +kphp_tracing_register_enums_provider +kphp_tracing_register_on_finish +kphp_tracing_register_rpc_details_provider +msgpack_deserialize_safe +kphp_tracing_set_level +kphp_tracing_start_span +KphpDiv$$assignTraceCtx +KphpDiv$$generateTraceCtxForChild +preg_split +KphpSpan$$addAttributeBool +KphpSpan$$addAttributeFloat +KphpSpan$$addAttributeInt +KphpSpan$$addAttributeString +KphpSpan$$addEvent +KphpSpan$$exclude +KphpSpan$$finish +KphpSpan$$finishWithError +KphpSpan$$updateName +KphpSpanEvent$$addAttributeInt +KphpSpanEvent$$addAttributeString +ignore_user_abort +send_http_103_early_hints +profiler_set_function_label +vk_dot_product +vk_flex diff --git a/tests/kphp_ci_pipeline_runner.py b/tests/kphp_ci_pipeline_runner.py index a3a3e37b27..1d915cb68c 100755 --- a/tests/kphp_ci_pipeline_runner.py +++ b/tests/kphp_ci_pipeline_runner.py @@ -4,6 +4,8 @@ import math import multiprocessing import os +import pathlib +import shlex import signal import subprocess import sys @@ -13,6 +15,10 @@ from python.lib.nocc_for_kphp_tester import nocc_env +K2_KPHP_TRACKED_BUILTINS_LIST_PATH = pathlib.Path(__file__).parent / "k2_kphp_tracked_builtins_list.txt" +K2_KPHP_TRACKED_BUILTINS_LIST = K2_KPHP_TRACKED_BUILTINS_LIST_PATH.read_text() + + class TestStatus(Enum): FAILED = red("failed") PASSED = green("passed") @@ -348,9 +354,11 @@ def _calculate_pytest_jobs_count(default_percent: int = 95) -> int: name="k2-kphp-tests", description="run k2-kphp tests with cxx={}".format(args.cxx_name), cmd="KPHP_TESTS_POLYFILLS_REPO={kphp_polyfills_repo} " + "KPHP_TRACKED_BUILTINS_LIST={K2_KPHP_TRACKED_BUILTINS_LIST} " "{kphp_runner} -j{jobs} --cxx-name {cxx_name} --k2-bin {k2_bin}".format( jobs=n_cpu, kphp_polyfills_repo=kphp_polyfills_repo, + K2_KPHP_TRACKED_BUILTINS_LIST=shlex.quote(K2_KPHP_TRACKED_BUILTINS_LIST), kphp_runner=kphp_test_runner, cxx_name=args.cxx_name, k2_bin=args.k2_bin, @@ -427,10 +435,17 @@ def _calculate_pytest_jobs_count(default_percent: int = 95) -> int: runner.add_test_group( name="k2-functional-tests", description="run k2-kphp functional tests with cxx={}".format(args.cxx_name), - cmd="KPHP_TESTS_POLYFILLS_REPO={kphp_polyfills_repo} KPHP_CXX={cxx_name} K2_BIN={k2_bin} K2_MONITORING_ABORT_HANGING_GLOBAL_TASK=false K2_MONITORING_ABORT_HANGING_REQUEST_TASK=false python3 -m pytest --basetemp={base_tempdir} --tb=native -n{jobs} {functional_tests_dir}".format( + cmd="KPHP_TESTS_POLYFILLS_REPO={kphp_polyfills_repo} " + "KPHP_CXX={cxx_name} " + "K2_BIN={k2_bin} " + "K2_MONITORING_ABORT_HANGING_GLOBAL_TASK=false " + "K2_MONITORING_ABORT_HANGING_REQUEST_TASK=false " + "KPHP_TRACKED_BUILTINS_LIST={K2_KPHP_TRACKED_BUILTINS_LIST} " + "python3 -m pytest --basetemp={base_tempdir} --tb=native -n{jobs} {functional_tests_dir}".format( kphp_polyfills_repo=kphp_polyfills_repo, cxx_name=args.cxx_name, k2_bin=args.k2_bin, + K2_KPHP_TRACKED_BUILTINS_LIST=shlex.quote(K2_KPHP_TRACKED_BUILTINS_LIST), jobs=n_cpu, functional_tests_dir=functional_tests_dir, base_tempdir=os.path.expanduser( @@ -474,6 +489,7 @@ def _calculate_pytest_jobs_count(default_percent: int = 95) -> int: "KPHP_TESTS_INTERGRATION_TESTS_ENABLED=1 " "KPHP_CXX={cxx_name} " "K2_BIN={k2_bin} " + "KPHP_TRACKED_BUILTINS_LIST={K2_KPHP_TRACKED_BUILTINS_LIST} " "python3 -m pytest --tb=native -n{jobs} {tests_dir}".format( jobs=n_cpu, lib_dir=os.path.join(runner_dir, "python"), @@ -482,6 +498,7 @@ def _calculate_pytest_jobs_count(default_percent: int = 95) -> int: kphp_polyfills_repo=kphp_polyfills_repo, cxx_name=args.cxx_name, k2_bin=args.k2_bin, + K2_KPHP_TRACKED_BUILTINS_LIST=shlex.quote(K2_KPHP_TRACKED_BUILTINS_LIST), tests_dir=" ".join([ os.path.join(args.kphp_tests_repo, "python/tests/k2_rpc_client/"), os.path.join(args.kphp_tests_repo, "python/tests/k2_rpc_server/"), diff --git a/tests/kphp_tester.py b/tests/kphp_tester.py index fe653525d7..940fb54b3a 100755 --- a/tests/kphp_tester.py +++ b/tests/kphp_tester.py @@ -3,20 +3,26 @@ import math import multiprocessing import os +import pathlib import re import signal import sys from functools import partial from multiprocessing.dummy import Pool as ThreadPool +import typing from python.lib.colors import red, green, yellow, blue, cyan from python.lib.file_utils import search_php_bin from python.lib.nocc_for_kphp_tester import nocc_start_daemon_in_background from python.lib.kphp_run_once import KphpRunOnce +from python.lib import std_function from python.lib import tcp TCP_SERVER_TAG_PREFIX = "tcp_server:" +FILE = pathlib.Path(__file__) +TMP_DIR = FILE.with_name("{}_tmp".format(FILE.stem)) + class TestFile: def __init__(self, file_path, test_tmp_dir, tags, env_vars: dict, out_regexps=None, forbidden_regexps=None): @@ -55,13 +61,16 @@ def is_php8(self): def is_available_for_k2(self): return "k2_skip" not in self.tags - def make_kphp_once_runner(self, use_nocc, cxx_name, k2_bin): + def make_kphp_once_runner( + self, use_nocc, cxx_name, k2_bin, std_function_invocations: typing.Optional[std_function.Invocations] + ): tester_dir = os.path.abspath(os.path.dirname(__file__)) return KphpRunOnce( php_script_path=self.file_path, working_dir=os.path.abspath(os.path.join(self.test_tmp_dir, "working_dir")), artifacts_dir=os.path.abspath(os.path.join(self.test_tmp_dir, "artifacts")), php_bin=search_php_bin(php_version=self.php_version), + std_function_invocations=std_function_invocations, extra_include_dirs=[os.path.join(tester_dir, "php_include")], vkext_dir=os.path.abspath(os.path.join(tester_dir, os.path.pardir, "objs", "vkext")), use_nocc=use_nocc, @@ -75,7 +84,6 @@ def set_up_env_for_k2(self): self.env_vars["KPHP_ENABLE_GLOBAL_VARS_MEMORY_STATS"] = "0" self.env_vars["KPHP_PROFILER"] = "0" self.env_vars["KPHP_FORCE_LINK_RUNTIME"] = "1" - self.env_vars["KPHP_TRACKED_BUILTINS_LIST"] = KphpRunOnce.K2_KPHP_TRACKED_BUILTINS_LIST def make_test_file(file_path, test_tmp_dir, test_tags): @@ -152,12 +160,11 @@ def test_files_from_list(tests_dir, test_list): def collect_tests(tests_dir, test_tags, test_list): tests = [] - tmp_dir = "{}_tmp".format(__file__[:-3]) file_it = test_files_from_list(tests_dir, test_list) if test_list else test_files_from_dir(tests_dir) for root, file in file_it: if file.endswith(".php") or file.endswith(".phpt"): test_file_path = os.path.join(root, file) - test_tmp_dir = os.path.join(tmp_dir, os.path.relpath(test_file_path, os.path.dirname(tests_dir))) + test_tmp_dir = os.path.join(TMP_DIR, os.path.relpath(test_file_path, os.path.dirname(tests_dir))) test_tmp_dir = test_tmp_dir[:-4] if test_tmp_dir.endswith(".php") else test_tmp_dir[:-5] test_file = make_test_file(test_file_path, test_tmp_dir, test_tags) if test_file: @@ -350,11 +357,13 @@ def run_ok_test(test: TestFile, runner): return TestResult.passed(test, runner.artifacts) -def run_test(use_nocc, cxx_name, k2_bin, test: TestFile): +def run_test( + use_nocc, cxx_name, k2_bin, std_function_invocations: typing.Optional[std_function.Invocations], test: TestFile +): if not os.path.exists(test.file_path): return TestResult.failed(test, None, "can't find test file") - runner = test.make_kphp_once_runner(use_nocc, cxx_name, k2_bin) + runner = test.make_kphp_once_runner(use_nocc, cxx_name, k2_bin, std_function_invocations) runner.remove_artifacts_dir() if k2_bin is not None: test.set_up_env_for_k2() @@ -393,10 +402,15 @@ def run_all_tests(tests_dir, jobs, test_tags, no_report, passed_list, test_list, "tag" if len(test_tags) == 1 else "tags")) sys.exit(1) + if "KPHP_TRACKED_BUILTINS_LIST" in os.environ: + std_function_invocations = std_function.Invocations(os.environ["KPHP_TRACKED_BUILTINS_LIST"]) + else: + std_function_invocations = None + results = [] with ThreadPool(jobs) as pool: tests_completed = 0 - for test_result in pool.imap_unordered(partial(run_test, use_nocc, cxx_name, k2_bin), tests): + for test_result in pool.imap_unordered(partial(run_test, use_nocc, cxx_name, k2_bin, std_function_invocations), tests): if hack_reference_exit: print(yellow("Testing process was interrupted"), flush=True) break @@ -404,6 +418,16 @@ def run_all_tests(tests_dir, jobs, test_tags, no_report, passed_list, test_list, test_result.print_short_report(len(tests), tests_completed) results.append(test_result) + if std_function_invocations: + std_function_invocations_output_dir = TMP_DIR / "artifacts" + std_function_invocations_output_dir.mkdir(parents=True, exist_ok=True) + + std_function_invocations_filename = "std_function_invocations.json" + std_function_invocations_output_path = std_function_invocations_output_dir / std_function_invocations_filename + + with open(std_function_invocations_output_path, "w", encoding="utf-8") as f: + std_function_invocations.dump(f) + print("\nTesting results:", flush=True) skipped = len(tests) - len(results) diff --git a/tests/python/lib/conftest_impl.py b/tests/python/lib/conftest_impl.py index 7fd046c988..fc88d7ffff 100644 --- a/tests/python/lib/conftest_impl.py +++ b/tests/python/lib/conftest_impl.py @@ -1,7 +1,29 @@ import os +import shutil +import pathlib +import uuid import pytest from .file_utils import search_k2_bin +from . import std_function +from . import testcase + + +def _sync_data(tmp_dir: str, test_parent_dir: pathlib.Path): + data_dir = test_parent_dir / "php/data" + tmp_data_dir = pathlib.Path(tmp_dir) / "data" + + if data_dir.is_dir(): + tmp_data_dir.mkdir(parents=True, exist_ok=True) + for full_data_file in data_dir.iterdir(): + full_tmp_file = tmp_data_dir / full_data_file.name + if full_tmp_file.exists(): + continue + + if full_data_file.is_file(): + shutil.copy(full_data_file, tmp_data_dir) + elif full_data_file.is_dir(): + shutil.copytree(full_data_file, full_tmp_file) @pytest.fixture(autouse=True) @@ -37,3 +59,65 @@ def skip_kphp_unsupported_test_suite(request): request.cls.custom_setup = lambda: None request.cls.custom_teardown = lambda: None pytest.skip("KPHP skipped test") + + +@pytest.fixture(scope="session") +def session_tmp_dir(request: pytest.FixtureRequest): + return request.config.rootpath.parent / "_tmp" + + +@pytest.fixture(scope="class") +def class_tmp_dir(request: pytest.FixtureRequest, session_tmp_dir: pathlib.Path): + relative_subpath = pathlib.Path(request.module.__file__).parent.relative_to(request.config.rootpath) + + return session_tmp_dir / relative_subpath + + +@pytest.fixture(scope="class") +def kphp_build_working_dir(class_tmp_dir: pathlib.Path): + res = class_tmp_dir / "working_dir" + res.mkdir(parents=True, exist_ok=True) + return res + + +@pytest.fixture(scope="class") +def artifacts_dir(class_tmp_dir: pathlib.Path): + res = class_tmp_dir / "artifacts" + res.mkdir(parents=True, exist_ok=True) + return res + + +@pytest.fixture(scope="class") +def tmp_dir_root(request: pytest.FixtureRequest, artifacts_dir: pathlib.Path): + test_suite_name = pathlib.Path(request.module.__file__).stem + res = artifacts_dir / "tmp_{}".format(test_suite_name) + res.mkdir(parents=True, exist_ok=True) + return res + + + +@pytest.fixture(scope="class") +def kphp_server_working_dir(request: pytest.FixtureRequest, tmp_dir_root: pathlib.Path): + server_working_dir = testcase.make_test_tmp_dir(tmp_dir_root) + _sync_data(server_working_dir, pathlib.Path(request.module.__file__).parent) + return server_working_dir + + +@pytest.fixture(scope="session") +def std_function_invocations(session_tmp_dir: pathlib.Path): + if "KPHP_TRACKED_BUILTINS_LIST" not in os.environ: + yield None + return + + function_invocations = std_function.Invocations(os.environ["KPHP_TRACKED_BUILTINS_LIST"]) + + yield function_invocations + + output_dir = session_tmp_dir / "artifacts" + output_dir.mkdir(parents=True, exist_ok=True) + + filename = f"{uuid.uuid4()}.json" + output_path = output_dir / filename + + with open(output_path, "w", encoding="utf-8") as f: + function_invocations.dump(f) diff --git a/tests/python/lib/engine.py b/tests/python/lib/engine.py index ab066cbaab..34643d2ab2 100644 --- a/tests/python/lib/engine.py +++ b/tests/python/lib/engine.py @@ -1,5 +1,6 @@ import atexit import os +import typing import psutil import re @@ -44,7 +45,7 @@ def __init__(self, engine_bin, working_dir, options=None): self._engine_process = None self._log_file_write_fd = None self._log_file_read_fd = None - self._engine_logs = [] + self._engine_logs: typing.List[str] = [] self._binlog_path = None self._ignore_log_errors = False if options: @@ -173,6 +174,8 @@ def stop(self, signo = signal.SIGTERM): if not self._ignore_log_errors: self._assert_no_errors() + else: + self._read_new_logs() self._log_file_read_fd.close() self._log_file_write_fd.close() self._stats_receiver.stop() diff --git a/tests/python/lib/k2_server.py b/tests/python/lib/k2_server.py index 9fa4c77a04..493de94c09 100644 --- a/tests/python/lib/k2_server.py +++ b/tests/python/lib/k2_server.py @@ -30,6 +30,8 @@ def __init__(self, k2_server_bin, working_dir, kphp_build_dir, options=None, aut "--linking": self._linking_file} os.environ["RUNTIME_CONFIG_PATH"] = os.path.join(working_dir, "data/runtime_configuration.json") + if "RUST_LOG" not in os.environ: + os.environ["RUST_LOG"] = "Debug" if options: self.update_options(options) @@ -81,3 +83,9 @@ def _process_json_log(self, log_record): # remove trace log_record.pop("trace", "") return log_record + + def get_log(self): + if self._is_json_log_enabled(): + return list(map(json.dumps, self.get_json_log())) + + return super(K2Server, self).get_log() diff --git a/tests/python/lib/kphp_run_once.py b/tests/python/lib/kphp_run_once.py index 49db1d4885..afeed27f86 100644 --- a/tests/python/lib/kphp_run_once.py +++ b/tests/python/lib/kphp_run_once.py @@ -1,22 +1,28 @@ -import collections import os -import pathlib -import re import shutil import subprocess import sys +import typing +from . import std_function from .kphp_builder import KphpBuilder from .file_utils import error_can_be_ignored class KphpRunOnce(KphpBuilder): - K2_KPHP_TRACKED_BUILTINS_LIST_PATH = pathlib.Path(__file__).parent / "k2_kphp_tracked_builtins_list.txt" - K2_KPHP_TRACKED_BUILTINS_LIST = K2_KPHP_TRACKED_BUILTINS_LIST_PATH.read_text() - K2_BUILTIN_CALLED_MSG_RE = re.compile(b"built-in called: ([\w$]+)") - - def __init__(self, php_script_path, artifacts_dir, working_dir, php_bin, - extra_include_dirs=None, vkext_dir=None, use_nocc=False, cxx_name="g++", k2_bin=None): + def __init__( + self, + php_script_path, + artifacts_dir, + working_dir, + php_bin, + std_function_invocations: typing.Optional[std_function.Invocations], + extra_include_dirs=None, + vkext_dir=None, + use_nocc=False, + cxx_name="g++", + k2_bin=None + ): super(KphpRunOnce, self).__init__( php_script_path=php_script_path, artifacts_dir=artifacts_dir, @@ -33,8 +39,8 @@ def __init__(self, php_script_path, artifacts_dir, working_dir, php_bin, self._include_dirs.extend(extra_include_dirs) self._vkext_dir = vkext_dir self._php_bin = php_bin + self._std_function_invocations= std_function_invocations self.k2_bin = k2_bin - self.builtin_calls = collections.defaultdict(int) def _get_extensions(self): if sys.platform == "darwin": @@ -128,10 +134,15 @@ def run_with_kphp_server(self, runs_cnt=1, args=[]): self._move_sanitizer_logs_to_artifacts(sanitizer_glob_mask, kphp_server_proc, sanitizer_log_name) ignore_stderr = error_can_be_ignored( ignore_patterns=[ - "^\\[\\d+\\]\\[\\d{4}\\-\\d{2}\\-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d+ php\\-runner\\.cpp\\s+\\d+\\].+$" + "^\\[\\d+\\]\\[\\d{4}\\-\\d{2}\\-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d+ php\\-runner\\.cpp\\s+\\d+\\].+$", + ".*Debug.*", + ".*Info.*", ], binary_error_text=kphp_runtime_stderr) + if self._std_function_invocations: + self._std_function_invocations.update(kphp_runtime_stderr) + if not ignore_stderr: self._kphp_runtime_stderr = self._move_to_artifacts( "kphp_runtime_stderr", @@ -149,7 +160,7 @@ def run_with_kphp_and_k2(self, runs_cnt=1, args=[]): env = os.environ.copy() if "RUST_LOG" not in env: - env["RUST_LOG"] = "Warn,component-log=Debug" + env["RUST_LOG"] = "Debug" k2_runtime_proc = subprocess.Popen(cmd, cwd=self._kphp_runtime_tmp_dir, @@ -168,9 +179,9 @@ def run_with_kphp_and_k2(self, runs_cnt=1, args=[]): "k2_runtime_stderr", k2_runtime_proc.returncode, content=_kphp_server_stderr) - - for match in self.K2_BUILTIN_CALLED_MSG_RE.finditer(_kphp_server_stderr): - self.builtin_calls[match[1]] += 1 + + if self._std_function_invocations: + self._std_function_invocations.update(_kphp_server_stderr) return k2_runtime_proc.returncode == 0 diff --git a/tests/python/lib/std_function.py b/tests/python/lib/std_function.py new file mode 100644 index 0000000000..cbf31dd319 --- /dev/null +++ b/tests/python/lib/std_function.py @@ -0,0 +1,22 @@ +import json +import re +import typing + + +class Invocations: + STD_FUNCTION_MSG_RE = re.compile(b"std function called: ([\\w$]+)") + + def __init__(self, kphp_tracked_builtins_list: str): + self._data = {builtin: 0 for builtin in kphp_tracked_builtins_list.split()} + + def update(self, logs: bytes): + for match in self.STD_FUNCTION_MSG_RE.finditer(logs): + self._data[match[1].decode()] += 1 + + def dump(self, target_file: typing.TextIO): + """ + Write the current statistics as JSON into the provided file descriptor. + + The caller is responsible for opening and closing the file. + """ + json.dump(self._data, target_file, indent=4) diff --git a/tests/python/lib/testcase.py b/tests/python/lib/testcase.py index fda3e6939f..cb6a659fb9 100644 --- a/tests/python/lib/testcase.py +++ b/tests/python/lib/testcase.py @@ -8,7 +8,10 @@ from unittest import TestCase +import pytest + from .kphp_server import KphpServer +from . import std_function from .k2_server import K2Server from .kphp_builder import KphpBuilder from .kphp_run_once import KphpRunOnce @@ -19,42 +22,7 @@ logging.disable(logging.DEBUG) -def _sync_data(tmp_dir, test_parent_dir): - data_dir = os.path.join(test_parent_dir, "php/data") - tmp_data_dir = os.path.join(tmp_dir, "data") - - if os.path.isdir(data_dir): - os.makedirs(tmp_data_dir, exist_ok=True) - for data_file in os.listdir(data_dir): - full_tmp_file = os.path.join(tmp_data_dir, data_file) - if os.path.exists(full_tmp_file): - continue - - full_data_file = os.path.join(data_dir, data_file) - if os.path.isfile(full_data_file): - shutil.copy(full_data_file, tmp_data_dir) - elif os.path.isdir(full_data_file): - shutil.copytree(full_data_file, full_tmp_file) - - -def _get_tmp_folder_path(test_script_file): - test_script_dir = os.path.dirname(os.path.realpath(test_script_file)) - tests_root_dir = test_script_dir - while not tests_root_dir.endswith("python/tests"): - tests_root_dir = os.path.dirname(tests_root_dir) - if "python/tests" not in tests_root_dir: - raise RuntimeError("Can't find tests root dir") - - python_tests_dir = os.path.dirname(tests_root_dir) - tmp_dir = os.path.join(python_tests_dir, "_tmp/", test_script_dir[len(tests_root_dir) + 1:]) - test_suite_name, _ = os.path.splitext(os.path.basename(test_script_file)) - working_dir = os.path.join(tmp_dir, "working_dir") - artifacts_dir = os.path.join(tmp_dir, "artifacts") - tmp_dir_root = os.path.join(artifacts_dir, "tmp_{}".format(test_suite_name)) - return working_dir, tmp_dir_root, artifacts_dir, test_script_dir - - -def _make_test_tmp_dir(tmp_dir_root): +def make_test_tmp_dir(tmp_dir_root): all_dirs = next(os.walk(tmp_dir_root))[1] ppid = str(os.getppid()) for tmp_dir in all_dirs: @@ -70,32 +38,28 @@ def _make_test_tmp_dir(tmp_dir_root): return test_tmp_dir -def _create_tmp_folders(test_script_file): - kphp_build_working_dir, tmp_dir_root, artifacts_dir, test_script_dir = _get_tmp_folder_path(test_script_file) - for test_dir in (kphp_build_working_dir, tmp_dir_root, artifacts_dir): - os.makedirs(test_dir, exist_ok=True) - - kphp_server_working_dir = _make_test_tmp_dir(tmp_dir_root) - _sync_data(kphp_server_working_dir, test_script_dir) - return kphp_build_working_dir, kphp_server_working_dir, artifacts_dir, test_script_dir - - class BaseTestCase(TestCase): kphp_build_working_dir = "" web_server_working_dir = "" artifacts_dir = "" test_dir = "" - @classmethod - def _setup_tmp_folder(cls): - script_file = sys.modules.get(cls.__module__).__file__ - cls.kphp_build_working_dir, cls.web_server_working_dir, cls.artifacts_dir, cls.test_dir = \ - _create_tmp_folders(script_file) - - @classmethod - def setup_class(cls): - cls._setup_tmp_folder() - cls.custom_setup() + @pytest.fixture(scope="class") + def setup_tmp_folder( + self, + request: pytest.FixtureRequest, + kphp_build_working_dir: pathlib.Path, + kphp_server_working_dir: pathlib.Path, + artifacts_dir: pathlib.Path, + ): + request.cls.kphp_build_working_dir = str(kphp_build_working_dir) + request.cls.web_server_working_dir = str(kphp_server_working_dir) + request.cls.artifacts_dir = str(artifacts_dir) + request.cls.test_dir = str(pathlib.Path(request.module.__file__).parent) + + @pytest.fixture(scope="class", autouse=True) + def _base_setup(self, request: pytest.FixtureRequest, setup_tmp_folder): + request.cls.custom_setup() @classmethod def teardown_class(cls): @@ -186,6 +150,11 @@ class WebServerAutoTestCase(BaseTestCase): kphp_builder = None sanitizer_pattern = None + + @pytest.fixture(scope="class", autouse=True) + def web_server_std_functions_updater(self, request, std_function_invocations): + request.cls.std_function_invocations = std_function_invocations + @classmethod def custom_setup(cls): if cls.should_use_nocc(): @@ -247,6 +216,9 @@ def custom_setup(cls): @classmethod def custom_teardown(cls): cls.web_server.stop() + if cls.std_function_invocations: + for line in cls.web_server._engine_logs: + cls.std_function_invocations.update(line.encode()) cls.extra_class_teardown() if not cls.should_use_k2(): try: @@ -261,6 +233,9 @@ def custom_setup_method(self, method): pass def custom_teardown_method(self, method): + if self.std_function_invocations: + for line in self.web_server._engine_logs: + self.std_function_invocations.update(line.encode()) self.web_server._engine_logs = [] @classmethod @@ -298,8 +273,7 @@ def should_use_k2(cls): @classmethod def kphp_env_for_k2_server_component(cls): env = {"KPHP_MODE": "k2-server", "KPHP_ENABLE_FULL_PERFORMANCE_ANALYZE": "0", - "KPHP_PROFILER": "0", "KPHP_USER_BINARY_PATH": "component.so", "KPHP_FORCE_LINK_RUNTIME": "1", - "KPHP_TRACKED_BUILTINS_LIST": KphpRunOnce.K2_KPHP_TRACKED_BUILTINS_LIST} + "KPHP_PROFILER": "0", "KPHP_USER_BINARY_PATH": "component.so", "KPHP_FORCE_LINK_RUNTIME": "1"} return env def assertKphpNoTerminatedRequests(self): @@ -325,6 +299,10 @@ def assertKphpNoTerminatedRequests(self): class KphpCompilerAutoTestCase(BaseTestCase): once_runner_trash_bin = [] + @pytest.fixture(scope="class", autouse=True) + def kphp_compiler_std_functions(self, request, std_function_invocations): + request.cls.std_function_invocations = std_function_invocations + def __init__(self, method_name): super().__init__(method_name) self.php_version = "php7.4" @@ -347,7 +325,7 @@ def extra_class_teardown(cls): @classmethod def custom_setup(cls): - cls.kphp_build_working_dir = _make_test_tmp_dir(cls.kphp_build_working_dir) + cls.kphp_build_working_dir = make_test_tmp_dir(cls.kphp_build_working_dir) cls.extra_class_setup() @classmethod @@ -389,6 +367,7 @@ def make_kphp_once_runner(self, php_script_path): artifacts_dir=self.web_server_working_dir, working_dir=self.kphp_build_working_dir, php_bin=search_php_bin(php_version=self.php_version), + std_function_invocations=self.std_function_invocations, use_nocc=self.should_use_nocc(), k2_bin=os.path.abspath(search_k2_bin()) if self.should_use_k2() else None, ) diff --git a/tests/python/lib/web_server.py b/tests/python/lib/web_server.py index d47cfd0df7..195a33adc0 100644 --- a/tests/python/lib/web_server.py +++ b/tests/python/lib/web_server.py @@ -92,6 +92,10 @@ def _read_new_json_logs(self): def _process_json_log(self, log_record): return log_record + def get_json_log(self): + self._read_new_json_logs() + return self._json_logs + def assert_json_log(self, expect, message="Can't wait expected json log", timeout=60): """ Check kphp server json log diff --git a/tests/python/tests/conftest.py b/tests/python/tests/conftest.py index 4af7ee6c4a..fd5cb9d26d 100644 --- a/tests/python/tests/conftest.py +++ b/tests/python/tests/conftest.py @@ -1,4 +1,4 @@ import os import pytest -from python.lib.conftest_impl import skip_k2_unsupported_test, skip_k2_unsupported_test_suite, skip_kphp_unsupported_test, skip_kphp_unsupported_test_suite +pytest_plugins = ["python.lib.conftest_impl"]