diff --git a/Dockerfile b/Dockerfile index b7ebae39..37caac52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,12 @@ ENV PYTHONPATH=$PYTHONPATH:/dislib ENV LC_ALL=C.UTF-8 RUN python3 -m pip install --upgrade -r /dislib/requirements.txt +# RUN apt-get update && apt-get install -y htop + ENV COMPSS_LOAD_SOURCE false +# RUN sed -i "s/>45 /dev/null || true' - sh 'docker rmi -f bscwdc/dislib &> /dev/null || true' } success { setGithubCommitStatus('success', 'Build Successful') diff --git a/run_tests.sh b/run_tests.sh index e68f5ef0..249efe37 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,13 +1,48 @@ #!/bin/bash -e # Default process per worker -export ComputingUnits=4 +export ComputingUnits=2 -# Run the tests/__main__.py file which calls all the tests named test_*.py -runcompss \ - --pythonpath=$(pwd) \ - --python_interpreter=python3 \ - ./tests/__main__.py &> >(tee output.log) +declare -a tests_group=("test_lasso" + "test_array test_pca test_daura" + "test_gm test_preproc test_decision_tree" + "test_qr test_kmeans test_knn" + "test_gridsearch test_tsqr test_linear_regression" + "test_dbscan test_matmul test_als" + "test_rf_classifier test_randomizedsearch test_data_utils test_kfold" + "test_csvm test_rf_regressor test_utils test_rf_dataset" + ) + +declare -a pids + +port=43000 +workerid=1 + +for t in "${tests_group[@]}" +do + + nextport=$((port + 1)) + + sed "s/43001<\/MinPort>/$port<\/MinPort>/g" /opt/COMPSs/Runtime/configuration/xml/resources/default_resources.xml > /tmp/resources-$port.xml + sed -i "s/43002<\/MaxPort>/$nextport<\/MaxPort>/g" /tmp/resources-$port.xml + + runcompss \ + --log_level=debug \ + --pythonpath=$(pwd) \ + --python_interpreter=python3 \ + --resources=/tmp/resources-$port.xml \ + --master_port=$port \ + ./tests/__main__.py $t -id $workerid &> >(tee output.log) & + + pids+=($!) + + workerid=$((workerid + 1)) + + port=$((port + 2)) + sleep 10 +done + +wait ${pids[@]} # Check the unittest output because PyCOMPSs exits with code 0 even if there # are failed tests (the execution itself is successful) @@ -15,6 +50,7 @@ result=$(cat output.log | egrep "OK|FAILED") echo "Tests result: ${result}" + # If word Failed is in the results, exit 1 so the pull request fails if [[ $result =~ FAILED ]]; then exit 1 diff --git a/tests/__init__.py b/tests/__init__.py index 62bea21c..24ed8a2c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,12 +1,16 @@ -from time import time +import datetime import unittest class BaseTimedTestCase(unittest.TestCase): def setUp(self): - self.start_time = time() + # self.start_time = time() + print(f'Test {self.id()} START AT', + datetime.datetime.now(), flush=True) def tearDown(self): - self.end_time = time() - print("Test %s took: %.3f seconds" % - (self.id(), self.end_time - self.start_time)) + print(f'Test {self.id()} END AT', + datetime.datetime.now(), flush=True) + # self.end_time = time() + # print("Test %s took: %.3f seconds" % + # (self.id(), self.end_time - self.start_time)) diff --git a/tests/__main__.py b/tests/__main__.py index ffc51850..bb4faef3 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,9 +1,31 @@ import unittest +import argparse +import datetime +if __name__ == '__main__': + suite = list(unittest.loader.defaultTestLoader.discover('./tests/')) + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('tests', metavar='T', type=str, nargs='+', + help='an integer for the accumulator') + parser.add_argument('-id', type=int) -def load_tests(loader, tests, pattern): - return loader.discover('./tests/') + args = parser.parse_args() + print(f'WORKER {args.id} FIND TESTS', + datetime.datetime.now(), flush=True) -if __name__ == '__main__': - unittest.main(verbosity=2) + tests_to_run = [] + for t in args.tests: + for test_case in suite: + if t.lower() in str(test_case).lower(): + tests_to_run.append(test_case) + + print(f'WORKER {args.id} START TEST AT', + datetime.datetime.now(), flush=True) + + test_suite = unittest.TestSuite() + test_suite.addTests(tests_to_run) + unittest.TextTestRunner(verbosity=2).run(test_suite) + + print(f'WORKER {args.id} END TEST AT', + datetime.datetime.now(), flush=True)