From 361da9ddcea892088e4fad6cc788d44e8de63f56 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Fri, 5 Aug 2022 17:06:31 +0200 Subject: [PATCH 01/28] parallel tests --- tests/__main__.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index ffc51850..e8f4bf29 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,9 +1,23 @@ import unittest +import threading -def load_tests(loader, tests, pattern): +def load_tests(loader): return loader.discover('./tests/') +def run_test(test): + suite = unittest.TestSuite() + suite.addTest(test) + unittest.TextTestRunner(verbosity=2).run(suite) + + if __name__ == '__main__': - unittest.main(verbosity=2) + suite = load_tests(unittest.loader.defaultTestLoader) + threads = [] + for case in suite: + thread = threading.Thread(target=run_test, args=(case,)) + thread.start() + threads.append(thread) + + [t.join() for t in threads] From d5a5fa35473f4548f862e84e4b8c7d82447ca683 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Mon, 8 Aug 2022 12:08:20 +0200 Subject: [PATCH 02/28] Update __main__.py --- tests/__main__.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index e8f4bf29..0e2543af 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,23 +1,32 @@ import unittest -import threading +import multiprocessing as mp +from concurrent.futures import ThreadPoolExecutor +T = [["test_lasso"], + ["test_array", "test_pca", "test_daura"], + ["test_gm", "test_preproc", "test_decision_tree"], + ["test_qr", "test_kmeans", "test_knn"], + ["test_gridsearch", "test_tsqr", "test_linear_regression"], + ["test_dbscan", "test_matmul", "test_als"], + ["test_rf_classifier", "test_randomizedsearch", "test_data_utils", "test_kfold"], + ["test_csvm", "test_rf_regressor", "test_utils", "test_rf_dataset"]] -def load_tests(loader): - return loader.discover('./tests/') - - -def run_test(test): - suite = unittest.TestSuite() - suite.addTest(test) - unittest.TextTestRunner(verbosity=2).run(suite) - +def run_tests(tests_to_run): + test_suite = unittest.TestSuite() + test_suite.addTests(tests_to_run) + unittest.TextTestRunner(verbosity=2).run(test_suite) if __name__ == '__main__': - suite = load_tests(unittest.loader.defaultTestLoader) - threads = [] - for case in suite: - thread = threading.Thread(target=run_test, args=(case,)) - thread.start() - threads.append(thread) + suite = list(unittest.loader.defaultTestLoader.discover('./tests/')) - [t.join() for t in threads] + tests_to_run = [] + for tests in T: + ttt = [] + for t in tests: + for test_case in suite: + if t.lower() in str(test_case).lower(): + ttt.append(test_case) + tests_to_run.append(ttt) + + with ThreadPoolExecutor(max_workers=mp.cpu_count) as exec: + exec.map(run_tests, tests_to_run) From ec004fbd328e9d39c5baaa13a03d15ebd7be0fab Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 11 Aug 2022 11:00:38 +0200 Subject: [PATCH 03/28] Update __main__.py --- tests/__main__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index 0e2543af..760178a4 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -8,14 +8,17 @@ ["test_qr", "test_kmeans", "test_knn"], ["test_gridsearch", "test_tsqr", "test_linear_regression"], ["test_dbscan", "test_matmul", "test_als"], - ["test_rf_classifier", "test_randomizedsearch", "test_data_utils", "test_kfold"], + ["test_rf_classifier", "test_randomizedsearch", + "test_data_utils", "test_kfold"], ["test_csvm", "test_rf_regressor", "test_utils", "test_rf_dataset"]] + def run_tests(tests_to_run): test_suite = unittest.TestSuite() test_suite.addTests(tests_to_run) unittest.TextTestRunner(verbosity=2).run(test_suite) + if __name__ == '__main__': suite = list(unittest.loader.defaultTestLoader.discover('./tests/')) @@ -27,6 +30,6 @@ def run_tests(tests_to_run): if t.lower() in str(test_case).lower(): ttt.append(test_case) tests_to_run.append(ttt) - + with ThreadPoolExecutor(max_workers=mp.cpu_count) as exec: exec.map(run_tests, tests_to_run) From a84ff79c855cce4286e394c8c043f4a390c15ece Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 11 Aug 2022 11:07:29 +0200 Subject: [PATCH 04/28] Update __main__.py --- tests/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index 760178a4..9272b569 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,5 +1,4 @@ import unittest -import multiprocessing as mp from concurrent.futures import ThreadPoolExecutor T = [["test_lasso"], @@ -31,5 +30,5 @@ def run_tests(tests_to_run): ttt.append(test_case) tests_to_run.append(ttt) - with ThreadPoolExecutor(max_workers=mp.cpu_count) as exec: + with ThreadPoolExecutor(max_workers=16) as exec: exec.map(run_tests, tests_to_run) From 0c1b95436fc5d73f1865feef8b3d3ffd838e9fcd Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 11 Aug 2022 11:49:53 +0200 Subject: [PATCH 05/28] Update run_tests.sh --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index e68f5ef0..4f820d59 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # Default process per worker -export ComputingUnits=4 +export ComputingUnits=1 # Run the tests/__main__.py file which calls all the tests named test_*.py runcompss \ From aef36bdf50608dddf7126f5f46e3a086300b4350 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 11 Aug 2022 12:22:09 +0200 Subject: [PATCH 06/28] Update Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index b7ebae39..571b4b1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ RUN python3 -m pip install --upgrade -r /dislib/requirements.txt ENV COMPSS_LOAD_SOURCE false +RUN sed -i 's/>416 Date: Thu, 11 Aug 2022 14:12:55 +0200 Subject: [PATCH 07/28] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 571b4b1f..5eba5859 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN python3 -m pip install --upgrade -r /dislib/requirements.txt ENV COMPSS_LOAD_SOURCE false RUN sed -i 's/>4164400045000 Date: Thu, 11 Aug 2022 14:14:19 +0200 Subject: [PATCH 08/28] Update __main__.py --- tests/__main__.py | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index 9272b569..0430f091 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,34 +1,24 @@ import unittest -from concurrent.futures import ThreadPoolExecutor - -T = [["test_lasso"], - ["test_array", "test_pca", "test_daura"], - ["test_gm", "test_preproc", "test_decision_tree"], - ["test_qr", "test_kmeans", "test_knn"], - ["test_gridsearch", "test_tsqr", "test_linear_regression"], - ["test_dbscan", "test_matmul", "test_als"], - ["test_rf_classifier", "test_randomizedsearch", - "test_data_utils", "test_kfold"], - ["test_csvm", "test_rf_regressor", "test_utils", "test_rf_dataset"]] - - -def run_tests(tests_to_run): - test_suite = unittest.TestSuite() - test_suite.addTests(tests_to_run) - unittest.TextTestRunner(verbosity=2).run(test_suite) +import argparse if __name__ == '__main__': suite = list(unittest.loader.defaultTestLoader.discover('./tests/')) + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('tests', metavar='T', type=str, nargs='+', + help='an integer for the accumulator') + args = parser.parse_args() tests_to_run = [] - for tests in T: - ttt = [] - for t in tests: - for test_case in suite: - if t.lower() in str(test_case).lower(): - ttt.append(test_case) - tests_to_run.append(ttt) + for t in args.tests: + for test_case in suite: + if t.lower() in str(test_case).lower(): + tests_to_run.append(test_case) + + print('********** TESTS TO RUN *************') + print(tests_to_run) + print('*************************************') - with ThreadPoolExecutor(max_workers=16) as exec: - exec.map(run_tests, tests_to_run) + test_suite = unittest.TestSuite() + test_suite.addTests(tests_to_run) + unittest.TextTestRunner(verbosity=2).run(test_suite) From cc4b5547e5757ff5c24af789317e8750d316767e Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 11 Aug 2022 14:15:29 +0200 Subject: [PATCH 09/28] Update run_tests.sh --- run_tests.sh | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 4f820d59..4fe507d8 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -3,11 +3,29 @@ # Default process per worker export ComputingUnits=1 -# Run the tests/__main__.py file which calls all the tests named test_*.py -runcompss \ - --pythonpath=$(pwd) \ - --python_interpreter=python3 \ - ./tests/__main__.py &> >(tee output.log) +declare -a tests_group=("test_lasso" + "test_array test_pca test_daura" + "test_gm test_preproc test_decision_tree" + "test_qr test_kmeans test_knn" + "test_gridsearch test_tsqr test_linear_regression" + "test_dbscan test_matmul test_als" + "test_rf_classifier test_randomizedsearch test_data_utils test_kfold" + "test_csvm test_rf_regressor test_utils test_rf_dataset" + ) + +port=43001 + +for t in "${tests_group[@]}" +do + port=$((port + 1)) + runcompss \ + --pythonpath=$(pwd) \ + --python_interpreter=python3 \ + --master_port=$port \ + ./tests/__main__.py $t &> >(tee output.log) & + + sleep 10 +done # Check the unittest output because PyCOMPSs exits with code 0 even if there # are failed tests (the execution itself is successful) From 3dca24cf939d0d61d3653df3f5dc8051638b01ea Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 11 Aug 2022 14:22:04 +0200 Subject: [PATCH 10/28] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5eba5859..0ac205a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN python3 -m pip install --upgrade -r /dislib/requirements.txt ENV COMPSS_LOAD_SOURCE false RUN sed -i 's/>41644000450004300245000 Date: Thu, 11 Aug 2022 14:34:19 +0200 Subject: [PATCH 11/28] Update Jenkinsfile --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7b7230a9..ada53916 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,7 +53,6 @@ pipeline { sh 'docker exec dislib /dislib/bin/print_tests_logs.sh' sh 'docker images' sh 'docker rm -f dislib &> /dev/null || true' - sh 'docker rmi -f bscwdc/dislib &> /dev/null || true' } success { setGithubCommitStatus('success', 'Build Successful') From e0aed0d601a4a0a859c70f9e4b9f282b96e2c4f1 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 11 Aug 2022 16:23:23 +0200 Subject: [PATCH 12/28] Update run_tests.sh --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 4fe507d8..41c61459 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -13,7 +13,7 @@ declare -a tests_group=("test_lasso" "test_csvm test_rf_regressor test_utils test_rf_dataset" ) -port=43001 +port=44001 for t in "${tests_group[@]}" do From 45f358e1e101e6694a0f7369e498725c360cf108 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 14:15:54 +0200 Subject: [PATCH 13/28] modify resources.xml Ports --- Dockerfile | 3 +-- run_tests.sh | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0ac205a7..85175904 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,7 @@ RUN python3 -m pip install --upgrade -r /dislib/requirements.txt ENV COMPSS_LOAD_SOURCE false -RUN sed -i 's/>41643002450004$(grep -c ^processor /proc/cpuinfo)43001<\/MinPort>/$port<\/MinPort>/g" /opt/COMPSs/Runtime/configuration/xml/resources/default_resources.xml > /tmp/resources-$port.xml + sed -i "s/43002<\/MaxPort>/$nextport<\/MaxPort>/g" /tmp/resources-$port.xml + runcompss \ --pythonpath=$(pwd) \ --python_interpreter=python3 \ + --resources=/tmp/resources-$port.xml \ --master_port=$port \ ./tests/__main__.py $t &> >(tee output.log) & + port=$((port + 2)) sleep 10 done @@ -33,6 +40,7 @@ result=$(cat output.log | egrep "OK|FAILED") echo "Tests result: ${result}" + # If word Failed is in the results, exit 1 so the pull request fails if [[ $result =~ FAILED ]]; then exit 1 From 745b095915fabfdcd12e1660c28c01a4b9263354 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 14:29:51 +0200 Subject: [PATCH 14/28] wait runcompss --- run_tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run_tests.sh b/run_tests.sh index b0240f4a..4dbd8a44 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -13,6 +13,8 @@ declare -a tests_group=("test_lasso" "test_csvm test_rf_regressor test_utils test_rf_dataset" ) +declare -a pids + port=43000 for t in "${tests_group[@]}" @@ -30,10 +32,14 @@ do --master_port=$port \ ./tests/__main__.py $t &> >(tee output.log) & + pids+=($!) + port=$((port + 2)) sleep 10 done +wait ${pids[@]} + # Check the unittest output because PyCOMPSs exits with code 0 even if there # are failed tests (the execution itself is successful) result=$(cat output.log | egrep "OK|FAILED") From 9e9f5e6868430d707374a8b3e2e3bdce1cc7eb77 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 16:14:53 +0200 Subject: [PATCH 15/28] debug --- run_tests.sh | 6 +++++- tests/__main__.py | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 4dbd8a44..e87c86cf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -16,6 +16,7 @@ declare -a tests_group=("test_lasso" declare -a pids port=43000 +workerid=0 for t in "${tests_group[@]}" do @@ -26,14 +27,17 @@ do sed -i "s/43002<\/MaxPort>/$nextport<\/MaxPort>/g" /tmp/resources-$port.xml runcompss \ + --log_level=debug \ --pythonpath=$(pwd) \ --python_interpreter=python3 \ --resources=/tmp/resources-$port.xml \ --master_port=$port \ - ./tests/__main__.py $t &> >(tee output.log) & + ./tests/__main__.py $t -id $workerid &> >(tee output.log) & pids+=($!) + workerid=$((workerid + 1)) + port=$((port + 2)) sleep 10 done diff --git a/tests/__main__.py b/tests/__main__.py index 0430f091..4e6902da 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,12 +1,13 @@ import unittest import argparse - +import datetime if __name__ == '__main__': suite = list(unittest.loader.defaultTestLoader.discover('./tests/')) parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument('tests', metavar='T', type=str, nargs='+', help='an integer for the accumulator') + parser.add_argument('-id', type=int) args = parser.parse_args() tests_to_run = [] @@ -15,10 +16,10 @@ if t.lower() in str(test_case).lower(): tests_to_run.append(test_case) - print('********** TESTS TO RUN *************') - print(tests_to_run) - print('*************************************') + print(f'WORKER {args.id} START TEST AT', datetime.datetime.now()) test_suite = unittest.TestSuite() test_suite.addTests(tests_to_run) unittest.TextTestRunner(verbosity=2).run(test_suite) + + print(f'WORKER {args.id} END TEST AT', datetime.datetime.now()) \ No newline at end of file From 505322258f0e9cebf5cda9281f96b6762e3d94ec Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 16:22:47 +0200 Subject: [PATCH 16/28] style --- tests/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__main__.py b/tests/__main__.py index 4e6902da..1225b03c 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -22,4 +22,4 @@ test_suite.addTests(tests_to_run) unittest.TextTestRunner(verbosity=2).run(test_suite) - print(f'WORKER {args.id} END TEST AT', datetime.datetime.now()) \ No newline at end of file + print(f'WORKER {args.id} END TEST AT', datetime.datetime.now()) From f39671590543a6c792d8efa61d7fecf10ffe2238 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 17:02:16 +0200 Subject: [PATCH 17/28] Update run_tests.sh --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index e87c86cf..71ab03ac 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # Default process per worker -export ComputingUnits=1 +export ComputingUnits=8 declare -a tests_group=("test_lasso" "test_array test_pca test_daura" From 95a855540b83191aa2ebe90b3165cf101da89317 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 17:57:07 +0200 Subject: [PATCH 18/28] 2 CU --- Dockerfile | 2 +- run_tests.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 85175904..9d5e11f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN python3 -m pip install --upgrade -r /dislib/requirements.txt ENV COMPSS_LOAD_SOURCE false -RUN sed -i "s/>4$(grep -c ^processor /proc/cpuinfo)4243002<\/MaxPort>/$nextport<\/MaxPort>/g" /tmp/resources-$port.xml runcompss \ - --log_level=debug \ --pythonpath=$(pwd) \ --python_interpreter=python3 \ --resources=/tmp/resources-$port.xml \ From c9d3a6c00bff7babc55f8bd1594bcdb34706fcb6 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 18:15:24 +0200 Subject: [PATCH 19/28] Update run_tests.sh --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 5e508804..1ffca3ed 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash -e # Default process per worker -export ComputingUnits=8 +export ComputingUnits=1 declare -a tests_group=("test_lasso" "test_array test_pca test_daura" From 1077575fdc1f0a7e862aef10804e9b5eeddd66f5 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 20:43:47 +0200 Subject: [PATCH 20/28] dd --- Dockerfile | 2 +- run_tests.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9d5e11f0..5463c559 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN python3 -m pip install --upgrade -r /dislib/requirements.txt ENV COMPSS_LOAD_SOURCE false -RUN sed -i "s/>424543002<\/MaxPort>/$nextport<\/MaxPort>/g" /tmp/resources-$port.xml runcompss \ + --log_level=debug \ --pythonpath=$(pwd) \ --python_interpreter=python3 \ --resources=/tmp/resources-$port.xml \ From 9bf5207f5a66508c06c9008f8352657e03a01b96 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 21:41:20 +0200 Subject: [PATCH 21/28] a --- Dockerfile | 2 +- run_tests.sh | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5463c559..0bf23792 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN python3 -m pip install --upgrade -r /dislib/requirements.txt ENV COMPSS_LOAD_SOURCE false -RUN sed -i "s/>4545 Date: Wed, 17 Aug 2022 23:23:50 +0200 Subject: [PATCH 22/28] aa --- run_tests.sh | 12 +++++++----- tests/__init__.py | 11 +++++++---- tests/__main__.py | 4 ++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index d6a3cbab..531c6c84 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,11 +4,13 @@ export ComputingUnits=2 declare -a tests_group=("test_lasso" - "test_array test_pca test_daura test_rf_classifier" - "test_gm test_preproc test_decision_tree test_csvm test_randomizedsearch" - "test_qr test_kmeans test_knn test_rf_regressor test_data_utils" - "test_gridsearch test_tsqr test_linear_regression test_utils" - "test_dbscan test_matmul test_als test_rf_dataset test_kfold" + "test_array test_pca test_daura" + "test_gm test_preproc test_decision_tree" + "test_qr test_kmeans test_knn" + "test_gridsearch test_tsqr test_linear_regression" + "test_dbscan test_matmul test_als" + "test_rf_classifier test_randomizedsearch test_data_utils test_kfold" + "test_csvm test_rf_regressor test_utils test_rf_dataset" ) declare -a pids diff --git a/tests/__init__.py b/tests/__init__.py index 62bea21c..e2b3dd3d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,12 +1,15 @@ +import datetime from time import time import unittest class BaseTimedTestCase(unittest.TestCase): def setUp(self): - self.start_time = time() + # self.start_time = time() + print(f'Test {self.id()} START AT', datetime.datetime.now(), flush=True) def tearDown(self): - self.end_time = time() - print("Test %s took: %.3f seconds" % - (self.id(), self.end_time - self.start_time)) + print(f'Test {self.id()} END AT', datetime.datetime.now(), flush=True) + # self.end_time = time() + # print("Test %s took: %.3f seconds" % + # (self.id(), self.end_time - self.start_time)) diff --git a/tests/__main__.py b/tests/__main__.py index 1225b03c..da1d7621 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -16,10 +16,10 @@ if t.lower() in str(test_case).lower(): tests_to_run.append(test_case) - print(f'WORKER {args.id} START TEST AT', datetime.datetime.now()) + print(f'WORKER {args.id} START TEST AT', datetime.datetime.now(), flush=True) test_suite = unittest.TestSuite() test_suite.addTests(tests_to_run) unittest.TextTestRunner(verbosity=2).run(test_suite) - print(f'WORKER {args.id} END TEST AT', datetime.datetime.now()) + print(f'WORKER {args.id} END TEST AT', datetime.datetime.now(), flush=True) From 02e820483289f7ad5870c75b0bcc442f7e04b7ca Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 23:33:49 +0200 Subject: [PATCH 23/28] ss --- tests/__init__.py | 7 ++++--- tests/__main__.py | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index e2b3dd3d..24ed8a2c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,15 +1,16 @@ import datetime -from time import time import unittest class BaseTimedTestCase(unittest.TestCase): def setUp(self): # self.start_time = time() - print(f'Test {self.id()} START AT', datetime.datetime.now(), flush=True) + print(f'Test {self.id()} START AT', + datetime.datetime.now(), flush=True) def tearDown(self): - print(f'Test {self.id()} END AT', datetime.datetime.now(), flush=True) + print(f'Test {self.id()} END AT', + datetime.datetime.now(), flush=True) # self.end_time = time() # print("Test %s took: %.3f seconds" % # (self.id(), self.end_time - self.start_time)) diff --git a/tests/__main__.py b/tests/__main__.py index da1d7621..45092f39 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -16,10 +16,12 @@ if t.lower() in str(test_case).lower(): tests_to_run.append(test_case) - print(f'WORKER {args.id} START TEST AT', datetime.datetime.now(), flush=True) + print(f'WORKER {args.id} START TEST AT', + datetime.datetime.now(), flush=True) test_suite = unittest.TestSuite() test_suite.addTests(tests_to_run) unittest.TextTestRunner(verbosity=2).run(test_suite) - print(f'WORKER {args.id} END TEST AT', datetime.datetime.now(), flush=True) + print(f'WORKER {args.id} END TEST AT', + datetime.datetime.now(), flush=True) From 8883bc76180480bec5d1b91bcd300f1b498c21f5 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 23:49:22 +0200 Subject: [PATCH 24/28] q --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 531c6c84..249efe37 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -16,7 +16,7 @@ declare -a tests_group=("test_lasso" declare -a pids port=43000 -workerid=0 +workerid=1 for t in "${tests_group[@]}" do From 4fe41b0431526a1822664ea17a71b42f519428af Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Wed, 17 Aug 2022 23:49:36 +0200 Subject: [PATCH 25/28] q --- Dockerfile | 2 ++ tests/__main__.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0bf23792..dda3e3a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,8 @@ ENV PYTHONPATH=$PYTHONPATH:/dislib ENV LC_ALL=C.UTF-8 RUN python3 -m pip install --upgrade -r /dislib/requirements.txt +RUN apt-get update && apt-get install -y htop + ENV COMPSS_LOAD_SOURCE false # RUN sed -i "s/>45 Date: Thu, 18 Aug 2022 00:00:09 +0200 Subject: [PATCH 26/28] dd --- run_tests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 249efe37..b5194f17 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -27,7 +27,6 @@ do sed -i "s/43002<\/MaxPort>/$nextport<\/MaxPort>/g" /tmp/resources-$port.xml runcompss \ - --log_level=debug \ --pythonpath=$(pwd) \ --python_interpreter=python3 \ --resources=/tmp/resources-$port.xml \ From f11e504ca5fb3430d5d54e184b841bd09ed12b56 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 18 Aug 2022 00:08:38 +0200 Subject: [PATCH 27/28] h --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dda3e3a4..37caac52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ ENV PYTHONPATH=$PYTHONPATH:/dislib ENV LC_ALL=C.UTF-8 RUN python3 -m pip install --upgrade -r /dislib/requirements.txt -RUN apt-get update && apt-get install -y htop +# RUN apt-get update && apt-get install -y htop ENV COMPSS_LOAD_SOURCE false From 33d7ee0d8f229f9cf44009244f80d086ef5ac406 Mon Sep 17 00:00:00 2001 From: Cristian Tatu Date: Thu, 18 Aug 2022 09:45:18 +0200 Subject: [PATCH 28/28] Update run_tests.sh --- run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run_tests.sh b/run_tests.sh index b5194f17..249efe37 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -27,6 +27,7 @@ do sed -i "s/43002<\/MaxPort>/$nextport<\/MaxPort>/g" /tmp/resources-$port.xml runcompss \ + --log_level=debug \ --pythonpath=$(pwd) \ --python_interpreter=python3 \ --resources=/tmp/resources-$port.xml \