From ecb8a41881aa9502c7bd28d2f765803de21f2b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 28 Aug 2023 15:09:07 +0200 Subject: [PATCH 01/97] First version of the ErrorListener --- .gitignore | 2 +- libraries/ErrorListener.py | 53 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 8 +++--- 3 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 libraries/ErrorListener.py diff --git a/.gitignore b/.gitignore index 5393a5e1..d4b2af99 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ resources/__pycache__ results *.pyc *.http -.vscode/ \ No newline at end of file +.vscode/ diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py new file mode 100644 index 00000000..6cd09f11 --- /dev/null +++ b/libraries/ErrorListener.py @@ -0,0 +1,53 @@ +from os.path import join +from os import getcwd +from re import compile, match, MULTILINE + + +class ErrorListener: + ROBOT_LISTENER_API_VERSION = 2 + + def __init__(self, filename='errors.log'): + self.cwd = getcwd() + out_path = join('results', filename) + self.max_length = 150 + self.outfile = open(out_path, 'w') + self.tests = str() + self.suite_name = str() + self.rx_dict = { + 'variables': compile('^\${.*$|^\&{.*$|^\@{.*'), + 'http_verbs': compile('^GET.*(Request|Response).*$|' + '^HEAD.*(Request|Response).*$|' + '^POST.*(Request|Response).*$|' + '^PUT.*(Request|Response).*$|' + '^DELETE.*(Request|Response).*$|' + '^CONNECT.*(Request|Response).*$|' + '^OPTIONS.*(Request|Response).*$|' + '^TRACE.*(Request|Response).*$|' + '^PATCH.*(Request|Response).*$', MULTILINE) + } + + def start_suite(self, name, attrs): + self.suite_name = attrs['source'].replace(self.cwd, '')[1:].replace('.robot', '').replace('/', ".") + self.outfile.write(f'{"=" * self.max_length}\n') + self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}\n') + self.outfile.write(f'{"=" * self.max_length}\n') + + def start_test(self, name, attrs): + self.tests = f"{name} :: {attrs['doc']}\n" + + def end_test(self, name, attrs): + if attrs['status'] != 'PASS': + self.outfile.write(self.tests) + self.outfile.write(f'| FAIL |\n{attrs["message"]}\n') + self.outfile.write(f'{"-" * self.max_length}\n') + + def end_suite(self, name, attrs): + self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}... | {attrs["status"]} |\n{attrs["statistics"]}\n') + + def log_message(self, msg): + if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and + not match(pattern=self.rx_dict['http_verbs'], string=msg['message'])): + self.outfile.write(f'{msg["message"]}\n') + + def close(self): + self.outfile.close() diff --git a/requirements.txt b/requirements.txt index 03455a4f..686ff92b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # python3.10 project -robotframework==6.0.2 +robotframework==6.1.1 robotframework-jsonlibrary==0.5 -robotframework-requests==0.9.4 -deepdiff==6.3.0 +robotframework-requests==0.9.5 +deepdiff==6.3.1 prettydiff==0.1.0 robotframework-httpctrl==0.3.1 -robotframework-tidy==4.2.1 +robotframework-tidy==4.5.0 -- GitLab From 432f92f97b88f41788c0ab1ab690abc1930bfd9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 28 Aug 2023 16:53:56 +0200 Subject: [PATCH 02/97] Update the content and adding PyCharm Execution Configurations --- .../Run_All_tests_with_Listener.xml | 19 +++++++++++++++++++ .../Run_Current_Test_Suite.xml | 17 +++++++++++++++++ .../Run_Current_Test_Suite_with_Listener.xml | 19 +++++++++++++++++++ resources/variables.py | 2 +- scripts/run_tests.sh | 14 ++++++++++---- 5 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 .idea/runConfigurations/Run_All_tests_with_Listener.xml create mode 100644 .idea/runConfigurations/Run_Current_Test_Suite.xml create mode 100644 .idea/runConfigurations/Run_Current_Test_Suite_with_Listener.xml diff --git a/.idea/runConfigurations/Run_All_tests_with_Listener.xml b/.idea/runConfigurations/Run_All_tests_with_Listener.xml new file mode 100644 index 00000000..dbf173a6 --- /dev/null +++ b/.idea/runConfigurations/Run_All_tests_with_Listener.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Run_Current_Test_Suite.xml b/.idea/runConfigurations/Run_Current_Test_Suite.xml new file mode 100644 index 00000000..43a559bb --- /dev/null +++ b/.idea/runConfigurations/Run_Current_Test_Suite.xml @@ -0,0 +1,17 @@ + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Run_Current_Test_Suite_with_Listener.xml b/.idea/runConfigurations/Run_Current_Test_Suite_with_Listener.xml new file mode 100644 index 00000000..538e85be --- /dev/null +++ b/.idea/runConfigurations/Run_Current_Test_Suite_with_Listener.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/resources/variables.py b/resources/variables.py index f0c933b8..017b8eb2 100644 --- a/resources/variables.py +++ b/resources/variables.py @@ -1,4 +1,4 @@ -url = 'localhost:8080/ngsi-ld/v1' +url = 'http://localhost:8080/ngsi-ld/v1' ngsild_test_suite_context = 'https://forge.etsi.org/rep/cim/ngsi-ld-test-suite/-/raw/develop/resources/jsonld-contexts/ngsi-ld-test-suite-compound.jsonld' notification_server_host = '0.0.0.0' notification_server_port = 8085 diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 6216528f..f5f3432a 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -9,19 +9,25 @@ robot --variable url:"URL_HERE" --outputdir ./results . # run by specific tag(s) robot --include mandatory --outputdir ./results . -# run all the tests for context information +# run all the test suites for context information robot --outputdir ./results ./TP/NGSI-LD/ContextInformation -# run a specific test case +# run a specific test suite robot --outputdir ./results ./TP/NGSI-LD/ContextInformation/Provision/Entities/CreateEntity/001_01.robot robot --outputdir ./results --suite 001_01 . -# run specific test case +# run specific test suite robot --outputdir ./results -t "SuccessCases_MinimalEntity" robot --outputdir ./results -t "SuccessCases_MinimalEntity" ./TP/NGSI-LD/ContextInformation/Provision/Entities/CreateEntity/SuccessCases.robot -# rerun failed tests +# rerun failed test cases robot --rerunfailedsuites ./results/output.xml --outputdir ./results . # stop the suite after a failed test robot --exitonfailure --outputdir ./results . + +# run specific test suite and generate errors.log file with the output only of the failed test cases +robot --listener libraries/ErrorListener.py --outputdir ./results ./TP/NGSI-LD/CommonBehaviours/043.robot + +# run all test suites and generate errors.log file with the output only of the failed test cases +robot --listener libraries/ErrorListener.py --outputdir ./results . -- GitLab From 036be6c0150b143bc87971111de5c3489c58bb31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 28 Aug 2023 16:55:44 +0200 Subject: [PATCH 03/97] Rename Pycharm Execution Configuration --- ..._with_Listener.xml => Run_All_Test_Suites_with_Listener.xml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .idea/runConfigurations/{Run_All_tests_with_Listener.xml => Run_All_Test_Suites_with_Listener.xml} (81%) diff --git a/.idea/runConfigurations/Run_All_tests_with_Listener.xml b/.idea/runConfigurations/Run_All_Test_Suites_with_Listener.xml similarity index 81% rename from .idea/runConfigurations/Run_All_tests_with_Listener.xml rename to .idea/runConfigurations/Run_All_Test_Suites_with_Listener.xml index dbf173a6..b718676e 100644 --- a/.idea/runConfigurations/Run_All_tests_with_Listener.xml +++ b/.idea/runConfigurations/Run_All_Test_Suites_with_Listener.xml @@ -1,5 +1,5 @@ - +