From a979f76271e72ffa62d3ca66b3b69a3acfbfda61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 31 Aug 2023 20:19:17 +0200 Subject: [PATCH 01/25] Changing output format of the execution of the tests --- libraries/ErrorListener.py | 112 +++++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 6cd09f11..73fbf576 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -1,6 +1,8 @@ from os.path import join from os import getcwd from re import compile, match, MULTILINE +import json +from http import HTTPStatus class ErrorListener: @@ -9,9 +11,10 @@ class ErrorListener: def __init__(self, filename='errors.log'): self.cwd = getcwd() out_path = join('results', filename) - self.max_length = 150 + self.max_length_suite = 150 + self.max_length_case = 80 self.outfile = open(out_path, 'w') - self.tests = str() + self.tests = list() self.suite_name = str() self.rx_dict = { 'variables': compile('^\${.*$|^\&{.*$|^\@{.*'), @@ -28,26 +31,115 @@ class ErrorListener: def start_suite(self, name, attrs): self.suite_name = attrs['source'].replace(self.cwd, '')[1:].replace('.robot', '').replace('/', ".") - self.outfile.write(f'{"=" * self.max_length}\n') + self.outfile.write(f'{"=" * self.max_length_suite}\n') self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}\n') - self.outfile.write(f'{"=" * self.max_length}\n') + self.outfile.write(f'{"=" * self.max_length_suite}\n') def start_test(self, name, attrs): - self.tests = f"{name} :: {attrs['doc']}\n" + self.tests.append(f"\n\n{name}\n") + self.tests.append(f'{"=" * self.max_length_case}\n') def end_test(self, name, attrs): if attrs['status'] != 'PASS': - self.outfile.write(self.tests) + flat_list = self.__flatten_concatenation__(matrix=self.tests) + [self.outfile.write(x) for x in flat_list] self.outfile.write(f'| FAIL |\n{attrs["message"]}\n') - self.outfile.write(f'{"-" * self.max_length}\n') + self.outfile.write(f'{"-" * self.max_length_case}\n') + self.tests.clear() def end_suite(self, name, attrs): - self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}... | {attrs["status"]} |\n{attrs["statistics"]}\n') - + # self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}... | {attrs["status"]} |\n{attrs["statistics"]}\n') + pass + def log_message(self, msg): if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and not match(pattern=self.rx_dict['http_verbs'], string=msg['message'])): - self.outfile.write(f'{msg["message"]}\n') + self.tests.append(self.__get_message__(msg["message"])) def close(self): self.outfile.close() + + def __get_message__(self, message: str) -> str: + result = str() + if message == 'Request ->': + result = f'\n\nRequest:\n{"-" * self.max_length_case}\n' + elif message == 'Response ->': + result = f'\n\nResponse:\n{"-" * self.max_length_case}\n' + elif self.__is_string_dict__(string=message): + result = self.__generate_prety_output__(data=message) + else: + result = message + + return result + + def __generate_prety_output__(self, data: str) -> list: + data = json.loads(data) + + output = list() + + try: + output.append(f' {data["method"]} {data["url"]}\n') + output.append(f' User-Agent: {data["headers"]["User-Agent"]}\n') + output.append(f' Accept-Encoding: {data["headers"]["Accept-Encoding"]}\n') + output.append(f' Accept: {data["headers"]["Accept"]}\n') + output.append(f' Connection: {data["headers"]["Connection"]}\n') + + try: + output.append(f' Content-Type: {data["headers"]["Content-Type"]}\n') + except KeyError: + pass + + output.append(f' Content-Length: {data["headers"]["Content-Length"]}\n\n') + + if data['body'] is None: + output.append(f' No body\n') + else: + aux = json.dumps(data['body'], indent=2) + aux = aux.replace('\n', '\n ').replace("{", " {") + output.append(aux) + except KeyError as e: + # This is a Response dictionary and not a Request dictionary + # robotframework-requests is based on python request, so it is using HTTP/1.1 + output.append(f' HTTP/1.1 {data["status_code"]} {self.__get_status_meaning__(data["status_code"])}\n') + + try: + output.append(f' Content-Type: {data["headers"]["Content-Type"]}\n') + except KeyError: + pass + + output.append(f' Date: REGEX(. *)\n\n') + + if data['body'] is None: + output.append(f' No body\n') + else: + aux = json.dumps(data['body'], indent=2) + aux = aux.replace('\n', '\n ').replace("{", " {") + output.append(aux) + + return output + + def __get_status_meaning__(self, status_code): + try: + status = HTTPStatus(status_code) + return status.phrase + except ValueError: + return "Unknown status code" + + def __is_string_dict__(self, string: str) -> bool: + try: + json_object = json.loads(string) + if isinstance(json_object, dict): + return True + except ValueError: + pass + return False + + def __flatten_concatenation__(self, matrix): + flat_list = [] + for row in matrix: + if isinstance(row, str): + flat_list.append(row) + else: + flat_list += row + + return flat_list -- GitLab From d1228008be228cecc2b9fda8a6b8ab5257176ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 31 Aug 2023 20:43:29 +0200 Subject: [PATCH 02/25] Resolution some blank lines and indentation of bodies --- libraries/ErrorListener.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 73fbf576..377ce481 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -1,7 +1,7 @@ from os.path import join from os import getcwd from re import compile, match, MULTILINE -import json +from json import loads, dumps from http import HTTPStatus @@ -43,14 +43,14 @@ class ErrorListener: if attrs['status'] != 'PASS': flat_list = self.__flatten_concatenation__(matrix=self.tests) [self.outfile.write(x) for x in flat_list] - self.outfile.write(f'| FAIL |\n{attrs["message"]}\n') - self.outfile.write(f'{"-" * self.max_length_case}\n') + #self.outfile.write(f'| FAIL |\n{attrs["message"]}\n') + #self.outfile.write(f'{"-" * self.max_length_case}\n') self.tests.clear() def end_suite(self, name, attrs): # self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}... | {attrs["status"]} |\n{attrs["statistics"]}\n') pass - + def log_message(self, msg): if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and not match(pattern=self.rx_dict['http_verbs'], string=msg['message'])): @@ -66,14 +66,18 @@ class ErrorListener: elif message == 'Response ->': result = f'\n\nResponse:\n{"-" * self.max_length_case}\n' elif self.__is_string_dict__(string=message): - result = self.__generate_prety_output__(data=message) - else: + result = self.__generate_pretty_output__(data=message) + elif message[0] == '\n': + # This is the title of a test case operation + #result = f'\nMismatch:\n{"-" * self.max_length_case}\nNo\n\n{message}\n' result = message + else: + result = f'\nMismatch:\n{"-" * self.max_length_case}\n{message}\n' return result - def __generate_prety_output__(self, data: str) -> list: - data = json.loads(data) + def __generate_pretty_output__(self, data: str) -> list: + data = loads(data) output = list() @@ -94,8 +98,11 @@ class ErrorListener: if data['body'] is None: output.append(f' No body\n') else: - aux = json.dumps(data['body'], indent=2) - aux = aux.replace('\n', '\n ').replace("{", " {") + aux = dumps(data['body'], indent=2) + aux = (aux.replace('\n', '\n ') + .replace("{", " {") + .replace("[", " [") + '\n') + output.append(aux) except KeyError as e: # This is a Response dictionary and not a Request dictionary @@ -112,8 +119,10 @@ class ErrorListener: if data['body'] is None: output.append(f' No body\n') else: - aux = json.dumps(data['body'], indent=2) - aux = aux.replace('\n', '\n ').replace("{", " {") + aux = dumps(data['body'], indent=2) + aux = (aux.replace('\n', '\n ') + .replace("{", " {") + .replace("[", " [") + '\n') output.append(aux) return output @@ -127,7 +136,7 @@ class ErrorListener: def __is_string_dict__(self, string: str) -> bool: try: - json_object = json.loads(string) + json_object = loads(string) if isinstance(json_object, dict): return True except ValueError: -- GitLab From 16c3eaddbff27dea97190ebff27e604763abfd19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 1 Sep 2023 10:07:06 +0200 Subject: [PATCH 03/25] Adding management of mismatches in dictionaries and management of errors --- libraries/ErrorListener.py | 155 +++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 67 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 377ce481..a73f9a3d 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -1,3 +1,4 @@ +import re from os.path import join from os import getcwd from re import compile, match, MULTILINE @@ -5,6 +6,57 @@ from json import loads, dumps from http import HTTPStatus +def __get_header__(dictionary: dict, key: str) -> str: + result = str() + try: + result = f' {key}: {dictionary["headers"][key]}\n' + return result + except KeyError: + pass + + +def __get_status_meaning__(status_code): + try: + status = HTTPStatus(status_code) + return status.phrase + except ValueError: + return "Unknown status code" + + +def __is_string_dict__(string: str) -> bool: + try: + json_object = loads(string) + if isinstance(json_object, dict): + return True + except ValueError: + pass + return False + + +def __flatten_concatenation__(matrix): + flat_list = [] + for row in matrix: + if isinstance(row, str): + flat_list.append(row) + else: + flat_list += row + + return flat_list + + +def __get_body__(dictionary: dict): + result = str() + if dictionary is None: + result = ' No body\n' + else: + result = dumps(dictionary, indent=2) + result = (result.replace('\n', '\n ') + .replace("{", " {") + .replace("[", " [") + '\n') + + return result + + class ErrorListener: ROBOT_LISTENER_API_VERSION = 2 @@ -26,14 +78,17 @@ class ErrorListener: '^CONNECT.*(Request|Response).*$|' '^OPTIONS.*(Request|Response).*$|' '^TRACE.*(Request|Response).*$|' - '^PATCH.*(Request|Response).*$', MULTILINE) + '^PATCH.*(Request|Response).*$', MULTILINE), + 'length_log': compile('^Length is \d+$') } def start_suite(self, name, attrs): self.suite_name = attrs['source'].replace(self.cwd, '')[1:].replace('.robot', '').replace('/', ".") - self.outfile.write(f'{"=" * self.max_length_suite}\n') - self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}\n') - self.outfile.write(f'{"=" * self.max_length_suite}\n') + + if attrs['doc'] != '': + self.outfile.write(f'{"=" * self.max_length_suite}\n') + self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}\n') + self.outfile.write(f'{"=" * self.max_length_suite}\n') def start_test(self, name, attrs): self.tests.append(f"\n\n{name}\n") @@ -41,15 +96,12 @@ class ErrorListener: def end_test(self, name, attrs): if attrs['status'] != 'PASS': - flat_list = self.__flatten_concatenation__(matrix=self.tests) - [self.outfile.write(x) for x in flat_list] - #self.outfile.write(f'| FAIL |\n{attrs["message"]}\n') - #self.outfile.write(f'{"-" * self.max_length_case}\n') + flat_list = __flatten_concatenation__(matrix=self.tests) + [self.outfile.write(x) for x in flat_list if x is not None] self.tests.clear() def end_suite(self, name, attrs): - # self.outfile.write(f'{self.suite_name} :: {attrs["doc"]}... | {attrs["status"]} |\n{attrs["statistics"]}\n') - pass + self.outfile.write('\n\n\n') def log_message(self, msg): if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and @@ -65,13 +117,14 @@ class ErrorListener: result = f'\n\nRequest:\n{"-" * self.max_length_case}\n' elif message == 'Response ->': result = f'\n\nResponse:\n{"-" * self.max_length_case}\n' - elif self.__is_string_dict__(string=message): + elif __is_string_dict__(string=message): result = self.__generate_pretty_output__(data=message) elif message[0] == '\n': # This is the title of a test case operation - #result = f'\nMismatch:\n{"-" * self.max_length_case}\nNo\n\n{message}\n' result = message - else: + elif message == 'Dictionary comparison failed with -> ': + result == None + elif re.match(pattern=self.rx_dict['length_log'], string=message) is None: result = f'\nMismatch:\n{"-" * self.max_length_case}\n{message}\n' return result @@ -80,6 +133,20 @@ class ErrorListener: data = loads(data) output = list() + my_header_keys = ['User-Agent', + 'Accept-Encoding', + 'Accept', + 'Connection', + 'Link', + 'Content-Type', + 'Content-Length'] + + received_header_keys = data['headers'].keys() + + final_keys = list(set(received_header_keys) - set(my_header_keys)) + + if len(final_keys) != 0: + output.append(f'\n\nERROR: Need to add the folloing header key(s): {final_keys}\n\n') try: output.append(f' {data["method"]} {data["url"]}\n') @@ -88,67 +155,21 @@ class ErrorListener: output.append(f' Accept: {data["headers"]["Accept"]}\n') output.append(f' Connection: {data["headers"]["Connection"]}\n') - try: - output.append(f' Content-Type: {data["headers"]["Content-Type"]}\n') - except KeyError: - pass - - output.append(f' Content-Length: {data["headers"]["Content-Length"]}\n\n') + output.append(__get_header__(dictionary=data, key='Link')) + output.append(__get_header__(dictionary=data, key='Content-Type')) + output.append(__get_header__(dictionary=data, key='Content-Length')) + output.append('\n') - if data['body'] is None: - output.append(f' No body\n') - else: - aux = dumps(data['body'], indent=2) - aux = (aux.replace('\n', '\n ') - .replace("{", " {") - .replace("[", " [") + '\n') - - output.append(aux) + output.append(__get_body__(dictionary=data['body'])) except KeyError as e: # This is a Response dictionary and not a Request dictionary # robotframework-requests is based on python request, so it is using HTTP/1.1 - output.append(f' HTTP/1.1 {data["status_code"]} {self.__get_status_meaning__(data["status_code"])}\n') + output.append(f' HTTP/1.1 {data["status_code"]} {__get_status_meaning__(data["status_code"])}\n') - try: - output.append(f' Content-Type: {data["headers"]["Content-Type"]}\n') - except KeyError: - pass + output.append(__get_header__(dictionary=data, key='Content-Type')) output.append(f' Date: REGEX(. *)\n\n') - if data['body'] is None: - output.append(f' No body\n') - else: - aux = dumps(data['body'], indent=2) - aux = (aux.replace('\n', '\n ') - .replace("{", " {") - .replace("[", " [") + '\n') - output.append(aux) + output.append(__get_body__(dictionary=data['body'])) return output - - def __get_status_meaning__(self, status_code): - try: - status = HTTPStatus(status_code) - return status.phrase - except ValueError: - return "Unknown status code" - - def __is_string_dict__(self, string: str) -> bool: - try: - json_object = loads(string) - if isinstance(json_object, dict): - return True - except ValueError: - pass - return False - - def __flatten_concatenation__(self, matrix): - flat_list = [] - for row in matrix: - if isinstance(row, str): - flat_list.append(row) - else: - flat_list += row - - return flat_list -- GitLab From 9e86e04d157520a1c753e1189831d0dc4b341f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 1 Sep 2023 10:33:01 +0200 Subject: [PATCH 04/25] Update README content --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index a8c5dc5d..e8e04b1e 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,17 @@ You must add a chevron at the end of the test launch command followed by the fil ```$ python3 -m robot.tidy --recursive TP/NGSI-LD``` +## Generate output file details only for failed tests + +It is possible to generate a report only for the failed tests through the use of a specific listener in the execution +of the robot framework. For example, if we want to execute the test suite number 043 and generate the report, we can +execute the following command: + +```robot --listener libraries/ErrorListener.py --outputdir ./results ./TP/NGSI-LD/CommonBehaviours/043.robot``` + +It will generate a specific `errors.log` file into the `results` folder with the description of the different steps +developed and the mismatched observed on them. + # Frameworks and libraries used in the project * [Robot Framework](https://github.com/robotframework/robotframework) -- GitLab From 4202ad469e108ee12b44e8f2e2123074a12af51b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Sun, 3 Sep 2023 14:25:22 +0200 Subject: [PATCH 05/25] Adding new header keys in the management of the responses --- libraries/ErrorListener.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index a73f9a3d..ba163f74 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -139,14 +139,16 @@ class ErrorListener: 'Connection', 'Link', 'Content-Type', - 'Content-Length'] + 'Content-Length', + 'Date', + 'Location'] received_header_keys = data['headers'].keys() final_keys = list(set(received_header_keys) - set(my_header_keys)) if len(final_keys) != 0: - output.append(f'\n\nERROR: Need to add the folloing header key(s): {final_keys}\n\n') + output.append(f'\n\nERROR: Need to add the following header key(s): {final_keys}\n\n') try: output.append(f' {data["method"]} {data["url"]}\n') @@ -168,7 +170,10 @@ class ErrorListener: output.append(__get_header__(dictionary=data, key='Content-Type')) - output.append(f' Date: REGEX(. *)\n\n') + output.append(f' Date: REGEX(. *)\n') + output.append(__get_header__(dictionary=data, key='Location')) + output.append(__get_header__(dictionary=data, key='Content-Length')) + output.append('\n') output.append(__get_body__(dictionary=data['body'])) -- GitLab From 7eeb558b29ae2891a001fb8eb3ec110fcbbbc18c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 4 Sep 2023 11:40:09 +0200 Subject: [PATCH 06/25] Create a class to convert output file generated by the listener into markdown format --- libraries/convertMD.py | 77 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 libraries/convertMD.py diff --git a/libraries/convertMD.py b/libraries/convertMD.py new file mode 100644 index 00000000..8dd2f8bc --- /dev/null +++ b/libraries/convertMD.py @@ -0,0 +1,77 @@ +from re import compile, match, findall, MULTILINE + + +class Markdown: + def __init__(self, filename: str): + # Read the content of the input file + with open(filename, 'r') as file: + self.content = file.read() + + self.data = { + "suite": str(), + "cases": list(), + "steps": list() + } + + self.markdown_content = str() + + def get_names(self): + pattern1 = compile('^(\S+.*)$', MULTILINE) + + aux = findall(pattern=pattern1, string=self.content) + + special_lines = ['Response:', 'Request:', 'Mismatch:', f'{"=" * 150}', f'{"=" * 80}', f'{"-" * 80}'] + xs = [x for x in aux if x not in special_lines] + + prefixes_to_remove = ["Item ", "+ ", "- "] + xs = [item for item in xs if not any(item.startswith(prefix) for prefix in prefixes_to_remove)] + + # Get the name of the Test Suite + self.data["suite"] = xs[0] + + # Get the names of the Test Cases + pattern = r"\d{3}\w+" + self.data["cases"] = [item for item in xs if match(pattern, item)] + + # Get the rest of values -> Steps + # Get items from listA not present in listB and not equal to exclude_string + self.data['steps'] = [item for item in xs if item not in self.data['cases'] and item != self.data['suite']] + self.data['steps'] = list(set(self.data['steps'])) + + def generate_md(self): + # Replace the title of the Test Suite + self.markdown_content = self.content + self.markdown_content = ( + self.markdown_content.replace(f'{"=" * 150}\n{self.data["suite"]}\n{"=" * 150}', f'# {self.data["suite"]}')) + + # Replace the name of the Test Cases + for x in self.data['cases']: + self.markdown_content = ( + self.markdown_content.replace(f'{x}\n{"=" * 80}\n', f'```\n## {x}\n')) + + # Replace Request, Response, and Mismatch + self.markdown_content = (self.markdown_content.replace(f'Request:\n{"-" * 80}', '#### Request:\n```') + .replace(f'Response:\n{"-" * 80}', '```\n\n#### Response:\n```') + .replace(f'Mismatch:\n{"-" * 80}', '```\n\n#### Mismatch:\n```')) + + # Replace the name of the steps + for x in self.data['steps']: + self.markdown_content = ( + self.markdown_content.replace(f'{x}\n', f'```\n### {x}\n')) + + # Final steps, correct the code style for the title of the Test Cases + # Define patterns and replacement strings + index = True + for x in self.data['cases']: + if index: + self.markdown_content = ( + self.markdown_content.replace(f'```\n## {x}\n\n```\n', f'## {x}')) + index = False + else: + self.markdown_content = ( + self.markdown_content.replace(f'```\n## {x}\n\n```\n', f'```\n## {x}')) + + def save_file(self, filename: str): + # Write the Markdown content to the output file + with open(filename, 'w') as file: + file.write(self.markdown_content) -- GitLab From 895477426037eecc08c3319f202d9a5dfe1540ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 4 Sep 2023 12:12:47 +0200 Subject: [PATCH 07/25] Correcting some cases from the 037_09 TS --- libraries/ErrorListener.py | 10 ++++++++++ libraries/convertMD.py | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index ba163f74..89fecca9 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -4,6 +4,7 @@ from os import getcwd from re import compile, match, MULTILINE from json import loads, dumps from http import HTTPStatus +from convertMD import Markdown def __get_header__(dictionary: dict, key: str) -> str: @@ -103,6 +104,15 @@ class ErrorListener: def end_suite(self, name, attrs): self.outfile.write('\n\n\n') + # If there was an error, generate the markdown content and upload an issue in the corresponding + # GitHub Repository + md = Markdown(filename='./results/errors.log') + md.get_names() + md.generate_md() + md.save_file(filename='./results/partial_error.md') + + # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload + def log_message(self, msg): if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and not match(pattern=self.rx_dict['http_verbs'], string=msg['message'])): diff --git a/libraries/convertMD.py b/libraries/convertMD.py index 8dd2f8bc..018783bb 100644 --- a/libraries/convertMD.py +++ b/libraries/convertMD.py @@ -23,7 +23,7 @@ class Markdown: special_lines = ['Response:', 'Request:', 'Mismatch:', f'{"=" * 150}', f'{"=" * 80}', f'{"-" * 80}'] xs = [x for x in aux if x not in special_lines] - prefixes_to_remove = ["Item ", "+ ", "- "] + prefixes_to_remove = ["Item ", "+ ", "- ", "Value of "] xs = [item for item in xs if not any(item.startswith(prefix) for prefix in prefixes_to_remove)] # Get the name of the Test Suite @@ -65,11 +65,11 @@ class Markdown: for x in self.data['cases']: if index: self.markdown_content = ( - self.markdown_content.replace(f'```\n## {x}\n\n```\n', f'## {x}')) + self.markdown_content.replace(f'```\n## {x}\n\n```\n', f'## {x}\n\n')) index = False else: self.markdown_content = ( - self.markdown_content.replace(f'```\n## {x}\n\n```\n', f'```\n## {x}')) + self.markdown_content.replace(f'```\n## {x}\n\n```\n', f'```\n## {x}\n\n')) def save_file(self, filename: str): # Write the Markdown content to the output file -- GitLab From 545fce839db38df68072d58cb036f448917694b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 4 Sep 2023 12:32:03 +0200 Subject: [PATCH 08/25] Adding GitHub details to the variables.py --- resources/variables.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/resources/variables.py b/resources/variables.py index 017b8eb2..ecbc03af 100644 --- a/resources/variables.py +++ b/resources/variables.py @@ -6,3 +6,8 @@ send_notification_server_host = '0.0.0.0' send_notification_server_port = 8085 context_source_host = '0.0.0.0' context_source_port = 8086 + +# GitHub repository details +github_owner = 'your_github_username' +github_broker_repo = 'context_broker_repository' +github_token = 'your_github_token' -- GitLab From 9f3be48c4ada07e0d313a9e2e0a17ef852a989d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 4 Sep 2023 15:21:08 +0200 Subject: [PATCH 09/25] Split the error message to fix the maximum length of GitHub API to 65535 characters and create the issues in the repo of the Broker --- libraries/ErrorListener.py | 3 ++ libraries/convertMD.py | 3 ++ libraries/githubIssue.py | 64 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 libraries/githubIssue.py diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 89fecca9..e3c4b612 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -5,6 +5,7 @@ from re import compile, match, MULTILINE from json import loads, dumps from http import HTTPStatus from convertMD import Markdown +from githubIssue import GitHubIssue def __get_header__(dictionary: dict, key: str) -> str: @@ -112,6 +113,8 @@ class ErrorListener: md.save_file(filename='./results/partial_error.md') # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload + gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) + gh.create_issue() def log_message(self, msg): if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and diff --git a/libraries/convertMD.py b/libraries/convertMD.py index 018783bb..0be7bc95 100644 --- a/libraries/convertMD.py +++ b/libraries/convertMD.py @@ -75,3 +75,6 @@ class Markdown: # Write the Markdown content to the output file with open(filename, 'w') as file: file.write(self.markdown_content) + + def get_markdown(self) -> str: + return self.markdown_content diff --git a/libraries/githubIssue.py b/libraries/githubIssue.py new file mode 100644 index 00000000..167a9303 --- /dev/null +++ b/libraries/githubIssue.py @@ -0,0 +1,64 @@ +from requests import post +from resources.variables import github_owner, github_broker_repo, github_token +from re import finditer + + +class GitHubIssue: + def __init__(self, issue_title: str, issue_content: str): + # Get the values of the parameter from the variables.py file + # GitHub repository details + repo = 'your_repository' + self.url = f'https://api.github.com/repos/{github_owner}/{github_broker_repo}/issues' + self.issue_title = issue_title + self.issue_content = issue_content + self.test_cases = list() + + def create_issue(self): + # Issue details + # Request headers + headers = { + 'Accept': 'application/vnd.github.v3+json' + } + + # Request body, the issue content need to be split into the different Test Cases in order to prevent + # body maximum of 65536 characters + self.generate_test_cases_info() + + for test_case in self.test_cases: + data = { + 'title': self.issue_title, + 'body': test_case + } + + # Add authentication token to headers if provided + headers['Authorization'] = f'Token {github_token}' + + # Send POST request to create the issue + response = post(url=self.url, headers=headers, json=data) + + # Check the response status code + if response.status_code == 201: + print('Issue created successfully.') + else: + print('Failed to create the issue.') + print(f'Response: {response.status_code} - {response.text}') + + def generate_test_cases_info(self): + pattern = r'##\s*[0-9]{3}_[0-9]{2}_[0-9]{2}.*\n' # Split on one or more non-word characters + + count = int() + indexes = list() + + for match in finditer(pattern, self.issue_content): + count += 1 + indexes.append(match.start()) + + title = self.issue_content[0:indexes[0]] + + for i in range(1, len(indexes) + 1): + if i < len(indexes): + self.test_cases.append(self.issue_content[indexes[i - 1]:indexes[i]]) + else: + self.test_cases.append(self.issue_content[indexes[i - 1]:]) + + self.test_cases = [f'{title}\n\n{x}' for x in self.test_cases] -- GitLab From f52f81018b93b9dd81db7112da0d13e8f5a5b0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 4 Sep 2023 16:53:11 +0200 Subject: [PATCH 10/25] Adding comprobation that the issues were not already created before and they are still open --- libraries/ErrorListener.py | 4 +- libraries/githubIssue.py | 96 +++++++++++++++++++++++++++----------- 2 files changed, 71 insertions(+), 29 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index e3c4b612..d9da9104 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -112,9 +112,11 @@ class ErrorListener: md.generate_md() md.save_file(filename='./results/partial_error.md') + #extended_issue_title = [f"({x.split(' ')[0]})" for x in attrs['tests']] # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) - gh.create_issue() + + gh.create_issues() def log_message(self, msg): if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and diff --git a/libraries/githubIssue.py b/libraries/githubIssue.py index 167a9303..1a22caaa 100644 --- a/libraries/githubIssue.py +++ b/libraries/githubIssue.py @@ -1,47 +1,63 @@ -from requests import post +from requests import post, get from resources.variables import github_owner, github_broker_repo, github_token -from re import finditer +from re import finditer, match class GitHubIssue: def __init__(self, issue_title: str, issue_content: str): # Get the values of the parameter from the variables.py file # GitHub repository details - repo = 'your_repository' - self.url = f'https://api.github.com/repos/{github_owner}/{github_broker_repo}/issues' + self.url_create = f'https://api.github.com/repos/{github_owner}/{github_broker_repo}/issues' + self.issue_title = issue_title self.issue_content = issue_content - self.test_cases = list() - def create_issue(self): - # Issue details - # Request headers - headers = { - 'Accept': 'application/vnd.github.v3+json' - } + self.test_cases = list() + self.test_cases_title = list() + def create_issues(self): # Request body, the issue content need to be split into the different Test Cases in order to prevent # body maximum of 65536 characters self.generate_test_cases_info() - for test_case in self.test_cases: - data = { - 'title': self.issue_title, - 'body': test_case - } - - # Add authentication token to headers if provided - headers['Authorization'] = f'Token {github_token}' + #for test_case in self.test_cases: + for i in range(0, len(self.test_cases_title)): + # We need to check that the issue was not already created previously + # if the issue is created previously and still open we do not create again, + # other case, we create the issue - # Send POST request to create the issue - response = post(url=self.url, headers=headers, json=data) + # Obtain the extended title of the issue + issue_title = f'{self.issue_title} {self.test_cases_title[i]}' - # Check the response status code - if response.status_code == 201: - print('Issue created successfully.') + # Check the issue + if self.check_duplicate_issue(issue_title=issue_title): + print('\nDuplicate issue found!') else: - print('Failed to create the issue.') - print(f'Response: {response.status_code} - {response.text}') + self.create_issue(body=self.test_cases[i]) + + def create_issue(self, body: str): + # Issue details + # Data of the issue + data = { + 'title': self.issue_title, + 'body': body + } + + # Request headers + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'Token {github_token}' + } + + # Send POST request to create the issue + response = post(url=self.url_create, headers=headers, json=data) + + # Check the response status code + if response.status_code == 201: + print('\nIssue created successfully.') + else: + print('\nFailed to create the issue.') + print(f'Response: {response.status_code} - {response.text}') def generate_test_cases_info(self): pattern = r'##\s*[0-9]{3}_[0-9]{2}_[0-9]{2}.*\n' # Split on one or more non-word characters @@ -55,10 +71,34 @@ class GitHubIssue: title = self.issue_content[0:indexes[0]] + # Get the list of Test Cases for i in range(1, len(indexes) + 1): + self.test_cases_title.append(f'({self.issue_content[indexes[i-1]+3:indexes[i-1]+12]})') + if i < len(indexes): - self.test_cases.append(self.issue_content[indexes[i - 1]:indexes[i]]) + self.test_cases.append(self.issue_content[indexes[i-1]:indexes[i]]) else: - self.test_cases.append(self.issue_content[indexes[i - 1]:]) + self.test_cases.append(self.issue_content[indexes[i-1]:]) self.test_cases = [f'{title}\n\n{x}' for x in self.test_cases] + + def check_duplicate_issue(self, issue_title): + # Generate the URL of the query + url = f'repo:{github_owner}/{github_broker_repo} is:issue is:open in:title "{issue_title}"' + + # Make the API request + response = get( + 'https://api.github.com/search/issues', + params={'q': url} + ) + + # Check the response status code + if response.status_code == 200: + # Parse the JSON response + data = response.json() + + # Check if any issues were found + if data['total_count'] > 0: + return True # Duplicate issue found + + return False # No duplicate issue found -- GitLab From 0b0261451d20646c0a29364ae7bec352caf99c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Mon, 4 Sep 2023 17:14:59 +0200 Subject: [PATCH 11/25] Adding check control in case not defined some of the expected GitHub parameters --- libraries/githubIssue.py | 192 +++++++++++++++++++++++---------------- 1 file changed, 114 insertions(+), 78 deletions(-) diff --git a/libraries/githubIssue.py b/libraries/githubIssue.py index 1a22caaa..6ebbe869 100644 --- a/libraries/githubIssue.py +++ b/libraries/githubIssue.py @@ -1,104 +1,140 @@ from requests import post, get -from resources.variables import github_owner, github_broker_repo, github_token -from re import finditer, match +from re import finditer + +try: + from resources.variables import github_owner, github_broker_repo, github_token +except ImportError: + # Some of the variables were not defiled, therefore we cannot execute the operation + classError = True +else: + classError = False class GitHubIssue: def __init__(self, issue_title: str, issue_content: str): - # Get the values of the parameter from the variables.py file - # GitHub repository details - self.url_create = f'https://api.github.com/repos/{github_owner}/{github_broker_repo}/issues' + if classError: + # There is some GitHub parameters not defined, therefore this function does not effect + print("\nSome GitHub parameters were not defined in variables.py") + print("Expected parameters: github_owner, github_broker_repo, github_token") + return + else: + # Get the values of the parameter from the variables.py file + # GitHub repository details + self.url_create = f'https://api.github.com/repos/{github_owner}/{github_broker_repo}/issues' - self.issue_title = issue_title - self.issue_content = issue_content + self.issue_title = issue_title + self.issue_content = issue_content - self.test_cases = list() - self.test_cases_title = list() + self.test_cases = list() + self.test_cases_title = list() def create_issues(self): - # Request body, the issue content need to be split into the different Test Cases in order to prevent - # body maximum of 65536 characters - self.generate_test_cases_info() - - #for test_case in self.test_cases: - for i in range(0, len(self.test_cases_title)): - # We need to check that the issue was not already created previously - # if the issue is created previously and still open we do not create again, - # other case, we create the issue - - # Obtain the extended title of the issue - issue_title = f'{self.issue_title} {self.test_cases_title[i]}' - - # Check the issue - if self.check_duplicate_issue(issue_title=issue_title): - print('\nDuplicate issue found!') - else: - self.create_issue(body=self.test_cases[i]) + if classError: + # There is some GitHub parameters not defined, therefore this function does not effect + print("\nSome GitHub parameters were not defined in variables.py") + print("Expected parameters: github_owner, github_broker_repo, github_token") + return + else: + # Request body, the issue content need to be split into the different Test Cases in order to prevent + # body maximum of 65536 characters + self.generate_test_cases_info() + + for i in range(0, len(self.test_cases_title)): + # We need to check that the issue was not already created previously + # if the issue is created previously and still open we do not create again, + # other case, we create the issue + + # Obtain the extended title of the issue + issue_title = f'{self.issue_title} {self.test_cases_title[i]}' + + # Check the issue + if self.check_duplicate_issue(issue_title=issue_title): + print('\nDuplicate issue found!') + else: + self.create_issue(body=self.test_cases[i]) def create_issue(self, body: str): - # Issue details - # Data of the issue - data = { - 'title': self.issue_title, - 'body': body - } - - # Request headers - headers = { - 'Accept': 'application/vnd.github.v3+json', - 'Authorization': f'Token {github_token}' - } - - # Send POST request to create the issue - response = post(url=self.url_create, headers=headers, json=data) - - # Check the response status code - if response.status_code == 201: - print('\nIssue created successfully.') + if classError: + # There is some GitHub parameters not defined, therefore this function does not effect + print("\nSome GitHub parameters were not defined in variables.py") + print("Expected parameters: github_owner, github_broker_repo, github_token") + return else: - print('\nFailed to create the issue.') - print(f'Response: {response.status_code} - {response.text}') + # Issue details + # Data of the issue + data = { + 'title': self.issue_title, + 'body': body + } + + # Request headers + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'Token {github_token}' + } + + # Send POST request to create the issue + response = post(url=self.url_create, headers=headers, json=data) + + # Check the response status code + if response.status_code == 201: + print('\nIssue created successfully.') + else: + print('\nFailed to create the issue.') + print(f'Response: {response.status_code} - {response.text}') def generate_test_cases_info(self): - pattern = r'##\s*[0-9]{3}_[0-9]{2}_[0-9]{2}.*\n' # Split on one or more non-word characters + if classError: + # There is some GitHub parameters not defined, therefore this function does not effect + print("\nSome GitHub parameters were not defined in variables.py") + print("Expected parameters: github_owner, github_broker_repo, github_token") + return + else: + pattern = r'##\s*[0-9]{3}_[0-9]{2}_[0-9]{2}.*\n' # Split on one or more non-word characters - count = int() - indexes = list() + count = int() + indexes = list() - for match in finditer(pattern, self.issue_content): - count += 1 - indexes.append(match.start()) + for match in finditer(pattern, self.issue_content): + count += 1 + indexes.append(match.start()) - title = self.issue_content[0:indexes[0]] + title = self.issue_content[0:indexes[0]] - # Get the list of Test Cases - for i in range(1, len(indexes) + 1): - self.test_cases_title.append(f'({self.issue_content[indexes[i-1]+3:indexes[i-1]+12]})') + # Get the list of Test Cases + for i in range(1, len(indexes) + 1): + self.test_cases_title.append(f'({self.issue_content[indexes[i-1]+3:indexes[i-1]+12]})') - if i < len(indexes): - self.test_cases.append(self.issue_content[indexes[i-1]:indexes[i]]) - else: - self.test_cases.append(self.issue_content[indexes[i-1]:]) + if i < len(indexes): + self.test_cases.append(self.issue_content[indexes[i-1]:indexes[i]]) + else: + self.test_cases.append(self.issue_content[indexes[i-1]:]) - self.test_cases = [f'{title}\n\n{x}' for x in self.test_cases] + self.test_cases = [f'{title}\n\n{x}' for x in self.test_cases] def check_duplicate_issue(self, issue_title): - # Generate the URL of the query - url = f'repo:{github_owner}/{github_broker_repo} is:issue is:open in:title "{issue_title}"' + if classError: + # There is some GitHub parameters not defined, therefore this function does not effect + print("\nSome GitHub parameters were not defined in variables.py") + print("Expected parameters: github_owner, github_broker_repo, github_token") + return + else: + # Generate the URL of the query + url = f'repo:{github_owner}/{github_broker_repo} is:issue is:open in:title "{issue_title}"' - # Make the API request - response = get( - 'https://api.github.com/search/issues', - params={'q': url} - ) + # Make the API request + response = get( + 'https://api.github.com/search/issues', + params={'q': url} + ) - # Check the response status code - if response.status_code == 200: - # Parse the JSON response - data = response.json() + # Check the response status code + if response.status_code == 200: + # Parse the JSON response + data = response.json() - # Check if any issues were found - if data['total_count'] > 0: - return True # Duplicate issue found + # Check if any issues were found + if data['total_count'] > 0: + return True # Duplicate issue found - return False # No duplicate issue found + return False # No duplicate issue found -- GitLab From 61fd09a1e7a3595c3d704aade0fbee8617dec1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 5 Sep 2023 19:02:36 +0200 Subject: [PATCH 12/25] Resolution Merge comments --- README.md | 14 +++++++ .../{043.robot => 043_01.robot} | 10 ++--- libraries/ErrorListener.py | 41 +++++-------------- libraries/convertMD.py | 2 + 4 files changed, 32 insertions(+), 35 deletions(-) rename TP/NGSI-LD/CommonBehaviours/{043.robot => 043_01.robot} (96%) diff --git a/README.md b/README.md index e8e04b1e..e5eaa528 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,20 @@ In the `resources/variables.py` file, configure the following: * `send_notification_server_host` and `send_notification_server_port` : This is the address and port used when creating the subscription on the context broker (generally it is the same information as `notification_server_host` and `notification_server_port`). * `context_source_host` and `context_source_port` : The address and port used for the context source. +Optionally, there are some extra variables that can be used during the generation of fail logs detail with the Listener +to automatically generate in the Context Broker GitHub the corresponding issue about the Test Cases that failed during +the execution of the Test Suites. In case that you cannot use this functionality, delete those variables from the file. +As an explanation of the process, the GitHub Issue will be created if there is no other issue in the repository with +the same name and the status of the issue is not closed. + +In order to create these issues, we are using the [GitHub REST API](https://docs.github.com/en/rest). For this purpose, +we are using the authentication process with a personal access token. The variables that we need are the following: +* `github_owner`: Your GitHub user account. +* `github_broker_repo` : The corresponding url of the Context Broker repository. +* `github_token` : Your personal access token. Please take a look to the GitHub documentation if you want to generate +your own [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens). + + ## Install the project requirements Browse the base project root folder and execute the following command: diff --git a/TP/NGSI-LD/CommonBehaviours/043.robot b/TP/NGSI-LD/CommonBehaviours/043_01.robot similarity index 96% rename from TP/NGSI-LD/CommonBehaviours/043.robot rename to TP/NGSI-LD/CommonBehaviours/043_01.robot index 3834035c..e92acf6b 100644 --- a/TP/NGSI-LD/CommonBehaviours/043.robot +++ b/TP/NGSI-LD/CommonBehaviours/043_01.robot @@ -19,7 +19,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration *** Test Cases *** -043_01 Create entity +043_01_01 Create entity [Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create entity) [Tags] e-create 5_2_2 ${entity_id}= Generate Random Entity Id ${building_id_prefix} @@ -34,7 +34,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration Check Response Body Containing ProblemDetails Element Containing Title Element ${response.json()} [Teardown] Delete Entity by Id ${entity_id} -043_02 Create subscription +043_01_02 Create subscription [Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create subscription) [Tags] sub-create 5_2_2 ${subscription_id}= Generate Random Entity Id ${subscription_id_prefix} @@ -46,7 +46,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration Check Response Body Containing ProblemDetails Element Containing Title Element ${response.json()} [Teardown] Delete Subscription ${subscription_id} -043_03 Create Temporal Representation of Entities +043_01_03 Create Temporal Representation of Entities [Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create Temporal Representation of Entities) [Tags] te-create 5_2_2 ${temporal_entity_representation_id}= Generate Random Entity Id ${tea_id_prefix} @@ -61,7 +61,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration Check Response Body Containing ProblemDetails Element Containing Title Element ${response.json()} [Teardown] Delete Temporal Representation Of Entity ${temporal_entity_representation_id} -043_04 Batch entity create +043_01_04 Batch entity create [Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Batch entity create) [Tags] be-create 5_2_2 ${first_entity_id}= Generate Random Entity Id ${building_id_prefix} @@ -78,7 +78,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration Check Response Body Containing ProblemDetails Element Containing Title Element ${response.json()} [Teardown] Batch Delete Entities @{entities_ids_to_be_created} -043_05 Create context source registration +043_01_05 Create context source registration [Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create context source registration) [Tags] csr-create 5_2_2 ${registration_id}= Generate Random Entity Id ${registration_id_prefix} diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index d9da9104..59895f74 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -65,9 +65,10 @@ class ErrorListener: def __init__(self, filename='errors.log'): self.cwd = getcwd() out_path = join('results', filename) + self.outfile = open(out_path, 'w') + self.max_length_suite = 150 self.max_length_case = 80 - self.outfile = open(out_path, 'w') self.tests = list() self.suite_name = str() self.rx_dict = { @@ -110,9 +111,8 @@ class ErrorListener: md = Markdown(filename='./results/errors.log') md.get_names() md.generate_md() - md.save_file(filename='./results/partial_error.md') + md.save_file(filename='./results/errors.md') - #extended_issue_title = [f"({x.split(' ')[0]})" for x in attrs['tests']] # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) @@ -148,46 +148,27 @@ class ErrorListener: data = loads(data) output = list() - my_header_keys = ['User-Agent', - 'Accept-Encoding', - 'Accept', - 'Connection', - 'Link', - 'Content-Type', - 'Content-Length', - 'Date', - 'Location'] received_header_keys = data['headers'].keys() - final_keys = list(set(received_header_keys) - set(my_header_keys)) + if 'User-Agent' in received_header_keys: + # User-Agent is a Request Header, therefore we generate the request header + output.append(f' {data["method"]} {data["url"]}\n') - if len(final_keys) != 0: - output.append(f'\n\nERROR: Need to add the following header key(s): {final_keys}\n\n') + [output.append(__get_header__(dictionary=data, key=x)) for x in list(received_header_keys)] - try: - output.append(f' {data["method"]} {data["url"]}\n') - output.append(f' User-Agent: {data["headers"]["User-Agent"]}\n') - output.append(f' Accept-Encoding: {data["headers"]["Accept-Encoding"]}\n') - output.append(f' Accept: {data["headers"]["Accept"]}\n') - output.append(f' Connection: {data["headers"]["Connection"]}\n') - - output.append(__get_header__(dictionary=data, key='Link')) - output.append(__get_header__(dictionary=data, key='Content-Type')) - output.append(__get_header__(dictionary=data, key='Content-Length')) output.append('\n') output.append(__get_body__(dictionary=data['body'])) - except KeyError as e: - # This is a Response dictionary and not a Request dictionary + else: + # This is a Response header # robotframework-requests is based on python request, so it is using HTTP/1.1 output.append(f' HTTP/1.1 {data["status_code"]} {__get_status_meaning__(data["status_code"])}\n') - output.append(__get_header__(dictionary=data, key='Content-Type')) + [output.append(__get_header__(dictionary=data, key=x)) for x in list(received_header_keys)] output.append(f' Date: REGEX(. *)\n') - output.append(__get_header__(dictionary=data, key='Location')) - output.append(__get_header__(dictionary=data, key='Content-Length')) + output.append('\n') output.append(__get_body__(dictionary=data['body'])) diff --git a/libraries/convertMD.py b/libraries/convertMD.py index 0be7bc95..48e571d4 100644 --- a/libraries/convertMD.py +++ b/libraries/convertMD.py @@ -76,5 +76,7 @@ class Markdown: with open(filename, 'w') as file: file.write(self.markdown_content) + file.close() + def get_markdown(self) -> str: return self.markdown_content -- GitLab From 8fe0e66d698a9660438b2d170bbf1aecdbe3f232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 5 Sep 2023 19:19:50 +0200 Subject: [PATCH 13/25] Resolve some special Mismatches text issues in the generation of markdown --- libraries/ErrorListener.py | 4 +++- libraries/convertMD.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 59895f74..c3e6a980 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -105,6 +105,7 @@ class ErrorListener: def end_suite(self, name, attrs): self.outfile.write('\n\n\n') + self.outfile.close() # If there was an error, generate the markdown content and upload an issue in the corresponding # GitHub Repository @@ -124,7 +125,8 @@ class ErrorListener: self.tests.append(self.__get_message__(msg["message"])) def close(self): - self.outfile.close() + #self.outfile.close() + pass def __get_message__(self, message: str) -> str: result = str() diff --git a/libraries/convertMD.py b/libraries/convertMD.py index 48e571d4..26d4ecac 100644 --- a/libraries/convertMD.py +++ b/libraries/convertMD.py @@ -7,6 +7,8 @@ class Markdown: with open(filename, 'r') as file: self.content = file.read() + file.close() + self.data = { "suite": str(), "cases": list(), @@ -23,7 +25,7 @@ class Markdown: special_lines = ['Response:', 'Request:', 'Mismatch:', f'{"=" * 150}', f'{"=" * 80}', f'{"-" * 80}'] xs = [x for x in aux if x not in special_lines] - prefixes_to_remove = ["Item ", "+ ", "- ", "Value of "] + prefixes_to_remove = ["Item ", "+ ", "- ", "Value of ", "HTTP status code", "HTTPError:", "AttributeError:"] xs = [item for item in xs if not any(item.startswith(prefix) for prefix in prefixes_to_remove)] # Get the name of the Test Suite -- GitLab From c8a46d12aba6ded351fcf453af63c1c9e5795420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Thu, 7 Sep 2023 10:43:15 +0200 Subject: [PATCH 14/25] Clean ErrorListener class --- libraries/ErrorListener.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index c3e6a980..3a05f479 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -1,4 +1,3 @@ -import re from os.path import join from os import getcwd from re import compile, match, MULTILINE @@ -141,7 +140,7 @@ class ErrorListener: result = message elif message == 'Dictionary comparison failed with -> ': result == None - elif re.match(pattern=self.rx_dict['length_log'], string=message) is None: + elif match(pattern=self.rx_dict['length_log'], string=message) is None: result = f'\nMismatch:\n{"-" * self.max_length_case}\n{message}\n' return result -- GitLab From ad915ae4aa1684628d75bb82f7510887c12a5dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 8 Sep 2023 12:00:01 +0200 Subject: [PATCH 15/25] Adding control of output result data based on --outputdir cli parameter --- libraries/ErrorListener.py | 32 +++++++++++++++++++++++++------- libraries/githubIssue.py | 10 +++++++++- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 3a05f479..2ad65025 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -1,10 +1,11 @@ -from os.path import join +from os.path import join, splitext from os import getcwd from re import compile, match, MULTILINE from json import loads, dumps from http import HTTPStatus from convertMD import Markdown from githubIssue import GitHubIssue +from robot.running.context import EXECUTION_CONTEXTS def __get_header__(dictionary: dict, key: str) -> str: @@ -62,9 +63,11 @@ class ErrorListener: ROBOT_LISTENER_API_VERSION = 2 def __init__(self, filename='errors.log'): - self.cwd = getcwd() - out_path = join('results', filename) - self.outfile = open(out_path, 'w') + self.filename_md = None + self.filename_log = None + self.cwd = None + self.outfile = None + self.filename = filename self.max_length_suite = 150 self.max_length_case = 80 @@ -84,7 +87,22 @@ class ErrorListener: 'length_log': compile('^Length is \d+$') } + def generate_file_path(self): + if self.outfile is None: + ns = EXECUTION_CONTEXTS.current + output_dir = ns.variables.current.store.data['OUTPUT_DIR'] + + basename = splitext(self.filename)[0] + self.filename_log = join(output_dir, self.filename) + self.filename_md = join(output_dir, f'{basename}.md') + + self.cwd = getcwd() + out_path = join('results', self.filename) + self.outfile = open(out_path, 'w') + def start_suite(self, name, attrs): + self.generate_file_path() + self.suite_name = attrs['source'].replace(self.cwd, '')[1:].replace('.robot', '').replace('/', ".") if attrs['doc'] != '': @@ -108,10 +126,10 @@ class ErrorListener: # If there was an error, generate the markdown content and upload an issue in the corresponding # GitHub Repository - md = Markdown(filename='./results/errors.log') + md = Markdown(filename=self.filename_log) md.get_names() md.generate_md() - md.save_file(filename='./results/errors.md') + md.save_file(filename=self.filename_md) # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) @@ -124,7 +142,7 @@ class ErrorListener: self.tests.append(self.__get_message__(msg["message"])) def close(self): - #self.outfile.close() + # self.outfile.close() pass def __get_message__(self, message: str) -> str: diff --git a/libraries/githubIssue.py b/libraries/githubIssue.py index 6ebbe869..8163a660 100644 --- a/libraries/githubIssue.py +++ b/libraries/githubIssue.py @@ -1,5 +1,6 @@ from requests import post, get from re import finditer +from json import loads try: from resources.variables import github_owner, github_broker_repo, github_token @@ -95,11 +96,16 @@ class GitHubIssue: count = int() indexes = list() + match = None for match in finditer(pattern, self.issue_content): count += 1 indexes.append(match.start()) - title = self.issue_content[0:indexes[0]] + if match: + title = self.issue_content[0:indexes[0]] + else: + raise KeyError("Search unsuccessful. It was expected the the name of the Test Cases start with " + "_
_
, where d is a digit, e.g., 027_01_01") # Get the list of Test Cases for i in range(1, len(indexes) + 1): @@ -136,5 +142,7 @@ class GitHubIssue: # Check if any issues were found if data['total_count'] > 0: return True # Duplicate issue found + else: + raise Exception(loads(response.text)['errors'][0]['message']) return False # No duplicate issue found -- GitLab From 2a6c354626f5df60dda4b04dc3e37805c2ad8a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 8 Sep 2023 12:05:39 +0200 Subject: [PATCH 16/25] Control the case that no --outputdir parameter is provided in the CLI --- libraries/ErrorListener.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 2ad65025..2937862d 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -97,8 +97,7 @@ class ErrorListener: self.filename_md = join(output_dir, f'{basename}.md') self.cwd = getcwd() - out_path = join('results', self.filename) - self.outfile = open(out_path, 'w') + self.outfile = open(self.filename_log, 'w') def start_suite(self, name, attrs): self.generate_file_path() -- GitLab From 6c826f89102fcda4e4e5d51b271cdf41df1da952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Fri, 8 Sep 2023 14:17:36 +0200 Subject: [PATCH 17/25] Allow multiple execution of TSs in order to generate the file --- libraries/ErrorListener.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 2937862d..f89b8dad 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -135,14 +135,17 @@ class ErrorListener: gh.create_issues() + # We need to reopen the file in case that we are executing several TCs + self.outfile = open(self.filename_log, 'a') + def log_message(self, msg): if (not match(pattern=self.rx_dict['variables'], string=msg['message']) and not match(pattern=self.rx_dict['http_verbs'], string=msg['message'])): self.tests.append(self.__get_message__(msg["message"])) def close(self): - # self.outfile.close() - pass + self.outfile.write('\n\n\n') + self.outfile.close() def __get_message__(self, message: str) -> str: result = str() -- GitLab From d58aa2dca19889395ba080e155d5c19aa678c482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 12 Sep 2023 09:59:59 +0200 Subject: [PATCH 18/25] Multiple execution of TSs --- libraries/ErrorListener.py | 10 +++++++--- libraries/convertMD.py | 28 ++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index f89b8dad..17b9f3e4 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -67,6 +67,7 @@ class ErrorListener: self.filename_log = None self.cwd = None self.outfile = None + self.previous_content = str() self.filename = filename self.max_length_suite = 150 @@ -125,15 +126,18 @@ class ErrorListener: # If there was an error, generate the markdown content and upload an issue in the corresponding # GitHub Repository - md = Markdown(filename=self.filename_log) + md = Markdown(filename=self.filename_log, previous_content=self.previous_content) md.get_names() md.generate_md() - md.save_file(filename=self.filename_md) + self.previous_content = md.save_file(filename=self.filename_md) # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) - gh.create_issues() + try: + gh.create_issues() + except Exception as err: + print(f'\nUnexpected {err=}, {type(err)=}\n\n') # We need to reopen the file in case that we are executing several TCs self.outfile = open(self.filename_log, 'a') diff --git a/libraries/convertMD.py b/libraries/convertMD.py index 26d4ecac..763ac99c 100644 --- a/libraries/convertMD.py +++ b/libraries/convertMD.py @@ -1,14 +1,33 @@ from re import compile, match, findall, MULTILINE +from difflib import SequenceMatcher + + +def get_string_difference(string1: str, string2: str) -> str: + differ = SequenceMatcher(None, string1, string2) + differences = differ.get_opcodes() + diff_string = "" + + for tag, i1, i2, j1, j2 in differences: + if tag == 'delete' or tag == 'replace': + diff_string += string1[i1:i2] + elif tag == 'insert' or tag == 'replace': + diff_string += string2[j1:j2] + + return diff_string class Markdown: - def __init__(self, filename: str): + def __init__(self, filename: str, previous_content: str): # Read the content of the input file with open(filename, 'r') as file: self.content = file.read() - file.close() + # Initial previous content + if previous_content != '': + # If there was a previous content in the file, take the difference to do the process + self.content = get_string_difference(string1=previous_content, string2=self.content) + self.data = { "suite": str(), "cases": list(), @@ -75,10 +94,11 @@ class Markdown: def save_file(self, filename: str): # Write the Markdown content to the output file - with open(filename, 'w') as file: + with open(filename, 'a') as file: file.write(self.markdown_content) - file.close() + return self.content + def get_markdown(self) -> str: return self.markdown_content -- GitLab From 2e08f2a42affba2cc9c721d2ff0f410f377e511b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 12 Sep 2023 10:52:02 +0200 Subject: [PATCH 19/25] Controlling if the *.md file already exists in the folder --- libraries/ErrorListener.py | 10 ++++++++-- libraries/convertMD.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index 17b9f3e4..ab8dc892 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -1,5 +1,5 @@ -from os.path import join, splitext -from os import getcwd +from os.path import join, splitext, exists +from os import getcwd, remove from re import compile, match, MULTILINE from json import loads, dumps from http import HTTPStatus @@ -90,6 +90,7 @@ class ErrorListener: def generate_file_path(self): if self.outfile is None: + # This is the first time that we execute the test therefore we configure the filenames ns = EXECUTION_CONTEXTS.current output_dir = ns.variables.current.store.data['OUTPUT_DIR'] @@ -100,6 +101,11 @@ class ErrorListener: self.cwd = getcwd() self.outfile = open(self.filename_log, 'w') + # Check if a previous version of the markdown file exists in the folder, then we delete it in order + # not to append to this file + if exists(self.filename_md): + remove(self.filename_md) + def start_suite(self, name, attrs): self.generate_file_path() diff --git a/libraries/convertMD.py b/libraries/convertMD.py index 763ac99c..5f703d37 100644 --- a/libraries/convertMD.py +++ b/libraries/convertMD.py @@ -92,6 +92,13 @@ class Markdown: self.markdown_content = ( self.markdown_content.replace(f'```\n## {x}\n\n```\n', f'```\n## {x}\n\n')) + # If the final number of "```" is odd, means that we need to close the last code section + # this is a workaround to close the last section of code if this is keep open + count = self.markdown_content.count("```") + if count % 2 == 1: + print(True) + self.markdown_content = f"{self.markdown_content}\n```" + def save_file(self, filename: str): # Write the Markdown content to the output file with open(filename, 'a') as file: -- GitLab From 66bcca0cee5f85d8ace6ceb57600e5f34bfbe477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Tue, 12 Sep 2023 18:09:49 +0200 Subject: [PATCH 20/25] Adding exception control --- libraries/ErrorListener.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index ab8dc892..a528aa3c 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -130,20 +130,24 @@ class ErrorListener: self.outfile.write('\n\n\n') self.outfile.close() - # If there was an error, generate the markdown content and upload an issue in the corresponding - # GitHub Repository - md = Markdown(filename=self.filename_log, previous_content=self.previous_content) - md.get_names() - md.generate_md() - self.previous_content = md.save_file(filename=self.filename_md) + try: + # If there was an error, generate the markdown content and upload an issue in the corresponding + # GitHub Repository + md = Markdown(filename=self.filename_log, previous_content=self.previous_content) + md.get_names() + md.generate_md() + self.previous_content = md.save_file(filename=self.filename_md) - # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload - gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) + # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload + gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) - try: gh.create_issues() + except KeyError as err: + print(f'\n[ERROR] Unexpected {err=}, {type(err)=} in TC {self.suite_name}\n\n') + except IndexError as err: + print(f'\n[ERROR] Unexpected {err=}, {type(err)=} in TC {self.suite_name}\n\n') except Exception as err: - print(f'\nUnexpected {err=}, {type(err)=}\n\n') + print(f'\n[ERROR] Unexpected {err=}, {type(err)=} in TC {self.suite_name}\n\n') # We need to reopen the file in case that we are executing several TCs self.outfile = open(self.filename_log, 'a') -- GitLab From ab92329d73cec3f5280cab46fbb2d035a94617ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20L=C3=B3pez=20Aguilar?= Date: Sat, 23 Sep 2023 09:43:55 +0200 Subject: [PATCH 21/25] Resolving IndexError --- libraries/ErrorListener.py | 6 +++--- libraries/convertMD.py | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libraries/ErrorListener.py b/libraries/ErrorListener.py index a528aa3c..2e6ae719 100644 --- a/libraries/ErrorListener.py +++ b/libraries/ErrorListener.py @@ -135,13 +135,13 @@ class ErrorListener: # GitHub Repository md = Markdown(filename=self.filename_log, previous_content=self.previous_content) md.get_names() - md.generate_md() + # md.generate_md() self.previous_content = md.save_file(filename=self.filename_md) # Check if we have defined the GitHub parameters in the variables.py file, if this is the case upload - gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) + # gh = GitHubIssue(issue_title=f'{attrs["longname"]} - {attrs["doc"]}', issue_content=md.get_markdown()) - gh.create_issues() + # gh.create_issues() except KeyError as err: print(f'\n[ERROR] Unexpected {err=}, {type(err)=} in TC {self.suite_name}\n\n') except IndexError as err: diff --git a/libraries/convertMD.py b/libraries/convertMD.py index 5f703d37..5fb39483 100644 --- a/libraries/convertMD.py +++ b/libraries/convertMD.py @@ -51,8 +51,11 @@ class Markdown: self.data["suite"] = xs[0] # Get the names of the Test Cases - pattern = r"\d{3}\w+" - self.data["cases"] = [item for item in xs if match(pattern, item)] + try: + pattern = r"\d{3}\w+" + self.data["cases"] = [item for item in xs if match(pattern, item)] + except IndexError as err: + print(f'\n[ERROR] Unexpected {err=}, {type(err)=} in TC {self.suite_name}\n\n') # Get the rest of values -> Steps # Get items from listA not present in listB and not equal to exclude_string -- GitLab From b3f944e42d70e4a3270307a06794922b2ed279e7 Mon Sep 17 00:00:00 2001 From: Luis A Frigolet Date: Mon, 19 Feb 2024 15:53:59 +0100 Subject: [PATCH 22/25] Fixed bug at ngsildtest suites --- ngsildtest | 600 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 600 insertions(+) create mode 100755 ngsildtest diff --git a/ngsildtest b/ngsildtest new file mode 100755 index 00000000..469fb84d --- /dev/null +++ b/ngsildtest @@ -0,0 +1,600 @@ +#!/usr/bin/env python + +# Documentation +# https://kislyuk.github.io/argcomplete/ +# +# + +# +# Installation +# pip install argcomplete +# activate-global-python-argcomplete +# +# +# In global completion mode, you don’t have to register each argcomplete-capable executable separately. +# Instead, the shell will look for the string PYTHON_ARGCOMPLETE_OK in the first 1024 bytes of any +# executable that it’s running completion for, and if it’s found, follow the rest of the argcomplete +# protocol as described above. +# +# Additionally, completion is activated for scripts run as python