From 13c98733c1232861061491c803fa33b3ba231742 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 14 Sep 2023 10:16:45 +0100 Subject: [PATCH 1/7] Adding extra example --- .../examples/json/request5-JSON-Delivery.json | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 103120/examples/json/request5-JSON-Delivery.json diff --git a/103120/examples/json/request5-JSON-Delivery.json b/103120/examples/json/request5-JSON-Delivery.json new file mode 100644 index 0000000..8608393 --- /dev/null +++ b/103120/examples/json/request5-JSON-Delivery.json @@ -0,0 +1,65 @@ +{ + "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", + "@xmlns:delivery": "http://uri.etsi.org/03120/common/2019/10/Delivery", + "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", + "Header": { + "SenderIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR2" + }, + "ReceiverIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR1" + }, + "TransactionIdentifier": "64cd73e1-a438-4d47-9cfb-cead9125a502", + "Timestamp": "2019-09-30T13:37:37.000000Z", + "Version": { + "ETSIVersion": "V1.14.1", + "NationalProfileOwner": "XX", + "NationalProfileVersion": "v1.0" + } + }, + "Payload": { + "RequestPayload": { + "ActionRequests": { + "ActionRequest": [ + { + "ActionIdentifier": 0, + "DELIVER": { + "Identifier": "11f49f11-39df-4596-aa58-1f7199b587df", + "HI1Object": { + "@xsi:type": "{http://uri.etsi.org/03120/common/2019/10/Delivery}DeliveryObject", + "ObjectIdentifier": "11f49f11-39df-4596-aa58-1f7199b587df", + "AssociatedObjects": { + "AssociatedObject": [ + "2b36a78b-b628-416d-bd22-404e68a0cd36" + ] + }, + "delivery:Reference": { + "delivery:LDID": "XX-ACTOR01-1234" + }, + "delivery:SequenceNumber": 1, + "delivery:LastSequence": true, + "delivery:Manifest": { + "delivery:Specification": { + "common:Owner": "ETSI", + "common:Name": "ManifestSpecification", + "common:Value": "TS102657-XML" + } + }, + "delivery:Delivery": { + "delivery:EmbeddedJSONData": { + "field1" : "this is native JSON embedded data", + "field2" : 1234 + } + } + } + } + } + ] + } + } + } +} -- GitLab From 8d9d6ddeb27886ae672691d80dc018100dce869e Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 14 Sep 2023 10:40:58 +0100 Subject: [PATCH 2/7] Updating XSD to include JSON data elements --- .../examples/json/request5-JSON-Delivery.json | 2 +- .../json/ts_103120_Delivery.schema.json | 25 ++++++++++++++++++- 103120/schema/xsd/ts_103120_Delivery.xsd | 7 ++++++ utils/translate_spec.py | 9 ++++--- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/103120/examples/json/request5-JSON-Delivery.json b/103120/examples/json/request5-JSON-Delivery.json index 8608393..b0796ca 100644 --- a/103120/examples/json/request5-JSON-Delivery.json +++ b/103120/examples/json/request5-JSON-Delivery.json @@ -50,7 +50,7 @@ } }, "delivery:Delivery": { - "delivery:EmbeddedJSONData": { + "delivery:JSONData": { "field1" : "this is native JSON embedded data", "field2" : 1234 } diff --git a/103120/schema/json/ts_103120_Delivery.schema.json b/103120/schema/json/ts_103120_Delivery.schema.json index 093d127..f87f7f5 100644 --- a/103120/schema/json/ts_103120_Delivery.schema.json +++ b/103120/schema/json/ts_103120_Delivery.schema.json @@ -145,6 +145,17 @@ "required": [ "delivery:XMLSchema" ] + }, + { + "type": "object", + "properties": { + "delivery:JSONSchema": { + "$ref": "#/$defs/EmbeddedJSONData" + } + }, + "required": [ + "delivery:JSONSchema" + ] } ] }, @@ -180,6 +191,17 @@ "required": [ "delivery:XMLData" ] + }, + { + "type": "object", + "properties": { + "delivery:JSONData": { + "$ref": "#/$defs/EmbeddedJSONData" + } + }, + "required": [ + "delivery:JSONData" + ] } ] }, @@ -204,6 +226,7 @@ "delivery:Data" ] }, - "EmbeddedXMLData": {} + "EmbeddedXMLData": {}, + "EmbeddedJSONData": {} } } \ No newline at end of file diff --git a/103120/schema/xsd/ts_103120_Delivery.xsd b/103120/schema/xsd/ts_103120_Delivery.xsd index d1daddd..0cb4b4c 100644 --- a/103120/schema/xsd/ts_103120_Delivery.xsd +++ b/103120/schema/xsd/ts_103120_Delivery.xsd @@ -39,6 +39,7 @@ + @@ -50,6 +51,7 @@ + @@ -65,4 +67,9 @@ + + + + + diff --git a/utils/translate_spec.py b/utils/translate_spec.py index eac24e4..bac9554 100644 --- a/utils/translate_spec.py +++ b/utils/translate_spec.py @@ -70,10 +70,13 @@ if __name__ == "__main__": json_schemas = {} for schema_tuple in schema_locations: logging.info(f" Translating {schema_tuple}") - if 'xmldsig' in (schema_tuple[1]): - # TODO - work out what to do here - logging.info(" Skipping XML Dsig...") + if ns_map[schema_tuple[0]].get('skip'): + logging.info(f" Skipping {schema_tuple[0]} due to config...") continue + # if 'xmldsig' in (schema_tuple[1]): + # # TODO - work out what to do here + # logging.info(" Skipping XML Dsig...") + # continue js = translate_schema(schema_tuple[1], ns_map, schema_locations) # TODO - Special case, get rid of signature -- GitLab From 2c773d950c81bfb64c6665e5c0786e0ccefb04c2 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 14 Sep 2023 10:43:56 +0100 Subject: [PATCH 3/7] Tidying translation code --- utils/translate_spec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/translate_spec.py b/utils/translate_spec.py index bac9554..73b4ab7 100644 --- a/utils/translate_spec.py +++ b/utils/translate_spec.py @@ -73,10 +73,6 @@ if __name__ == "__main__": if ns_map[schema_tuple[0]].get('skip'): logging.info(f" Skipping {schema_tuple[0]} due to config...") continue - # if 'xmldsig' in (schema_tuple[1]): - # # TODO - work out what to do here - # logging.info(" Skipping XML Dsig...") - # continue js = translate_schema(schema_tuple[1], ns_map, schema_locations) # TODO - Special case, get rid of signature -- GitLab From 495654f863201d62bba86373951aab1ba996b33f Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 14 Sep 2023 10:48:07 +0100 Subject: [PATCH 4/7] Adding schema to example --- 103120/examples/json/request5-JSON-Delivery.json | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/103120/examples/json/request5-JSON-Delivery.json b/103120/examples/json/request5-JSON-Delivery.json index b0796ca..e3d9d1c 100644 --- a/103120/examples/json/request5-JSON-Delivery.json +++ b/103120/examples/json/request5-JSON-Delivery.json @@ -43,10 +43,13 @@ "delivery:SequenceNumber": 1, "delivery:LastSequence": true, "delivery:Manifest": { - "delivery:Specification": { - "common:Owner": "ETSI", - "common:Name": "ManifestSpecification", - "common:Value": "TS102657-XML" + "delivery:ExternalSchema": { + "delivery:ManifestID" : "ExampleJSONSchema", + "delivery:ManifestContents" : { + "delivery:JSONSchema" : { + "jsonStuff" : "goesHere" + } + } } }, "delivery:Delivery": { -- GitLab From 9b488ef7d1ef0ba1d231a5867b09981e4f4b8796 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 14 Sep 2023 10:48:20 +0100 Subject: [PATCH 5/7] Adding schema to example --- 103120/examples/json/request5-JSON-Delivery.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/103120/examples/json/request5-JSON-Delivery.json b/103120/examples/json/request5-JSON-Delivery.json index e3d9d1c..fa510ca 100644 --- a/103120/examples/json/request5-JSON-Delivery.json +++ b/103120/examples/json/request5-JSON-Delivery.json @@ -47,7 +47,7 @@ "delivery:ManifestID" : "ExampleJSONSchema", "delivery:ManifestContents" : { "delivery:JSONSchema" : { - "jsonStuff" : "goesHere" + "actualJSONschema" : "goeshere" } } } -- GitLab From 39fd9621cd426a73258065f4427f6ff928dbda1f Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 10 Oct 2023 09:31:46 +0100 Subject: [PATCH 6/7] Updating examples --- .../examples/json/request5-JSON-Delivery.json | 2 +- 103120/schema/json/ts_103120_Core.schema.json | 16 +- utils/json_validator.py | 156 ++++++++++++++++++ utils/translate_spec.py | 21 ++- 4 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 utils/json_validator.py diff --git a/103120/examples/json/request5-JSON-Delivery.json b/103120/examples/json/request5-JSON-Delivery.json index fa510ca..34d84a0 100644 --- a/103120/examples/json/request5-JSON-Delivery.json +++ b/103120/examples/json/request5-JSON-Delivery.json @@ -47,7 +47,7 @@ "delivery:ManifestID" : "ExampleJSONSchema", "delivery:ManifestContents" : { "delivery:JSONSchema" : { - "actualJSONschema" : "goeshere" + "schema_goes_here" : "schema_goes_here" } } } diff --git a/103120/schema/json/ts_103120_Core.schema.json b/103120/schema/json/ts_103120_Core.schema.json index 8225cf3..63e2ad4 100644 --- a/103120/schema/json/ts_103120_Core.schema.json +++ b/103120/schema/json/ts_103120_Core.schema.json @@ -13,9 +13,19 @@ "Payload": { "$ref": "#/$defs/MessagePayload" }, - "Signature": {}, - "xmldsig:Signature": { - "$ref": "www.w3.org_2000_09_xmldsig##/$defs/SignatureType" + "Signature": { + "properties": { + "protected": { + "type": "string" + }, + "signature": { + "type": "string" + } + }, + "required": [ + "protected", + "signature" + ] } }, "required": [ diff --git a/utils/json_validator.py b/utils/json_validator.py new file mode 100644 index 0000000..fb46193 --- /dev/null +++ b/utils/json_validator.py @@ -0,0 +1,156 @@ +import sys +from jsonschema import validate, RefResolver, Draft202012Validator +from jsonschema.exceptions import ValidationError +import json +from pathlib import Path +import logging +import argparse +from itertools import chain + +class JsonValidator: + def __init__(self, core_schema: str, other_schemas : dict): + self._core_schema = json.load(Path(core_schema).open()) + self._schema_dict = { self._core_schema['$id'] : self._core_schema } + self._supporting_paths = [] + for thing in other_schemas: + path = Path(thing) + if path.is_dir(): + logging.debug(f"Searching {path} for schema files") + self._supporting_paths.extend(path.rglob("*.schema.json")) + else: + logging.debug(f"Appending {path} as schema file") + self._supporting_paths.append(path) + logging.info(f"Supporting schema paths: {self._supporting_paths}") + self._supporting_schemas = [json.load(p.open()) for p in self._supporting_paths] + self._schema_dict = self._schema_dict | { s['$id'] : s for s in self._supporting_schemas } + logging.info(f"Loaded schema IDs: {[k for k in self._schema_dict.keys()]}") + self._resolver = RefResolver(None, + referrer=None, + store=self._schema_dict) + logging.info("Created RefResolver") + self._validator = Draft202012Validator(self._core_schema, resolver=self._resolver) + logging.info("Created validator") + + def validate(self, instance_doc: str): + errors = list(self._validator.iter_errors(instance_doc)) + return errors + +class TS103120Validator (JsonValidator): + def __init__ (self, path_to_repo): + repo_path = Path(path_to_repo) + schema_dirs = [str(repo_path / "103120/schema/json"), str("103280/")] + core_schema = str(repo_path / "103120/schema/json/ts_103120_Core.schema.json") + JsonValidator.__init__(self, core_schema, schema_dirs) + request_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/RequestPayload" } + self._request_fragment_validator = Draft202012Validator(request_fragment_schema, resolver=self._resolver) + response_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/ResponsePayload" } + self._response_fragment_validator = Draft202012Validator(response_fragment_schema, resolver=self._resolver) + + def expand_request_response_exception (self, ex): + if list(ex.schema_path) == ['properties', 'Payload', 'oneOf']: + logging.info ("Error detected validating payload oneOf - attempting explicit validation...") + if 'RequestPayload' in instance_doc['Payload'].keys(): + ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['RequestPayload'])])) + for r in ret_list: + r.path = ex.path + r.path + return ret_list + elif 'ResponsePayload' in instance_doc['Payload'].keys(): + ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['ResponsePayload'])])) + for r in ret_list: + r.path = ex.path + r.path + return ret_list + else: + logging.error("No RequestPayload or ResponsePayload found - is the Payload malformed?") + return [ex] + else: + return [ex] + + def expand_action_exception (self, ex): + logging.error("Error detected in ActionRequests/ActionResponses") + error_path = list(ex.schema_path) + if error_path != ['properties', 'ActionRequests', 'properties', 'ActionRequest', 'items', 'allOf', 1, 'oneOf'] and error_path != ['properties', 'ActionResponses', 'properties', 'ActionResponse', 'items', 'allOf', 1, 'oneOf']: + logging.error("Error not in inner Request/Response allOf/oneOf constraint") + return[ex] + j = ex.instance + j.pop('ActionIdentifier') # Remove ActionIdentifier - one remaining key will be the verb + verb = list(j.keys())[0] + message = "Request" if error_path[1] == "ActionRequests" else "Response" + v = Draft202012Validator({"$ref" : f"ts_103120_Core_2019_10#/$defs/{verb}{message}"}, resolver=self._resolver) + ret_list = list(chain(*[self.expand_object_exception(x) for x in v.iter_errors(j[verb])])) + for r in ret_list: + r.path = ex.path + r.path + return ret_list + + def expand_object_exception (self, ex): + logging.error("Error detected in verb") + # The final level of validation is for the actual HI1Object validation + if list(ex.schema_path) != ['properties', 'HI1Object', 'oneOf']: + logging.error("Error not inside HI1Object") + return [ex] + object_type = ex.instance['@xsi:type'].split('}')[-1] + object_ref = { + 'AuthorisationObject': 'ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject', + 'LITaskObject': 'ts_103120_Task_2020_09#/$defs/LITaskObject', + 'LDTaskObject': 'ts_103120_Task_2020_09#/$defs/LDTaskObject', + 'LPTaskObject': 'ts_103120_Task_2020_09#/$defs/LPTaskObject', + 'DocumentObject': 'ts_103120_Document_2020_09#/$defs/DocumentObject', + 'NotificationObject': 'ts_103120_Notification_2016_02#/$defs/NotificationObject', + 'DeliveryObject': 'ts_103120_Delivery_2019_10#/$defs/DeliveryObject', + 'TrafficPolicyObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject', + 'TrafficRuleObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject', + }[object_type] + v = Draft202012Validator({"$ref" : object_ref}, resolver=self._resolver) + return list(v.iter_errors(ex.instance)) + + def validate(self, instance_doc: str): + errors = JsonValidator.validate(self, instance_doc) + out_errors = list(chain(*[self.expand_request_response_exception(ex) for ex in errors])) + return out_errors + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") + parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument('--ts103120', action="store_true", help="Validate a TS 103 120 JSON document") + parser.add_argument('--schema', default=None, help="Primary schema to validate against") + parser.add_argument('-p', '--printerror', action="count", help="Controls how verbose validation error printing is (can be specified multiple times)") + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + if (args.ts103120): + v = TS103120Validator("./") + else: + v = JsonValidator(args.schema, args.schemadir) + + logging.info(f"Taking instance doc input from {args.input.name}") + instance_doc = json.loads(args.input.read()) + args.input.close() + + errors = v.validate(instance_doc) + for error in errors: + if args.printerror == 2: + logging.error(error) + elif args.printerror == 1: + logging.error(f"{list(error.path)} - {error.message}") + if len(errors) > 0: + logging.error(f"{len(errors)} errors detected") + else: + logging.info(f"{len(errors)} errors detected") + + if len(errors) > 0: + exit(-1) + else: + exit(0) diff --git a/utils/translate_spec.py b/utils/translate_spec.py index 73b4ab7..ce7a055 100644 --- a/utils/translate_spec.py +++ b/utils/translate_spec.py @@ -10,6 +10,14 @@ from translate import * logging.basicConfig(level = logging.INFO) +json_signature_struct = { + "properties" : { + "protected" : { "type" : "string" }, + "signature" : { "type" : "string" } + }, + "required" : ["protected", "signature" ] +} + def build_schema_locations (paths): schema_locations = [] for schemaFile in paths: @@ -70,14 +78,17 @@ if __name__ == "__main__": json_schemas = {} for schema_tuple in schema_locations: logging.info(f" Translating {schema_tuple}") - if ns_map[schema_tuple[0]].get('skip'): - logging.info(f" Skipping {schema_tuple[0]} due to config...") + if 'skip' in ns_map[schema_tuple[0]]: + logging.info(f" Skipping {schema_tuple[0]}...") continue js = translate_schema(schema_tuple[1], ns_map, schema_locations) - # TODO - Special case, get rid of signature - if ns_map[schema_tuple[0]] == 'core.json': - js['$defs']['HI1Message']['properties'].pop('Signature') + # TODO - Special case, get rid of XML Dsig signature and insert JSON signature + if schema_tuple[0] == 'http://uri.etsi.org/03120/common/2019/10/Core': + logging.info ("Modifying signature elements") + js['$defs']['HI1Message']['properties'].pop('xmldsig:Signature') + js['$defs']['HI1Message']['properties']['Signature'] = json_signature_struct + js_path = output_path / convert_xsd_to_filename(schema_tuple[1]) # TODO - Special case - abstract HI1Object -- GitLab From 5e55f1b394d0adc48474aa8847cdc8ee57d83cd1 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 10 Oct 2023 09:36:20 +0100 Subject: [PATCH 7/7] Backing out signature changes --- 103120/schema/json/ts_103120_Core.schema.json | 15 +-------------- utils/translate_spec.py | 1 - 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/103120/schema/json/ts_103120_Core.schema.json b/103120/schema/json/ts_103120_Core.schema.json index 63e2ad4..bc75842 100644 --- a/103120/schema/json/ts_103120_Core.schema.json +++ b/103120/schema/json/ts_103120_Core.schema.json @@ -13,20 +13,7 @@ "Payload": { "$ref": "#/$defs/MessagePayload" }, - "Signature": { - "properties": { - "protected": { - "type": "string" - }, - "signature": { - "type": "string" - } - }, - "required": [ - "protected", - "signature" - ] - } + "Signature": {} }, "required": [ "Header", diff --git a/utils/translate_spec.py b/utils/translate_spec.py index ce7a055..8ef393f 100644 --- a/utils/translate_spec.py +++ b/utils/translate_spec.py @@ -87,7 +87,6 @@ if __name__ == "__main__": if schema_tuple[0] == 'http://uri.etsi.org/03120/common/2019/10/Core': logging.info ("Modifying signature elements") js['$defs']['HI1Message']['properties'].pop('xmldsig:Signature') - js['$defs']['HI1Message']['properties']['Signature'] = json_signature_struct js_path = output_path / convert_xsd_to_filename(schema_tuple[1]) -- GitLab