From 0071f4d0769a2bcc54f9b9ad44c5e4b99a696c73 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 31 Aug 2023 10:16:32 +0100 Subject: [PATCH 1/5] Adding signing/verification code and removing temp files --- utils/sign_json.py | 36 ++++++++++++++++++++++++++ utils/verify_json.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 utils/sign_json.py create mode 100644 utils/verify_json.py diff --git a/utils/sign_json.py b/utils/sign_json.py new file mode 100644 index 0000000..5203995 --- /dev/null +++ b/utils/sign_json.py @@ -0,0 +1,36 @@ + +import argparse +from jose import jws +from pathlib import Path + +import json + + +def insert_sig_block (j): + j['signature'] = { + 'protected_header' : '', + 'signature' : '' + } + return j + +if __name__ == "__main__": + json_path = Path("103120/examples/json/request1.json") + json_text = json_path.read_text() + + j = json.loads(json_text) + j = insert_sig_block(j) + + presigned_json_text = json.dumps(j) + Path('presigned.json').write_text(presigned_json_text) + presigned_json_bytes = presigned_json_text.encode('utf-8') + + signed = jws.sign(presigned_json_bytes, 'secret_key', algorithm="HS256") + components = signed.split('.') + + j['signature']['protected_header'] = components[0] + j['signature']['signature'] = components[2] + + signed_json_text = json.dumps(j) + print(signed_json_text) + + diff --git a/utils/verify_json.py b/utils/verify_json.py new file mode 100644 index 0000000..eca6d57 --- /dev/null +++ b/utils/verify_json.py @@ -0,0 +1,60 @@ + +import argparse +import sys +import logging +import base64 +from jose import jws +from pathlib import Path + +import json + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + signed_json_text = args.input.read() + args.input.close() + + j = json.loads(signed_json_text) + + protected_header = j['signature']['protected_header'] + signature = j['signature']['signature'] + + # TODO some safety checks needed here + + # Remove the newline that appears from the console + if signed_json_text.endswith('\n'): signed_json_text = signed_json_text[:-1] + signed_json_text = signed_json_text.replace(protected_header, "").replace(signature, "") + + print ("\n\nPayload for verification ================================") + print(signed_json_text) + + payload_bytes = signed_json_text.encode('utf-8') + payload_token = base64.b64encode(payload_bytes).decode('ascii') + + # Un-pad the token, as per RFC7515 annex C + payload_token = payload_token.split('=')[0] + payload_token = payload_token.replace('+','-') + payload_token = payload_token.replace('/','_') + + print ("Payload bytes:", payload_bytes) + print ("Payload token:", payload_token) + + token = protected_header + "." + payload_token + "." + signature + result = jws.verify(token, key="secret_key", algorithms=['HS256']) + + print("Signature verified") + -- GitLab From 6d851b39e0081848abd2e156d6508a049eb1f3a5 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 31 Aug 2023 10:18:38 +0100 Subject: [PATCH 2/5] Signing code now takes text from stdin --- presigned.json | 1 + utils/sign_json.py | 21 +++++++++++++++++++-- utils/verify_json.py | 6 ------ 3 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 presigned.json diff --git a/presigned.json b/presigned.json new file mode 100644 index 0000000..3cd11e8 --- /dev/null +++ b/presigned.json @@ -0,0 +1 @@ +{"@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", "Header": {"SenderIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR01"}, "ReceiverIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR02"}, "TransactionIdentifier": "c02358b2-76cf-4ba4-a8eb-f6436ccaea2e", "Timestamp": "2015-09-01T12:00:00.000000Z", "Version": {"ETSIVersion": "V1.13.1", "NationalProfileOwner": "XX", "NationalProfileVersion": "v1.0"}}, "Payload": {"RequestPayload": {"ActionRequests": {"ActionRequest": [{"ActionIdentifier": 0, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Authorisation}AuthorisationObject", "ObjectIdentifier": "7dbbc880-8750-4d3c-abe7-ea4a17646045", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "auth:AuthorisationReference": "W000001", "auth:AuthorisationTimespan": {"auth:StartTime": "2015-09-01T12:00:00Z", "auth:EndTime": "2015-12-01T12:00:00Z"}}}}, {"ActionIdentifier": 1, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Task}LITaskObject", "ObjectIdentifier": "2b36a78b-b628-416d-bd22-404e68a0cd36", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "AssociatedObjects": {"AssociatedObject": ["7dbbc880-8750-4d3c-abe7-ea4a17646045"]}, "task:Reference": "LIID1", "task:TargetIdentifier": {"task:TargetIdentifierValues": {"task:TargetIdentifierValue": [{"task:FormatType": {"task:FormatOwner": "ETSI", "task:FormatName": "InternationalE164"}, "task:Value": "442079460223"}]}}, "task:DeliveryType": {"common:Owner": "ETSI", "common:Name": "TaskDeliveryType", "common:Value": "IRIandCC"}, "task:DeliveryDetails": {"task:DeliveryDestination": [{"task:DeliveryAddress": {"task:IPv4Address": "192.0.2.0"}}]}, "task:CSPID": {"CountryCode": "XX", "UniqueIdentifier": "RECVER01"}}}}]}}}, "signature": {"protected_header": "", "signature": ""}} \ No newline at end of file diff --git a/utils/sign_json.py b/utils/sign_json.py index 5203995..4c1a3c0 100644 --- a/utils/sign_json.py +++ b/utils/sign_json.py @@ -1,5 +1,7 @@ import argparse +import logging +import sys from jose import jws from pathlib import Path @@ -14,8 +16,23 @@ def insert_sig_block (j): return j if __name__ == "__main__": - json_path = Path("103120/examples/json/request1.json") - json_text = json_path.read_text() + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + json_text = args.input.read() + args.input.close() j = json.loads(json_text) j = insert_sig_block(j) diff --git a/utils/verify_json.py b/utils/verify_json.py index eca6d57..28f2c39 100644 --- a/utils/verify_json.py +++ b/utils/verify_json.py @@ -39,9 +39,6 @@ if __name__ == "__main__": if signed_json_text.endswith('\n'): signed_json_text = signed_json_text[:-1] signed_json_text = signed_json_text.replace(protected_header, "").replace(signature, "") - print ("\n\nPayload for verification ================================") - print(signed_json_text) - payload_bytes = signed_json_text.encode('utf-8') payload_token = base64.b64encode(payload_bytes).decode('ascii') @@ -50,9 +47,6 @@ if __name__ == "__main__": payload_token = payload_token.replace('+','-') payload_token = payload_token.replace('/','_') - print ("Payload bytes:", payload_bytes) - print ("Payload token:", payload_token) - token = protected_header + "." + payload_token + "." + signature result = jws.verify(token, key="secret_key", algorithms=['HS256']) -- GitLab From b8d1af6a5200c2be770844b31176b2098f953859 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 12 Sep 2023 15:24:42 +0100 Subject: [PATCH 3/5] Removing presigned file --- presigned.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 presigned.json diff --git a/presigned.json b/presigned.json deleted file mode 100644 index 3cd11e8..0000000 --- a/presigned.json +++ /dev/null @@ -1 +0,0 @@ -{"@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", "Header": {"SenderIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR01"}, "ReceiverIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR02"}, "TransactionIdentifier": "c02358b2-76cf-4ba4-a8eb-f6436ccaea2e", "Timestamp": "2015-09-01T12:00:00.000000Z", "Version": {"ETSIVersion": "V1.13.1", "NationalProfileOwner": "XX", "NationalProfileVersion": "v1.0"}}, "Payload": {"RequestPayload": {"ActionRequests": {"ActionRequest": [{"ActionIdentifier": 0, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Authorisation}AuthorisationObject", "ObjectIdentifier": "7dbbc880-8750-4d3c-abe7-ea4a17646045", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "auth:AuthorisationReference": "W000001", "auth:AuthorisationTimespan": {"auth:StartTime": "2015-09-01T12:00:00Z", "auth:EndTime": "2015-12-01T12:00:00Z"}}}}, {"ActionIdentifier": 1, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Task}LITaskObject", "ObjectIdentifier": "2b36a78b-b628-416d-bd22-404e68a0cd36", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "AssociatedObjects": {"AssociatedObject": ["7dbbc880-8750-4d3c-abe7-ea4a17646045"]}, "task:Reference": "LIID1", "task:TargetIdentifier": {"task:TargetIdentifierValues": {"task:TargetIdentifierValue": [{"task:FormatType": {"task:FormatOwner": "ETSI", "task:FormatName": "InternationalE164"}, "task:Value": "442079460223"}]}}, "task:DeliveryType": {"common:Owner": "ETSI", "common:Name": "TaskDeliveryType", "common:Value": "IRIandCC"}, "task:DeliveryDetails": {"task:DeliveryDestination": [{"task:DeliveryAddress": {"task:IPv4Address": "192.0.2.0"}}]}, "task:CSPID": {"CountryCode": "XX", "UniqueIdentifier": "RECVER01"}}}}]}}}, "signature": {"protected_header": "", "signature": ""}} \ No newline at end of file -- GitLab From 1d03a7afc55e24607151bc23947057a880e1242f Mon Sep 17 00:00:00 2001 From: mark Date: Wed, 27 Sep 2023 15:12:38 +0100 Subject: [PATCH 4/5] Altering example --- presigned.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 presigned.json diff --git a/presigned.json b/presigned.json new file mode 100644 index 0000000..3cd11e8 --- /dev/null +++ b/presigned.json @@ -0,0 +1 @@ +{"@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", "Header": {"SenderIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR01"}, "ReceiverIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR02"}, "TransactionIdentifier": "c02358b2-76cf-4ba4-a8eb-f6436ccaea2e", "Timestamp": "2015-09-01T12:00:00.000000Z", "Version": {"ETSIVersion": "V1.13.1", "NationalProfileOwner": "XX", "NationalProfileVersion": "v1.0"}}, "Payload": {"RequestPayload": {"ActionRequests": {"ActionRequest": [{"ActionIdentifier": 0, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Authorisation}AuthorisationObject", "ObjectIdentifier": "7dbbc880-8750-4d3c-abe7-ea4a17646045", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "auth:AuthorisationReference": "W000001", "auth:AuthorisationTimespan": {"auth:StartTime": "2015-09-01T12:00:00Z", "auth:EndTime": "2015-12-01T12:00:00Z"}}}}, {"ActionIdentifier": 1, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Task}LITaskObject", "ObjectIdentifier": "2b36a78b-b628-416d-bd22-404e68a0cd36", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "AssociatedObjects": {"AssociatedObject": ["7dbbc880-8750-4d3c-abe7-ea4a17646045"]}, "task:Reference": "LIID1", "task:TargetIdentifier": {"task:TargetIdentifierValues": {"task:TargetIdentifierValue": [{"task:FormatType": {"task:FormatOwner": "ETSI", "task:FormatName": "InternationalE164"}, "task:Value": "442079460223"}]}}, "task:DeliveryType": {"common:Owner": "ETSI", "common:Name": "TaskDeliveryType", "common:Value": "IRIandCC"}, "task:DeliveryDetails": {"task:DeliveryDestination": [{"task:DeliveryAddress": {"task:IPv4Address": "192.0.2.0"}}]}, "task:CSPID": {"CountryCode": "XX", "UniqueIdentifier": "RECVER01"}}}}]}}}, "signature": {"protected_header": "", "signature": ""}} \ No newline at end of file -- GitLab From 141b7f7670dc39d340cd29344a362d41c4d13e16 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 6 Oct 2023 10:18:12 +0100 Subject: [PATCH 5/5] Updating verification code and schema translation --- 103120/examples/json/request1_signed.json | 100 +++++++++++ 103120/schema/json/ts_103120_Core.schema.json | 16 +- presigned.json | 101 +++++++++++- utils/json_validator.py | 156 ++++++++++++++++++ utils/sign_json.py | 16 +- utils/translate_spec.py | 22 ++- utils/validate_json.py | 122 -------------- utils/verify_json.py | 4 +- 8 files changed, 397 insertions(+), 140 deletions(-) create mode 100644 103120/examples/json/request1_signed.json create mode 100644 utils/json_validator.py delete mode 100644 utils/validate_json.py diff --git a/103120/examples/json/request1_signed.json b/103120/examples/json/request1_signed.json new file mode 100644 index 0000000..4339ce3 --- /dev/null +++ b/103120/examples/json/request1_signed.json @@ -0,0 +1,100 @@ +{ + "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", + "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", + "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", + "Header": { + "SenderIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR01" + }, + "ReceiverIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR02" + }, + "TransactionIdentifier": "c02358b2-76cf-4ba4-a8eb-f6436ccaea2e", + "Timestamp": "2015-09-01T12:00:00.000000Z", + "Version": { + "ETSIVersion": "V1.13.1", + "NationalProfileOwner": "XX", + "NationalProfileVersion": "v1.0" + } + }, + "Payload": { + "RequestPayload": { + "ActionRequests": { + "ActionRequest": [ + { + "ActionIdentifier": 0, + "CREATE": { + "HI1Object": { + "@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Authorisation}AuthorisationObject", + "ObjectIdentifier": "7dbbc880-8750-4d3c-abe7-ea4a17646045", + "CountryCode": "XX", + "OwnerIdentifier": "ACTOR01", + "auth:AuthorisationReference": "W000001", + "auth:AuthorisationTimespan": { + "auth:StartTime": "2015-09-01T12:00:00Z", + "auth:EndTime": "2015-12-01T12:00:00Z" + } + } + } + }, + { + "ActionIdentifier": 1, + "CREATE": { + "HI1Object": { + "@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Task}LITaskObject", + "ObjectIdentifier": "2b36a78b-b628-416d-bd22-404e68a0cd36", + "CountryCode": "XX", + "OwnerIdentifier": "ACTOR01", + "AssociatedObjects": { + "AssociatedObject": [ + "7dbbc880-8750-4d3c-abe7-ea4a17646045" + ] + }, + "task:Reference": "LIID1", + "task:TargetIdentifier": { + "task:TargetIdentifierValues": { + "task:TargetIdentifierValue": [ + { + "task:FormatType": { + "task:FormatOwner": "ETSI", + "task:FormatName": "InternationalE164" + }, + "task:Value": "442079460223" + } + ] + } + }, + "task:DeliveryType": { + "common:Owner": "ETSI", + "common:Name": "TaskDeliveryType", + "common:Value": "IRIandCC" + }, + "task:DeliveryDetails": { + "task:DeliveryDestination": [ + { + "task:DeliveryAddress": { + "task:IPv4Address": "192.0.2.0" + } + } + ] + }, + "task:CSPID": { + "CountryCode": "XX", + "UniqueIdentifier": "RECVER01" + } + } + } + } + ] + } + } + }, + "Signature": { + "protected": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + "signature": "RImkRSJkh46537Bh4LpNbkL2O64jInUv0JLGeoKJ-2M" + } +} diff --git a/103120/schema/json/ts_103120_Core.schema.json b/103120/schema/json/ts_103120_Core.schema.json index 8225cf3..63e2ad4 100644 --- a/103120/schema/json/ts_103120_Core.schema.json +++ b/103120/schema/json/ts_103120_Core.schema.json @@ -13,9 +13,19 @@ "Payload": { "$ref": "#/$defs/MessagePayload" }, - "Signature": {}, - "xmldsig:Signature": { - "$ref": "www.w3.org_2000_09_xmldsig##/$defs/SignatureType" + "Signature": { + "properties": { + "protected": { + "type": "string" + }, + "signature": { + "type": "string" + } + }, + "required": [ + "protected", + "signature" + ] } }, "required": [ diff --git a/presigned.json b/presigned.json index 3cd11e8..16329e0 100644 --- a/presigned.json +++ b/presigned.json @@ -1 +1,100 @@ -{"@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", "Header": {"SenderIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR01"}, "ReceiverIdentifier": {"CountryCode": "XX", "UniqueIdentifier": "ACTOR02"}, "TransactionIdentifier": "c02358b2-76cf-4ba4-a8eb-f6436ccaea2e", "Timestamp": "2015-09-01T12:00:00.000000Z", "Version": {"ETSIVersion": "V1.13.1", "NationalProfileOwner": "XX", "NationalProfileVersion": "v1.0"}}, "Payload": {"RequestPayload": {"ActionRequests": {"ActionRequest": [{"ActionIdentifier": 0, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Authorisation}AuthorisationObject", "ObjectIdentifier": "7dbbc880-8750-4d3c-abe7-ea4a17646045", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "auth:AuthorisationReference": "W000001", "auth:AuthorisationTimespan": {"auth:StartTime": "2015-09-01T12:00:00Z", "auth:EndTime": "2015-12-01T12:00:00Z"}}}}, {"ActionIdentifier": 1, "CREATE": {"HI1Object": {"@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Task}LITaskObject", "ObjectIdentifier": "2b36a78b-b628-416d-bd22-404e68a0cd36", "CountryCode": "XX", "OwnerIdentifier": "ACTOR01", "AssociatedObjects": {"AssociatedObject": ["7dbbc880-8750-4d3c-abe7-ea4a17646045"]}, "task:Reference": "LIID1", "task:TargetIdentifier": {"task:TargetIdentifierValues": {"task:TargetIdentifierValue": [{"task:FormatType": {"task:FormatOwner": "ETSI", "task:FormatName": "InternationalE164"}, "task:Value": "442079460223"}]}}, "task:DeliveryType": {"common:Owner": "ETSI", "common:Name": "TaskDeliveryType", "common:Value": "IRIandCC"}, "task:DeliveryDetails": {"task:DeliveryDestination": [{"task:DeliveryAddress": {"task:IPv4Address": "192.0.2.0"}}]}, "task:CSPID": {"CountryCode": "XX", "UniqueIdentifier": "RECVER01"}}}}]}}}, "signature": {"protected_header": "", "signature": ""}} \ No newline at end of file +{ + "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", + "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", + "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", + "Header": { + "SenderIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR01" + }, + "ReceiverIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR02" + }, + "TransactionIdentifier": "c02358b2-76cf-4ba4-a8eb-f6436ccaea2e", + "Timestamp": "2015-09-01T12:00:00.000000Z", + "Version": { + "ETSIVersion": "V1.13.1", + "NationalProfileOwner": "XX", + "NationalProfileVersion": "v1.0" + } + }, + "Payload": { + "RequestPayload": { + "ActionRequests": { + "ActionRequest": [ + { + "ActionIdentifier": 0, + "CREATE": { + "HI1Object": { + "@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Authorisation}AuthorisationObject", + "ObjectIdentifier": "7dbbc880-8750-4d3c-abe7-ea4a17646045", + "CountryCode": "XX", + "OwnerIdentifier": "ACTOR01", + "auth:AuthorisationReference": "W000001", + "auth:AuthorisationTimespan": { + "auth:StartTime": "2015-09-01T12:00:00Z", + "auth:EndTime": "2015-12-01T12:00:00Z" + } + } + } + }, + { + "ActionIdentifier": 1, + "CREATE": { + "HI1Object": { + "@xsi:type": "{http://uri.etsi.org/03120/common/2020/09/Task}LITaskObject", + "ObjectIdentifier": "2b36a78b-b628-416d-bd22-404e68a0cd36", + "CountryCode": "XX", + "OwnerIdentifier": "ACTOR01", + "AssociatedObjects": { + "AssociatedObject": [ + "7dbbc880-8750-4d3c-abe7-ea4a17646045" + ] + }, + "task:Reference": "LIID1", + "task:TargetIdentifier": { + "task:TargetIdentifierValues": { + "task:TargetIdentifierValue": [ + { + "task:FormatType": { + "task:FormatOwner": "ETSI", + "task:FormatName": "InternationalE164" + }, + "task:Value": "442079460223" + } + ] + } + }, + "task:DeliveryType": { + "common:Owner": "ETSI", + "common:Name": "TaskDeliveryType", + "common:Value": "IRIandCC" + }, + "task:DeliveryDetails": { + "task:DeliveryDestination": [ + { + "task:DeliveryAddress": { + "task:IPv4Address": "192.0.2.0" + } + } + ] + }, + "task:CSPID": { + "CountryCode": "XX", + "UniqueIdentifier": "RECVER01" + } + } + } + } + ] + } + } + }, + "Signature": { + "protected": "", + "signature": "" + } +} \ No newline at end of file diff --git a/utils/json_validator.py b/utils/json_validator.py new file mode 100644 index 0000000..fb46193 --- /dev/null +++ b/utils/json_validator.py @@ -0,0 +1,156 @@ +import sys +from jsonschema import validate, RefResolver, Draft202012Validator +from jsonschema.exceptions import ValidationError +import json +from pathlib import Path +import logging +import argparse +from itertools import chain + +class JsonValidator: + def __init__(self, core_schema: str, other_schemas : dict): + self._core_schema = json.load(Path(core_schema).open()) + self._schema_dict = { self._core_schema['$id'] : self._core_schema } + self._supporting_paths = [] + for thing in other_schemas: + path = Path(thing) + if path.is_dir(): + logging.debug(f"Searching {path} for schema files") + self._supporting_paths.extend(path.rglob("*.schema.json")) + else: + logging.debug(f"Appending {path} as schema file") + self._supporting_paths.append(path) + logging.info(f"Supporting schema paths: {self._supporting_paths}") + self._supporting_schemas = [json.load(p.open()) for p in self._supporting_paths] + self._schema_dict = self._schema_dict | { s['$id'] : s for s in self._supporting_schemas } + logging.info(f"Loaded schema IDs: {[k for k in self._schema_dict.keys()]}") + self._resolver = RefResolver(None, + referrer=None, + store=self._schema_dict) + logging.info("Created RefResolver") + self._validator = Draft202012Validator(self._core_schema, resolver=self._resolver) + logging.info("Created validator") + + def validate(self, instance_doc: str): + errors = list(self._validator.iter_errors(instance_doc)) + return errors + +class TS103120Validator (JsonValidator): + def __init__ (self, path_to_repo): + repo_path = Path(path_to_repo) + schema_dirs = [str(repo_path / "103120/schema/json"), str("103280/")] + core_schema = str(repo_path / "103120/schema/json/ts_103120_Core.schema.json") + JsonValidator.__init__(self, core_schema, schema_dirs) + request_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/RequestPayload" } + self._request_fragment_validator = Draft202012Validator(request_fragment_schema, resolver=self._resolver) + response_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/ResponsePayload" } + self._response_fragment_validator = Draft202012Validator(response_fragment_schema, resolver=self._resolver) + + def expand_request_response_exception (self, ex): + if list(ex.schema_path) == ['properties', 'Payload', 'oneOf']: + logging.info ("Error detected validating payload oneOf - attempting explicit validation...") + if 'RequestPayload' in instance_doc['Payload'].keys(): + ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['RequestPayload'])])) + for r in ret_list: + r.path = ex.path + r.path + return ret_list + elif 'ResponsePayload' in instance_doc['Payload'].keys(): + ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['ResponsePayload'])])) + for r in ret_list: + r.path = ex.path + r.path + return ret_list + else: + logging.error("No RequestPayload or ResponsePayload found - is the Payload malformed?") + return [ex] + else: + return [ex] + + def expand_action_exception (self, ex): + logging.error("Error detected in ActionRequests/ActionResponses") + error_path = list(ex.schema_path) + if error_path != ['properties', 'ActionRequests', 'properties', 'ActionRequest', 'items', 'allOf', 1, 'oneOf'] and error_path != ['properties', 'ActionResponses', 'properties', 'ActionResponse', 'items', 'allOf', 1, 'oneOf']: + logging.error("Error not in inner Request/Response allOf/oneOf constraint") + return[ex] + j = ex.instance + j.pop('ActionIdentifier') # Remove ActionIdentifier - one remaining key will be the verb + verb = list(j.keys())[0] + message = "Request" if error_path[1] == "ActionRequests" else "Response" + v = Draft202012Validator({"$ref" : f"ts_103120_Core_2019_10#/$defs/{verb}{message}"}, resolver=self._resolver) + ret_list = list(chain(*[self.expand_object_exception(x) for x in v.iter_errors(j[verb])])) + for r in ret_list: + r.path = ex.path + r.path + return ret_list + + def expand_object_exception (self, ex): + logging.error("Error detected in verb") + # The final level of validation is for the actual HI1Object validation + if list(ex.schema_path) != ['properties', 'HI1Object', 'oneOf']: + logging.error("Error not inside HI1Object") + return [ex] + object_type = ex.instance['@xsi:type'].split('}')[-1] + object_ref = { + 'AuthorisationObject': 'ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject', + 'LITaskObject': 'ts_103120_Task_2020_09#/$defs/LITaskObject', + 'LDTaskObject': 'ts_103120_Task_2020_09#/$defs/LDTaskObject', + 'LPTaskObject': 'ts_103120_Task_2020_09#/$defs/LPTaskObject', + 'DocumentObject': 'ts_103120_Document_2020_09#/$defs/DocumentObject', + 'NotificationObject': 'ts_103120_Notification_2016_02#/$defs/NotificationObject', + 'DeliveryObject': 'ts_103120_Delivery_2019_10#/$defs/DeliveryObject', + 'TrafficPolicyObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject', + 'TrafficRuleObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject', + }[object_type] + v = Draft202012Validator({"$ref" : object_ref}, resolver=self._resolver) + return list(v.iter_errors(ex.instance)) + + def validate(self, instance_doc: str): + errors = JsonValidator.validate(self, instance_doc) + out_errors = list(chain(*[self.expand_request_response_exception(ex) for ex in errors])) + return out_errors + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") + parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument('--ts103120', action="store_true", help="Validate a TS 103 120 JSON document") + parser.add_argument('--schema', default=None, help="Primary schema to validate against") + parser.add_argument('-p', '--printerror', action="count", help="Controls how verbose validation error printing is (can be specified multiple times)") + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + if (args.ts103120): + v = TS103120Validator("./") + else: + v = JsonValidator(args.schema, args.schemadir) + + logging.info(f"Taking instance doc input from {args.input.name}") + instance_doc = json.loads(args.input.read()) + args.input.close() + + errors = v.validate(instance_doc) + for error in errors: + if args.printerror == 2: + logging.error(error) + elif args.printerror == 1: + logging.error(f"{list(error.path)} - {error.message}") + if len(errors) > 0: + logging.error(f"{len(errors)} errors detected") + else: + logging.info(f"{len(errors)} errors detected") + + if len(errors) > 0: + exit(-1) + else: + exit(0) diff --git a/utils/sign_json.py b/utils/sign_json.py index 4c1a3c0..1ce0bba 100644 --- a/utils/sign_json.py +++ b/utils/sign_json.py @@ -9,8 +9,8 @@ import json def insert_sig_block (j): - j['signature'] = { - 'protected_header' : '', + j['Signature'] = { + 'protected' : '', 'signature' : '' } return j @@ -18,6 +18,7 @@ def insert_sig_block (j): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') + parser.add_argument('--pretty', action="store_true", help='Pretty-print the JSON document before signing') parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") args = parser.parse_args() @@ -37,17 +38,20 @@ if __name__ == "__main__": j = json.loads(json_text) j = insert_sig_block(j) - presigned_json_text = json.dumps(j) + indent = None + if args.pretty: + indent = ' ' + presigned_json_text = json.dumps(j, indent=indent) Path('presigned.json').write_text(presigned_json_text) presigned_json_bytes = presigned_json_text.encode('utf-8') signed = jws.sign(presigned_json_bytes, 'secret_key', algorithm="HS256") components = signed.split('.') - j['signature']['protected_header'] = components[0] - j['signature']['signature'] = components[2] + j['Signature']['protected'] = components[0] + j['Signature']['signature'] = components[2] - signed_json_text = json.dumps(j) + signed_json_text = json.dumps(j, indent=indent) print(signed_json_text) diff --git a/utils/translate_spec.py b/utils/translate_spec.py index eac24e4..ce7a055 100644 --- a/utils/translate_spec.py +++ b/utils/translate_spec.py @@ -10,6 +10,14 @@ from translate import * logging.basicConfig(level = logging.INFO) +json_signature_struct = { + "properties" : { + "protected" : { "type" : "string" }, + "signature" : { "type" : "string" } + }, + "required" : ["protected", "signature" ] +} + def build_schema_locations (paths): schema_locations = [] for schemaFile in paths: @@ -70,15 +78,17 @@ if __name__ == "__main__": json_schemas = {} for schema_tuple in schema_locations: logging.info(f" Translating {schema_tuple}") - if 'xmldsig' in (schema_tuple[1]): - # TODO - work out what to do here - logging.info(" Skipping XML Dsig...") + if 'skip' in ns_map[schema_tuple[0]]: + logging.info(f" Skipping {schema_tuple[0]}...") continue js = translate_schema(schema_tuple[1], ns_map, schema_locations) - # TODO - Special case, get rid of signature - if ns_map[schema_tuple[0]] == 'core.json': - js['$defs']['HI1Message']['properties'].pop('Signature') + # TODO - Special case, get rid of XML Dsig signature and insert JSON signature + if schema_tuple[0] == 'http://uri.etsi.org/03120/common/2019/10/Core': + logging.info ("Modifying signature elements") + js['$defs']['HI1Message']['properties'].pop('xmldsig:Signature') + js['$defs']['HI1Message']['properties']['Signature'] = json_signature_struct + js_path = output_path / convert_xsd_to_filename(schema_tuple[1]) # TODO - Special case - abstract HI1Object diff --git a/utils/validate_json.py b/utils/validate_json.py deleted file mode 100644 index 43f460b..0000000 --- a/utils/validate_json.py +++ /dev/null @@ -1,122 +0,0 @@ -import sys -from jsonschema import validate, RefResolver, Draft202012Validator -from jsonschema.exceptions import ValidationError -import json -from pathlib import Path -import logging -import argparse - - -def handle_uri(u): - print(u) - -def load_json(path : str): - with open(path) as f: - return json.load(f) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") - parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") - parser.add_argument('schema', help="Primary schema to validate against") - - args = parser.parse_args() - - match args.verbose: - case v if v and v >= 2: - logging.basicConfig(level=logging.DEBUG) - case 1: - logging.basicConfig(level=logging.INFO) - case _: - logging.basicConfig(level=logging.WARNING) - - logging.debug(f"Arguments: {args}") - - instance_doc = json.loads(args.input.read()) - args.input.close() - main_schema = load_json(args.schema) - schema_dict = { main_schema['$id'] : main_schema } - - if args.schemadir: - schema_paths = [] - for d in args.schemadir: - logging.info(f"Searching {d}") - logging.info(list(Path(d).rglob("*.schema.json"))) - schema_paths += [f for f in Path(d).rglob("*.schema.json")] - logging.info(f"Schema files loaded: {schema_paths}") - - schemas_json = [json.load(p.open()) for p in schema_paths] - schema_dict = schema_dict | { s['$id'] : s for s in schemas_json } - - logging.info(f"Schema IDs loaded: {[k for k in schema_dict.keys()]}") - - logging.debug (f"Instance doc: {instance_doc}") - logging.debug (f"Main schema: {main_schema}") - - resolver = RefResolver(None, - referrer=None, - store=schema_dict) - - v = Draft202012Validator(main_schema, resolver=resolver) - try: - v.validate(instance_doc) - except ValidationError as ex: - # Any failure within the Payload element results in a failure against the oneOf constraint in the Payload element - # This isn't terribly helpful in working out what is actually wrong, so in this case we attempt an explicit - # validation against the relevant oneOf alternation to try and get a more useful validation error - if list(ex.schema_path) == ['properties', 'Payload', 'oneOf']: - logging.error ("Error detected validating payload oneOf - attempting explicit validation...") - try: - if 'RequestPayload' in instance_doc['Payload'].keys(): - request_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/RequestPayload" } - v = Draft202012Validator(request_fragment_schema, resolver=resolver) - v.validate(instance_doc['Payload']['RequestPayload']) - elif 'ResponsePayload' in instance_doc['Payload'].keys(): - request_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/ResponsePayload" } - v = Draft202012Validator(request_fragment_schema, resolver=resolver) - v.validate(instance_doc['Payload']['ResponsePayload']) - else: - logging.error("No RequestPayload or ResponsePayload found - is the Payload malformed?") - raise ex - except ValidationError as ex2: - # Similar to above, this is inner validation to try and get a more useful error in the event - # that something fails the verb oneOf constraint - logging.error("Error detected in ActionRequests/ActionResponses") - error_path = list(ex2.schema_path) - if error_path != ['properties', 'ActionRequests', 'properties', 'ActionRequest', 'items', 'allOf', 1, 'oneOf'] and error_path != ['properties', 'ActionResponses', 'properties', 'ActionResponse', 'items', 'allOf', 1, 'oneOf']: - logging.error("Error not in inner Request/Response allOf/oneOf constraint") - raise ex2 - j = ex2.instance - j.pop('ActionIdentifier') # Remove ActionIdentifier - one remaining key will be the verb - verb = list(j.keys())[0] - message = "Request" if error_path[1] == "ActionRequests" else "Response" - v = Draft202012Validator({"$ref" : f"ts_103120_Core_2019_10#/$defs/{verb}{message}"}, resolver=resolver) - try: - v.validate(j[verb]) - except ValidationError as ex3: - logging.error("Error detected in verb") - # The final level of validation is for the actual HI1Object validation - if list(ex3.schema_path) != ['properties', 'HI1Object', 'oneOf']: - logging.error("Error not inside HI1Object") - raise ex3 - object_type = ex3.instance['@xsi:type'].split('}')[-1] - object_ref = { - 'AuthorisationObject': 'ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject', - 'LITaskObject': 'ts_103120_Task_2020_09#/$defs/LITaskObject', - 'LDTaskObject': 'ts_103120_Task_2020_09#/$defs/LDTaskObject', - 'LPTaskObject': 'ts_103120_Task_2020_09#/$defs/LPTaskObject', - 'DocumentObject': 'ts_103120_Document_2020_09#/$defs/DocumentObject', - 'NotificationObject': 'ts_103120_Notification_2016_02#/$defs/NotificationObject', - 'DeliveryObject': 'ts_103120_Delivery_2019_10#/$defs/DeliveryObject', - 'TrafficPolicyObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject', - 'TrafficRuleObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject', - }[object_type] - v = Draft202012Validator({"$ref" : object_ref}, resolver=resolver) - v.validate(ex3.instance) - - exit(-1) - - - logging.info("Done") diff --git a/utils/verify_json.py b/utils/verify_json.py index 28f2c39..329c069 100644 --- a/utils/verify_json.py +++ b/utils/verify_json.py @@ -30,8 +30,8 @@ if __name__ == "__main__": j = json.loads(signed_json_text) - protected_header = j['signature']['protected_header'] - signature = j['signature']['signature'] + protected_header = j['Signature']['protected'] + signature = j['Signature']['signature'] # TODO some safety checks needed here -- GitLab