diff --git a/.editorconfig b/.editorconfig index 14735321793ba01562a75ae1e2e509c6b8b07826..cb6f93bdf205892f7301735ef40840093df17697 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,3 +6,8 @@ root = true indent_style = space indent_size = 4 trim_trailing_whitespace = true + +[**.json] +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.gitignore b/.gitignore index 3e3760376b450b5a9b9e6c9c8ff79d2305903cc0..fc05895014b2a3d61b7fe31c6096ba5f0ab514ce 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ .vscode/ .idea/ +testing/dockerfiles/build.log + # Vagrant .vagrant/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4c0fc3081cc8cb383e9fc7c21aa5d73a7adae609..0305e9ef326156f7974ede392c958e6cc5eae89c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,9 +7,14 @@ workflow: stages: - preflight - - check + - compile + - lint - build + +# ----------------------------------------------------------- +# Preflight + preflight: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" stage: preflight @@ -19,19 +24,102 @@ preflight: script: - forgelib-preflight https://$CI_SERVER_HOST $CI_PROJECT_ID $CI_MERGE_REQUEST_IID -process_asn: + +# ----------------------------------------------------------- +# Compile + +compile_asn: image: "forge.etsi.org:5050/li/schemas-definitions/asn1test:latest" - stage: check + stage: compile interruptible: true script: - - python3 testing/asn_process.py + - python3 testing/asn/asn_process.py -process_xsd: - image: "forge.etsi.org:5050/li/schemas-definitions/xsdtest:latest" - stage: check +compile_xsd: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: compile interruptible: true script: - - python3 testing/xsd_process.py + - export PYCHARM_HOSTED=1 + - | + check_count=0 + fail_count=0 + while IFS= read -r -d '' file; do + let "check_count=check_count+1" + if ! forgeschema -c "$file"; then + echo "❌ failed schema checks for $file" + let "fail_count=fail_count+1" + fi + echo "" + echo "..." + echo "" + done < <(find ./testing/xml -type f -name "*.json" -print0) + if [ "$fail_count" -gt 0 ]; then + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "❌ Failed schema checks for $fail_count of $check_count files" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + exit 1 + fi + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "✅ XSD validation OK ($check_count files checked)" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "✅ XSD validation OK for $file" + +compile_json: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: compile + interruptible: true + script: + - export PYCHARM_HOSTED=1 + - | + check_count=0 + fail_count=0 + while IFS= read -r -d '' file; do + let "check_count=check_count+1" + if ! forgeschema -c "$file"; then + echo "❌ failed schema checks for $file" + let "fail_count=fail_count+1" + fi + echo "" + echo "..." + echo "" + done < <(find ./testing/json -type f -name "*.json" -print0) + if [ "$fail_count" -gt 0 ]; then + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "❌ Failed schema checks for $fail_count of $check_count files" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + exit 1 + fi + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "✅ JSON validation OK ($check_count files checked)" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json + + +# ----------------------------------------------------------- +# Lint + +lint_xml: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: lint + interruptible: true + allow_failure: true + script: + - export PYCHARM_HOSTED=1 + - python testing/xml/lint_xml.py + +lint_json: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: lint + interruptible: true + allow_failure: true + script: + - export PYCHARM_HOSTED=1 + - python testing/json/lint_json.py + + +# ----------------------------------------------------------- +# Build generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" diff --git a/103120/examples/json/request5-JSON-Delivery.json b/103120/examples/json/request5-JSON-Delivery.json index 34d84a01608f9b52c2ef1906b034da26b2894603..61ea27680211dd81000f89ad37e57b22b15e9cf8 100644 --- a/103120/examples/json/request5-JSON-Delivery.json +++ b/103120/examples/json/request5-JSON-Delivery.json @@ -44,18 +44,18 @@ "delivery:LastSequence": true, "delivery:Manifest": { "delivery:ExternalSchema": { - "delivery:ManifestID" : "ExampleJSONSchema", - "delivery:ManifestContents" : { - "delivery:JSONSchema" : { - "schema_goes_here" : "schema_goes_here" + "delivery:ManifestID": "ExampleJSONSchema", + "delivery:ManifestContents": { + "delivery:JSONSchema": { + "schema_goes_here": "schema_goes_here" } } } }, "delivery:Delivery": { "delivery:JSONData": { - "field1" : "this is native JSON embedded data", - "field2" : 1234 + "field1": "this is native JSON embedded data", + "field2": 1234 } } } diff --git a/103120/examples/json/request_ee_config.json b/103120/examples/json/request_ee_config.json index f1363f3e90c3ea898767fed3fd15fbc24740e825..80d46a70007a3f2b5354614eb19b2599025072d3 100644 --- a/103120/examples/json/request_ee_config.json +++ b/103120/examples/json/request_ee_config.json @@ -1,37 +1,36 @@ { - "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", - "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", - "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", - "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", - "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", - "Header": { - "SenderIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR01" - }, - "ReceiverIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR02" - }, - "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", - "Timestamp": "2025-05-14T15:21:00.000000Z", - "Version": { - "ETSIVersion": "V1.18.1", - "NationalProfileOwner": "XX", - "NationalProfileVersion": "v1.0" - } + "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", + "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", + "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", + "Header": { + "SenderIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR01" + }, + "ReceiverIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR02" }, - "Payload": { - "RequestPayload": { - "ActionRequests": { - "ActionRequest": [ - { - "ActionIdentifier": 0, - "GETCSPCONFIG": {} - } - ] - } + "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", + "Timestamp": "2025-05-14T15:21:00.000000Z", + "Version": { + "ETSIVersion": "V1.18.1", + "NationalProfileOwner": "XX", + "NationalProfileVersion": "v1.0" + } + }, + "Payload": { + "RequestPayload": { + "ActionRequests": { + "ActionRequest": [ + { + "ActionIdentifier": 0, + "GETCSPCONFIG": {} + } + ] } } } - \ No newline at end of file +} diff --git a/103120/examples/json/response_ee_config.json b/103120/examples/json/response_ee_config.json index 3024a39c482d899241dc34bb3814c0df3bbfd4a0..4ea6e8c85437aac5f7cb6692613961c5c79ad9de 100644 --- a/103120/examples/json/response_ee_config.json +++ b/103120/examples/json/response_ee_config.json @@ -1,210 +1,209 @@ { - "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", - "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", - "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", - "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", - "@xmlns:config": "http://uri.etsi.org/03120/common/2024/06/Config", - "@xmlns:dictionaries": "http://uri.etsi.org/03280/common/2019/10/Dictionaries", - "Header": { - "SenderIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR01" - }, - "ReceiverIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR02" - }, - "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", - "Timestamp": "2025-05-14T15:21:03.000000Z", - "Version": { - "ETSIVersion": "V1.18.1", - "NationalProfileOwner": "XX", - "NationalProfileVersion": "v1.0" - } + "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", + "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", + "@xmlns:config": "http://uri.etsi.org/03120/common/2024/06/Config", + "@xmlns:dictionaries": "http://uri.etsi.org/03280/common/2019/10/Dictionaries", + "Header": { + "SenderIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR01" }, - "Payload": { - "ResponsePayload": { - "ActionResponses": { - "ActionResponse": [ - { - "ActionIdentifier": 0, - "GETCSPCONFIGResponse": { - "LastChanged": "2025-04-03T07:25:21Z", - "Dictionaries": { - "dictionaries:Dictionary": [ - { - "dictionaries:Owner": "ACTOR02", - "dictionaries:Name": "LDRequestSubtype", - "dictionaries:DictionaryEntries": { - "dictionaries:DictionaryEntry": [ - { - "dictionaries:Value": "ServiceA", - "dictionaries:Meaning": "Service A allows users to ..." - }, - { - "dictionaries:Value": "ServiceB", - "dictionaries:Meaning": "Service B allows users to ..." - } - ] - } - }, - { - "dictionaries:Owner": "ACTOR02", - "dictionaries:Name": "LPRequestSubtype", - "dictionaries:DictionaryEntries": { - "dictionaries:DictionaryEntry": [ - { - "dictionaries:Value": "ServiceA", - "dictionaries:Meaning": "Service A allows users to ..." - }, - { - "dictionaries:Value": "ServiceB", - "dictionaries:Meaning": "Service B allows users to ..." - } - ] - } + "ReceiverIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR02" + }, + "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", + "Timestamp": "2025-05-14T15:21:03.000000Z", + "Version": { + "ETSIVersion": "V1.18.1", + "NationalProfileOwner": "XX", + "NationalProfileVersion": "v1.0" + } + }, + "Payload": { + "ResponsePayload": { + "ActionResponses": { + "ActionResponse": [ + { + "ActionIdentifier": 0, + "GETCSPCONFIGResponse": { + "LastChanged": "2025-04-03T07:25:21Z", + "Dictionaries": { + "dictionaries:Dictionary": [ + { + "dictionaries:Owner": "ACTOR02", + "dictionaries:Name": "LDRequestSubtype", + "dictionaries:DictionaryEntries": { + "dictionaries:DictionaryEntry": [ + { + "dictionaries:Value": "ServiceA", + "dictionaries:Meaning": "Service A allows users to ..." + }, + { + "dictionaries:Value": "ServiceB", + "dictionaries:Meaning": "Service B allows users to ..." + } + ] } - ] - }, - "TargetFormatTypeDefinitions": { - "config:FormatOwner": "ACTOR02", - "config:TargetFormatTypeDefinitionEntries": { - "config:TargetFormatTypeDefinitionEntry": [ - { - "config:FormatName": "ProprietaryIdentifier", - "config:Description": "This is an illustration of a proprietary identifier type specific to this CSP, which consists of the string \"CSP\" followed by 10 digits", - "config:FormatRegex": "^CSP[0-9]{10}$" - } - ] - } - }, - "TargetingConfigurations": { - "config:TargetingConfiguration": [ - { - "config:FormatName": "PropietaryIdentifier", - "config:FormatOwner": "ACTOR02", - "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for the propietary identifier format type described in the TargetFormatTypeDefinitions, associated with Service A for the \"Subscriber Data\" category only", - "config:AssociatedLIServiceTypes": {}, - "config:AssociatedLDRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "SubscriberData" - } - ] - }, - "config:AssociatedLDRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LDRequestSubtype", - "common:Value": "ServiceA" - } - ] - }, - "config:AssociatedLPRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "SubscriberData" - } - ] - }, - "config:AssociatedLPRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LPRequestSubtype", - "common:Value": "ServiceA" - } - ] - } - }, - { - "config:FormatName": "InternationalE164", - "config:FormatOwner": "ETSI", - "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for a telephone number (as ETSI standard format E164 number) associated with Service A and Service B for the \"Traffic Data\" and \"Content Data\" categories", - "config:AssociatedLIServiceTypes": {}, - "config:AssociatedLDRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "TrafficData" - }, - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "StoredContentData" - } - ] - }, - "config:AssociatedLDRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LDRequestSubtype", - "common:Value": "ServiceA" - }, - { - "common:Owner": "ACTOR02", - "common:Name": "LDRequestSubtype", - "common:Value": "ServiceB" - } - ] - }, - "config:AssociatedLPRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "TrafficData" - }, - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "StoredContentData" - } - ] - }, - "config:AssociatedLPRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LPRequestSubtype", - "common:Value": "ServiceA" - }, - { - "common:Owner": "ACTOR02", - "common:Name": "LPRequestSubtype", - "common:Value": "ServiceB" - } - ] - } + }, + { + "dictionaries:Owner": "ACTOR02", + "dictionaries:Name": "LPRequestSubtype", + "dictionaries:DictionaryEntries": { + "dictionaries:DictionaryEntry": [ + { + "dictionaries:Value": "ServiceA", + "dictionaries:Meaning": "Service A allows users to ..." + }, + { + "dictionaries:Value": "ServiceB", + "dictionaries:Meaning": "Service B allows users to ..." + } + ] } - ] - }, - "SupportedLIWorkflowEndpoints": {}, - "SupportedLPWorkflowEndpoints": { - "config:SupportedLPWorkflowEndpoint": [ + } + ] + }, + "TargetFormatTypeDefinitions": { + "config:FormatOwner": "ACTOR02", + "config:TargetFormatTypeDefinitionEntries": { + "config:TargetFormatTypeDefinitionEntry": [ { - "config:LPWorkflowEndpoint": { - "common:Owner": "ETSI", - "common:Name": "LPWorkflowEndpoint", - "common:Value": "NewAuthorisation" - }, - "config:Guidance": "This tells the LEA what endpoint to use for new warrants. For brevity, in this example, this is the only endpoint specified.", - "config:URL": "https://ts103120.example.com/lp/authorisation/new" + "config:FormatName": "ProprietaryIdentifier", + "config:Description": "This is an illustration of a proprietary identifier type specific to this CSP, which consists of the string \"CSP\" followed by 10 digits", + "config:FormatRegex": "^CSP[0-9]{10}$" } ] } + }, + "TargetingConfigurations": { + "config:TargetingConfiguration": [ + { + "config:FormatName": "PropietaryIdentifier", + "config:FormatOwner": "ACTOR02", + "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for the propietary identifier format type described in the TargetFormatTypeDefinitions, associated with Service A for the \"Subscriber Data\" category only", + "config:AssociatedLIServiceTypes": {}, + "config:AssociatedLDRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "SubscriberData" + } + ] + }, + "config:AssociatedLDRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LDRequestSubtype", + "common:Value": "ServiceA" + } + ] + }, + "config:AssociatedLPRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "SubscriberData" + } + ] + }, + "config:AssociatedLPRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LPRequestSubtype", + "common:Value": "ServiceA" + } + ] + } + }, + { + "config:FormatName": "InternationalE164", + "config:FormatOwner": "ETSI", + "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for a telephone number (as ETSI standard format E164 number) associated with Service A and Service B for the \"Traffic Data\" and \"Content Data\" categories", + "config:AssociatedLIServiceTypes": {}, + "config:AssociatedLDRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "TrafficData" + }, + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "StoredContentData" + } + ] + }, + "config:AssociatedLDRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LDRequestSubtype", + "common:Value": "ServiceA" + }, + { + "common:Owner": "ACTOR02", + "common:Name": "LDRequestSubtype", + "common:Value": "ServiceB" + } + ] + }, + "config:AssociatedLPRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "TrafficData" + }, + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "StoredContentData" + } + ] + }, + "config:AssociatedLPRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LPRequestSubtype", + "common:Value": "ServiceA" + }, + { + "common:Owner": "ACTOR02", + "common:Name": "LPRequestSubtype", + "common:Value": "ServiceB" + } + ] + } + } + ] + }, + "SupportedLIWorkflowEndpoints": {}, + "SupportedLPWorkflowEndpoints": { + "config:SupportedLPWorkflowEndpoint": [ + { + "config:LPWorkflowEndpoint": { + "common:Owner": "ETSI", + "common:Name": "LPWorkflowEndpoint", + "common:Value": "NewAuthorisation" + }, + "config:Guidance": "This tells the LEA what endpoint to use for new warrants. For brevity, in this example, this is the only endpoint specified.", + "config:URL": "https://ts103120.example.com/lp/authorisation/new" + } + ] } } - ] - } + } + ] } } } - \ No newline at end of file +} diff --git a/103705/schema/etsi_types.schema.json b/103705/schema/etsi_types.schema.json index 662bbc63abe2947b4b6edaec2f30d9d27d8e9652..00c95cd28688a2a00ac4b80e1773fe9a1e4e995a 100644 --- a/103705/schema/etsi_types.schema.json +++ b/103705/schema/etsi_types.schema.json @@ -66,7 +66,7 @@ "roamingRecord": { "$ref": "#/$defs/RoamingInformation", "description": "Information for inbound / outbound roaming record." - } + } } }, "MessagingRecord": { diff --git a/103976/examples/CommsIDtoVIN.json b/103976/examples/CommsIDtoVIN.json index 14bc4199ba9990b400311f7cc0a3f3bd5e69d467..25920a37803eef9bb8899cf5be25405594c97503 100644 --- a/103976/examples/CommsIDtoVIN.json +++ b/103976/examples/CommsIDtoVIN.json @@ -1,5 +1,7 @@ { - "CommsIDtoVINRecords" : [ - { "VIN" : "1G9Y817H34LSP7293" } - ] -} \ No newline at end of file + "CommsIDtoVINRecords": [ + { + "VIN": "1G9Y817H34LSP7293" + } + ] +} diff --git a/103976/examples/VINtoCommsID.json b/103976/examples/VINtoCommsID.json index 471523a361109866b7589eaa811a30ab2b483cbd..b130c84fb66c49e753c3c29e701ab90d0c18d476 100644 --- a/103976/examples/VINtoCommsID.json +++ b/103976/examples/VINtoCommsID.json @@ -1,67 +1,87 @@ { - "VINtoCommsIDRecords" : [ - { - "CommsID" : { "IMEI" : "00440123456789" }, - "AssociationTime" : { - "PeriodInTime" : { - "StartTime" : "2022-01-16T15:57:00Z", - "EndTime" : "2022-01-16T15:57:00Z" - } - } - }, - { - "CommsID" : { "IMSI" : "999990123456789" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "ICCID" : "89999012345678901234" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "PEIIMEI" : "00440123456789" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "SUPIIMSI" : "999990123456789" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "SUPINAI" : "example@example.com" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "MSISDN" : "491713920000" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "GPSIMSISDN" : "491713920000" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "MACAddress" : "00:00:5e:00:53:00" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "EUI164" : "00:00:5e:ef:10:00:00:00" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } + "VINtoCommsIDRecords": [ + { + "CommsID": { + "IMEI": "00440123456789" + }, + "AssociationTime": { + "PeriodInTime": { + "StartTime": "2022-01-16T15:57:00Z", + "EndTime": "2022-01-16T15:57:00Z" } - ] -} \ No newline at end of file + } + }, + { + "CommsID": { + "IMSI": "999990123456789" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "ICCID": "89999012345678901234" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "PEIIMEI": "00440123456789" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "SUPIIMSI": "999990123456789" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "SUPINAI": "example@example.com" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "MSISDN": "491713920000" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "GPSIMSISDN": "491713920000" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "MACAddress": "00:00:5e:00:53:00" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "EUI164": "00:00:5e:ef:10:00:00:00" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + } + ] +} diff --git a/103976/examples/VINtoLocation.json b/103976/examples/VINtoLocation.json index 708d23bc7ce49f0af98118a860539a8b444907b7..832fb2a459bc13cd842f271a274534f3c88d4636 100644 --- a/103976/examples/VINtoLocation.json +++ b/103976/examples/VINtoLocation.json @@ -1,28 +1,28 @@ { - "VINtoLocationRecords" : [ - { - "Location" : { - "WGS84CoordinateDecimal" : { - "etsi280:latitude" : "N43.617003", - "etsi280:longitude" : "E007.053222" - } - }, - "TimeOfLocation" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "Location" : { - "WGS84CoordinateDecimal" : { - "etsi280:latitude" : "N43.617003", - "etsi280:longitude" : "E007.053222" - } - }, - "TimeOfLocation" : { - "PointInTime" : "2022-01-16T15:57:00Z" - }, - "SourceOfLocation" : "GNSS", - "LocationRecordReason" : "Location record reason" + "VINtoLocationRecords": [ + { + "Location": { + "WGS84CoordinateDecimal": { + "etsi280:latitude": "N43.617003", + "etsi280:longitude": "E007.053222" } - ] -} \ No newline at end of file + }, + "TimeOfLocation": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "Location": { + "WGS84CoordinateDecimal": { + "etsi280:latitude": "N43.617003", + "etsi280:longitude": "E007.053222" + } + }, + "TimeOfLocation": { + "PointInTime": "2022-01-16T15:57:00Z" + }, + "SourceOfLocation": "GNSS", + "LocationRecordReason": "Location record reason" + } + ] +} diff --git a/103976/ts_103976.schema.json b/103976/ts_103976.schema.json index 84fe84cc8eef3a56f8938d4a055c11d88070d2b1..2f45d02434fc1530db81b6e5e4e94f3915461e02 100644 --- a/103976/ts_103976.schema.json +++ b/103976/ts_103976.schema.json @@ -1,204 +1,363 @@ { - "$id": "ts_103976_core", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$defs": { - "AssociationPeriod" : { - "type" : "object", - "properties" : { - "StartTime" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "EndTime" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" } - }, - "required" : ["StartTime"] - }, - "AssociationTime" : { - "oneOf" : [ - { - "type" : "object", - "properties" : { - "PointInTime" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" } - }, - "required" : ["PointInTime"] - }, - { - "type" : "object", - "properties" : { - "PeriodInTime" : { "$ref": "#/$defs/AssociationPeriod" } - }, - "required" : ["PeriodInTime"] - } - ] - }, - "CommsID" : { - "oneOf" : [ - { - "type" : "object", - "properties" : { "IMEI" : { "$ref" : "ts_103280_2017_07#/$defs/IMEI"} }, - "required" : ["IMEI"] - }, - { - "type" : "object", - "properties" : { "IMSI" : { "$ref" : "ts_103280_2017_07#/$defs/IMSI"} }, - "required" : ["IMSI"] - }, - { - "type" : "object", - "properties" : { "ICCID" : { "$ref" : "ts_103280_2017_07#/$defs/ICCID"} }, - "required" : ["ICCID"] - }, - { - "type" : "object", - "properties" : { "PEIIMEI" : { "$ref" : "ts_103280_2017_07#/$defs/PEIIMEI"} }, - "required" : ["PEIIMEI"] - }, - { - "type" : "object", - "properties" : { "SUPIIMSI" : { "$ref" : "ts_103280_2017_07#/$defs/SUPIIMSI"} }, - "required" : ["SUPIIMSI"] - }, - { - "type" : "object", - "properties" : { "SUPINAI" : { "$ref" : "ts_103280_2017_07#/$defs/SUPINAI"} }, - "required" : ["SUPINAI"] - }, - { - "type" : "object", - "properties" : { "MSISDN" : { "$ref" : "ts_103280_2017_07#/$defs/InternationalE164"} }, - "required" : ["MSISDN"] - }, - { - "type" : "object", - "properties" : { "GPSIMSISDN" : { "$ref" : "ts_103280_2017_07#/$defs/GPSIMSISDN"} }, - "required" : ["GPSIMSISDN"] - }, - { - "type" : "object", - "properties" : { "GPSINAI" : { "$ref" : "ts_103280_2017_07#/$defs/GPSINAI"} }, - "required" : ["GPSINAI"] - }, - { - "type" : "object", - "properties" : { "MACAddress" : { "$ref" : "ts_103280_2017_07#/$defs/MACAddress"} }, - "required" : ["MACAddress"] - }, - { - "type" : "object", - "properties" : { "EUI164" : { "$ref" : "ts_103280_2017_07#/$defs/EUI64"} }, - "required" : ["EUI164"] - } - ] - }, - "Location" : { - "oneOf" : [ - { - "type" : "object", - "properties" : { - "WGS84CoordinateDecimal" : { "$ref" : "ts_103280_2017_07#/$defs/WGS84CoordinateDecimal"} - }, - "required" : ["WGS84CoordinateDecimal"] - } - ] - }, - "SourceOfLocation" : { - "enum" : ["GNSS"] - }, - "VINtoCommsIDRecord" : { - "type" : "object", - "properties" : { - "CommsID" : { "$ref" : "#/$defs/CommsID" }, - "AssociationTime" : { "$ref" : "#/$defs/AssociationTime"} - }, - "required" : ["CommsID"] - }, - "CommsIDToVINRecord" : { - "type" : "object", - "properties" : { - "VIN" : { "$ref" : "ts_103280_2017_07#/$defs/VIN" }, - "AssociationTime" : { "$ref" : "#/$defs/AssociationTime"} - }, - "required" : ["VIN"] - }, - "VINtoLocationRecord" : { - "type" : "object", - "properties" : { - "Location" : { "$ref" : "#/$defs/Location" }, - "TimeOfLocation" : { "$ref" : "#/$defs/AssociationTime" }, - "SourceOfLocation" : { "$ref" : "#/$defs/SourceOfLocation" }, - "LocationRecordReason" : { "$ref" : "ts_103280_2017_07#/$defs/LongString" } - }, - "required" : ["Location", "TimeOfLocation"] - }, - "VINtoUniquePartNumberRecord" : { - "type" : "object", - "properties" : { - "UniquePartNumber" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "PartType" : { "$ref" : "ts_103120_Common_2016_02#/$defs/DictionaryEntry"}, - "PartTypeFreeText" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" } - }, - "required" : ["UniquePartNumber"] - }, - "UniquePartNumbertoVINRecord" : { - "type" : "object", - "properties" : { - "VIN" : { "$ref" : "ts_103280_2017_07#/$defs/VIN" } - }, - "required" : ["VIN"] - }, - "VINtoSubscribedServicesRecord" : { - "type" : "object", - "properties" : { - "NameOfService" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "FirstPaymentDate" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "MostRecentPaymentDate" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "TerminationDate" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "UserID" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "CustomerName" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "CustomerAddress": { "$ref" : "ts_103280_2017_07#/$defs/LongString" }, - "TypeOfPaymentMethod" : { "$ref" : "ts_103120_Common_2016_02#/$defs/DictionaryEntry"} - }, - "required" : [] - }, - "VINtoConnectedDevicesRecord" : { - "type" : "object", - "properties" : { - "ConnectionType" : { "$ref" : "ts_103120_Common_2016_02#/$defs/DictionaryEntry"}, - "DeviceIdentifiers" : { "$ref" : "#/$defs/ListOfCommsIDs"}, - "ConnectionTime" : { "$ref" : "#/$defs/AssociationTime" } - }, - "required" : ["ConnectionTime"] - }, - "ListOfCommsIDs" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/CommsID" } - }, - "ResultRecords" : { - "type" : "object", - "properties" : { - "VINtoCommsIDRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoCommsIDRecord"} - }, - "CommsIDtoVINRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/CommsIDToVINRecord"} - }, - "VINtoLocationRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoLocationRecord"} - }, - "VINtoUniquePartNumberRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoUniquePartNumberRecord"} - }, - "UniquePartNumbertoVINRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/UniquePartNumbertoVINRecord"} - }, - "VINtoSubscribedServicesRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoSubscribedServicesRecord"} - } + "$id": "ts_103976_core", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "AssociationPeriod": { + "type": "object", + "properties": { + "StartTime": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "EndTime": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + } + }, + "required": [ + "StartTime" + ] + }, + "AssociationTime": { + "oneOf": [ + { + "type": "object", + "properties": { + "PointInTime": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + } + }, + "required": [ + "PointInTime" + ] + }, + { + "type": "object", + "properties": { + "PeriodInTime": { + "$ref": "#/$defs/AssociationPeriod" + } + }, + "required": [ + "PeriodInTime" + ] + } + ] + }, + "CommsID": { + "oneOf": [ + { + "type": "object", + "properties": { + "IMEI": { + "$ref": "ts_103280_2017_07#/$defs/IMEI" + } + }, + "required": [ + "IMEI" + ] + }, + { + "type": "object", + "properties": { + "IMSI": { + "$ref": "ts_103280_2017_07#/$defs/IMSI" + } + }, + "required": [ + "IMSI" + ] + }, + { + "type": "object", + "properties": { + "ICCID": { + "$ref": "ts_103280_2017_07#/$defs/ICCID" + } + }, + "required": [ + "ICCID" + ] + }, + { + "type": "object", + "properties": { + "PEIIMEI": { + "$ref": "ts_103280_2017_07#/$defs/PEIIMEI" + } + }, + "required": [ + "PEIIMEI" + ] + }, + { + "type": "object", + "properties": { + "SUPIIMSI": { + "$ref": "ts_103280_2017_07#/$defs/SUPIIMSI" + } + }, + "required": [ + "SUPIIMSI" + ] + }, + { + "type": "object", + "properties": { + "SUPINAI": { + "$ref": "ts_103280_2017_07#/$defs/SUPINAI" } + }, + "required": [ + "SUPINAI" + ] + }, + { + "type": "object", + "properties": { + "MSISDN": { + "$ref": "ts_103280_2017_07#/$defs/InternationalE164" + } + }, + "required": [ + "MSISDN" + ] + }, + { + "type": "object", + "properties": { + "GPSIMSISDN": { + "$ref": "ts_103280_2017_07#/$defs/GPSIMSISDN" + } + }, + "required": [ + "GPSIMSISDN" + ] + }, + { + "type": "object", + "properties": { + "GPSINAI": { + "$ref": "ts_103280_2017_07#/$defs/GPSINAI" + } + }, + "required": [ + "GPSINAI" + ] + }, + { + "type": "object", + "properties": { + "MACAddress": { + "$ref": "ts_103280_2017_07#/$defs/MACAddress" + } + }, + "required": [ + "MACAddress" + ] + }, + { + "type": "object", + "properties": { + "EUI164": { + "$ref": "ts_103280_2017_07#/$defs/EUI64" + } + }, + "required": [ + "EUI164" + ] + } + ] + }, + "Location": { + "oneOf": [ + { + "type": "object", + "properties": { + "WGS84CoordinateDecimal": { + "$ref": "ts_103280_2017_07#/$defs/WGS84CoordinateDecimal" + } + }, + "required": [ + "WGS84CoordinateDecimal" + ] + } + ] + }, + "SourceOfLocation": { + "enum": [ + "GNSS" + ] + }, + "VINtoCommsIDRecord": { + "type": "object", + "properties": { + "CommsID": { + "$ref": "#/$defs/CommsID" + }, + "AssociationTime": { + "$ref": "#/$defs/AssociationTime" } + }, + "required": [ + "CommsID" + ] }, - "$ref": "#/$defs/ResultRecords" -} \ No newline at end of file + "CommsIDToVINRecord": { + "type": "object", + "properties": { + "VIN": { + "$ref": "ts_103280_2017_07#/$defs/VIN" + }, + "AssociationTime": { + "$ref": "#/$defs/AssociationTime" + } + }, + "required": [ + "VIN" + ] + }, + "VINtoLocationRecord": { + "type": "object", + "properties": { + "Location": { + "$ref": "#/$defs/Location" + }, + "TimeOfLocation": { + "$ref": "#/$defs/AssociationTime" + }, + "SourceOfLocation": { + "$ref": "#/$defs/SourceOfLocation" + }, + "LocationRecordReason": { + "$ref": "ts_103280_2017_07#/$defs/LongString" + } + }, + "required": [ + "Location", + "TimeOfLocation" + ] + }, + "VINtoUniquePartNumberRecord": { + "type": "object", + "properties": { + "UniquePartNumber": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "PartType": { + "$ref": "ts_103120_Common_2016_02#/$defs/DictionaryEntry" + }, + "PartTypeFreeText": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + } + }, + "required": [ + "UniquePartNumber" + ] + }, + "UniquePartNumbertoVINRecord": { + "type": "object", + "properties": { + "VIN": { + "$ref": "ts_103280_2017_07#/$defs/VIN" + } + }, + "required": [ + "VIN" + ] + }, + "VINtoSubscribedServicesRecord": { + "type": "object", + "properties": { + "NameOfService": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "FirstPaymentDate": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "MostRecentPaymentDate": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "TerminationDate": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "UserID": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "CustomerName": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "CustomerAddress": { + "$ref": "ts_103280_2017_07#/$defs/LongString" + }, + "TypeOfPaymentMethod": { + "$ref": "ts_103120_Common_2016_02#/$defs/DictionaryEntry" + } + }, + "required": [] + }, + "VINtoConnectedDevicesRecord": { + "type": "object", + "properties": { + "ConnectionType": { + "$ref": "ts_103120_Common_2016_02#/$defs/DictionaryEntry" + }, + "DeviceIdentifiers": { + "$ref": "#/$defs/ListOfCommsIDs" + }, + "ConnectionTime": { + "$ref": "#/$defs/AssociationTime" + } + }, + "required": [ + "ConnectionTime" + ] + }, + "ListOfCommsIDs": { + "type": "array", + "items": { + "$ref": "#/$defs/CommsID" + } + }, + "ResultRecords": { + "type": "object", + "properties": { + "VINtoCommsIDRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoCommsIDRecord" + } + }, + "CommsIDtoVINRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/CommsIDToVINRecord" + } + }, + "VINtoLocationRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoLocationRecord" + } + }, + "VINtoUniquePartNumberRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoUniquePartNumberRecord" + } + }, + "UniquePartNumbertoVINRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/UniquePartNumbertoVINRecord" + } + }, + "VINtoSubscribedServicesRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoSubscribedServicesRecord" + } + } + } + } + }, + "$ref": "#/$defs/ResultRecords" +} diff --git a/testing/asn/asn_compile_targets.json b/testing/asn/asn_compile_targets.json new file mode 100644 index 0000000000000000000000000000000000000000..f12b657b20ca861ca06c42479109ec09dc8791d0 --- /dev/null +++ b/testing/asn/asn_compile_targets.json @@ -0,0 +1,35 @@ +[ + [ + "./102232-1/LI-PS-PDU.asn", + "./103280/TS_103_280.asn1", + "./testing/deps/33128/TS33128Payloads.asn", + "./testing/deps/33108/Three3gppHI1Notifications.asn", + "./testing/deps/33108/UMTSHI2Operations.asn", + "./testing/deps/33108/UMTSHI3PS.asn", + "./testing/deps/33108/EpsHI3PS.asn", + "./testing/deps/33108/ConfHI3IMS.asn", + "./testing/deps/33108/VoipHI3IMS.asn", + "./testing/deps/33108/GCSEHI3.asn", + "./testing/deps/33108/CSVoiceHI3IP.asn", + "./testing/deps/33108/UMTSCSHI2Operations.asn", + "./testing/deps/33108/EpsHI2Operations.asn", + "./testing/deps/33108/ConfHI2Operations.asn", + "./testing/deps/33108/ProSeHI2Operations.asn", + "./testing/deps/33108/GCSEHI2Operations.asn", + "./testing/deps/101671/HI1NotificationOperations,ver7.asn", + "./testing/deps/101671/HI2Operations,ver18.asn", + "./testing/deps/101909/TS101909201.asn", + "./testing/deps/101909/TS101909202.asn", + "./testing/deps/101909/PCESP.asn", + "./testing/deps/301040/06132v203_C01.asn", + "./103462/ILHIPDU.asn", + "./102232-2/EmailPDU.asn", + "./102232-3/IPAccessPDU.asn", + "./102232-4/L2AccessPDU.asn", + "./102232-5/IPMultimediaPDU.asn", + "./102232-6/PstnIsdnPDU.asn" + ], + [ + "./102657/RDMessage.asn" + ] +] diff --git a/testing/asn_ignore.txt b/testing/asn/asn_ignore.txt similarity index 100% rename from testing/asn_ignore.txt rename to testing/asn/asn_ignore.txt diff --git a/testing/asn_ignore_lint.txt b/testing/asn/asn_ignore_lint.txt similarity index 100% rename from testing/asn_ignore_lint.txt rename to testing/asn/asn_ignore_lint.txt diff --git a/testing/asn_process.py b/testing/asn/asn_process.py similarity index 60% rename from testing/asn_process.py rename to testing/asn/asn_process.py index 5ca7aa5bf880d352c0ef9de3b7a3181fd0c680b5..508569153837abddc43900cd34773396890e0b56 100644 --- a/testing/asn_process.py +++ b/testing/asn/asn_process.py @@ -6,26 +6,32 @@ from shutil import which from pycrate_asn1c.asnproc import * -def reconstrainInteger (filename): - Path('temp.asn').write_text(Path(filename).read_text().replace("18446744073709551615", "65536")) - return 'temp.asn' + +def reconstrainInteger(filename): + Path("temp.asn").write_text( + Path(filename).read_text().replace("18446744073709551615", "65536") + ) + return "temp.asn" + filesWithBigInts = [ - '102232-1/LI-PS-PDU.asn', - '102232-3/IPAccessPDU.asn', - '102232-4/L2AccessPDU.asn' + "102232-1/LI-PS-PDU.asn", + "102232-3/IPAccessPDU.asn", + "102232-4/L2AccessPDU.asn", ] asn1c_path = "" change_path_to_unix = False + def fix_path(path): if change_path_to_unix: - return "./" + path.replace("\\","/") + return "./" + path.replace("\\", "/") else: return path -def syntaxCheckASN (fileList): + +def syntaxCheckASN(fileList): """ Performs ASN syntax checking on a list of filenames (or pathlib Paths) @@ -40,56 +46,48 @@ def syntaxCheckASN (fileList): try: if file.as_posix() in filesWithBigInts: newFile = reconstrainInteger(str(file)) - p = run([asn1c_path, '-E', fix_path(newFile)], capture_output=True) + p = run([asn1c_path, "-E", fix_path(newFile)], capture_output=True) Path(newFile).unlink() else: - p = run([asn1c_path, '-E', fix_path(str(file))], capture_output=True) - if (p.returncode != 0): + p = run([asn1c_path, "-E", fix_path(str(file))], capture_output=True) + if p.returncode != 0: errorMessage = p.stderr.decode().splitlines()[0] if errorMessage.startswith(' Value "18446744073709551615" at line'): - results[str(file)] = { 'ok' : True} + results[str(file)] = {"ok": True} continue results[str(file)] = { - 'ok' : False, - 'code' : p.returncode, - 'message' : p.stderr.decode().splitlines()[0] + "ok": False, + "code": p.returncode, + "message": p.stderr.decode().splitlines()[0], } else: - results[str(file)] = { - 'ok' : True - } + results[str(file)] = {"ok": True} except Exception as ex: raise ex - results[str(file)] = { - 'ok' : False, - 'code' : -1, - 'message' : f"{ex!r}" - } + results[str(file)] = {"ok": False, "code": -1, "message": f"{ex!r}"} return results duplicateObjects = { - '102232-1/LI-PS-PDU.asn' : [ - 'CCPayload', - 'IRIPayload', - 'Location' - ], - 'testing/mod1.asn' : [ - 'ClashField' - ] + "102232-1/LI-PS-PDU.asn": ["CCPayload", "IRIPayload", "Location"], + "testing/mod1.asn": ["ClashField"], } + + def fixDuplicateObjects(filename): stringContent = filename.read_text() for object in duplicateObjects[filename.as_posix()]: - stringContent = stringContent.replace(f'{object} ::=', f'Native{object} ::=', 1) - stringContent = stringContent.replace(f'SEQUENCE OF {object}', f'SEQUENCE OF Native{object}') - #stringContent = sub(f"]\\w{object}", f"] Native{object}", stringContent) + stringContent = stringContent.replace(f"{object} ::=", f"Native{object} ::=", 1) + stringContent = stringContent.replace( + f"SEQUENCE OF {object}", f"SEQUENCE OF Native{object}" + ) + # stringContent = sub(f"]\\w{object}", f"] Native{object}", stringContent) - Path('temp.asn').write_text(stringContent) - return 'temp.asn' + Path("temp.asn").write_text(stringContent) + return "temp.asn" -def compileAllTargets (compileTargets): +def compileAllTargets(compileTargets): """ Attempts to compile a set of compile targets using the pycrate ASN1 tools @@ -100,10 +98,10 @@ def compileAllTargets (compileTargets): to be the "primary" file. This doesn't have any relavance to the compilation, but will be used as the identifier when reporting any compile errors. The compilation is performed by the pycrate ASN compile functions; errors - are caught as exceptions and rendered into a list. - + are caught as exceptions and rendered into a list. + Unfortunately, the pycrate compiler doesn't report line numbers. - The asn1c compiler does, but doesn't properly handle identifiers with the + The asn1c compiler does, but doesn't properly handle identifiers with the same name in different modules; as this occurs multiple times in TS 33.108, we can't use it. """ @@ -120,107 +118,105 @@ def compileAllTargets (compileTargets): if pFile.as_posix() in duplicateObjects: tmpFile = Path(fixDuplicateObjects(pFile)) fileTexts.append(tmpFile.read_text()) - #tmpFile.unlink() + # tmpFile.unlink() else: fileTexts.append(pFile.read_text()) fileNames.append(filename) - logging.debug (f" Loading {filename}") - compile_text(fileTexts, filenames = fileNames) + logging.debug(f" Loading {filename}") + compile_text(fileTexts, filenames=fileNames) results[str(firstTarget)] = { - 'ok' : True, + "ok": True, } except Exception as ex: - results[str(firstTarget)] = { - 'ok' : False, - 'code' : -1, - 'message' : f"{ex!r}" - } + results[str(firstTarget)] = {"ok": False, "code": -1, "message": f"{ex!r}"} continue return results - -def processResults (results, stageName): +def processResults(results, stageName): """ Counts the number of errors and writes out the output per filename :param results: List of filenames (str or Pathlib Path) :param stageName: Name to decorate the output with :returns: The number of files which had errors - """ + """ print("") - errorCount = sum([1 for r in results.values() if not r['ok']]) + errorCount = sum([1 for r in results.values() if not r["ok"]]) logging.info(f"{errorCount} {stageName} errors encountered") - + print(f"{'-':-<60}") print(f"{stageName} results:") print(f"{'-':-<60}") for filename, result in results.items(): print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") - if not result['ok']: - if isinstance(result['message'], list): - for thing in result['message']: + if not result["ok"]: + if isinstance(result["message"], list): + for thing in result["message"]: print(f" {thing['message']}") else: print(f" {result['message']}") - + print(f"{'-':-<60}") print(f"{stageName} errors: {errorCount}") print(f"{'-':-<60}") - + return errorCount -if __name__ == '__main__': - logging.info ('Searching for ASN1C') +if __name__ == "__main__": + logging.info("Searching for ASN1C") asn1c_path = which("asn1c") if asn1c_path is None: - raise Exception ("No asn1c executable found. Please install asn1c") - logging.info (f"asn1c found at {asn1c_path}") + raise Exception("No asn1c executable found. Please install asn1c") + logging.info(f"asn1c found at {asn1c_path}") if asn1c_path.lower().endswith("bat"): - logging.info (f"asn1c is a batch file, so assume path separators need to be changed") + logging.info( + f"asn1c is a batch file, so assume path separators need to be changed" + ) change_path_to_unix = True - - logging.info('Searching for ASN.1 files') + logging.info("Searching for ASN.1 files") fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) - logging.info(f'{len(fileList)} ASN.1 files found') + logging.info(f"{len(fileList)} ASN.1 files found") for file in fileList: - logging.debug(f' {file}') - - ignoreList = Path('testing/asn_ignore.txt').read_text().splitlines() + logging.debug(f" {file}") + + ignoreList = Path("testing/asn/asn_ignore.txt").read_text().splitlines() ignoredFiles = [] for ignore in ignoreList: - logging.debug(f'Ignoring pattern {ignore}') + logging.debug(f"Ignoring pattern {ignore}") for file in fileList: if ignore in str(file): ignoredFiles.append(file) logging.debug(f" Ignoring {str(file)} as contains {ignore}") ignoredFiles = list(set(ignoredFiles)) - logging.info(f'{len(ignoredFiles)} files ignored') + logging.info(f"{len(ignoredFiles)} files ignored") for file in ignoredFiles: - logging.debug(f' {file}') - + logging.debug(f" {file}") + fileList = [file for file in fileList if file not in ignoredFiles] - logging.info(f'{len(fileList)} files to process') + logging.info(f"{len(fileList)} files to process") for file in fileList: - logging.debug(f' {file}') + logging.debug(f" {file}") if len(fileList) == 0: - logging.warning ("No files specified") + logging.warning("No files specified") exit(0) - + logging.info("Parsing ASN1 files") parseResults = syntaxCheckASN(fileList) if processResults(parseResults, "Parsing") > 0: exit(-1) - logging.info ("Getting compile targets") - compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text()) - logging.info (f"{len(compileTargets)} compile targets found") + logging.info("Getting compile targets") + compileTargets = json.loads( + Path("testing/asn/asn_compile_targets.json").read_text() + ) + logging.info(f"{len(compileTargets)} compile targets found") compileResults = compileAllTargets(compileTargets) if processResults(compileResults, "Compiling") > 0: exit(-1) - + exit(0) diff --git a/testing/asn_compile_targets.json b/testing/asn_compile_targets.json deleted file mode 100644 index e186addc0214f2ecf1f25f507336f33725fafe2f..0000000000000000000000000000000000000000 --- a/testing/asn_compile_targets.json +++ /dev/null @@ -1,33 +0,0 @@ -[ - [ - "./102232-1/LI-PS-PDU.asn", - "./103280/TS_103_280.asn1", - "./testing/deps/33128/TS33128Payloads.asn", - "./testing/deps/33108/Three3gppHI1Notifications.asn", - "./testing/deps/33108/UMTSHI2Operations.asn", - "./testing/deps/33108/UMTSHI3PS.asn", - "./testing/deps/33108/EpsHI3PS.asn", - "./testing/deps/33108/ConfHI3IMS.asn", - "./testing/deps/33108/VoipHI3IMS.asn", - "./testing/deps/33108/GCSEHI3.asn", - "./testing/deps/33108/CSVoiceHI3IP.asn", - "./testing/deps/33108/UMTSCSHI2Operations.asn", - "./testing/deps/33108/EpsHI2Operations.asn", - "./testing/deps/33108/ConfHI2Operations.asn", - "./testing/deps/33108/ProSeHI2Operations.asn", - "./testing/deps/33108/GCSEHI2Operations.asn", - "./testing/deps/101671/HI1NotificationOperations,ver7.asn", - "./testing/deps/101671/HI2Operations,ver18.asn", - "./testing/deps/101909/TS101909201.asn", - "./testing/deps/101909/TS101909202.asn", - "./testing/deps/101909/PCESP.asn", - "./testing/deps/301040/06132v203_C01.asn", - "./103462/ILHIPDU.asn", - "./102232-2/EmailPDU.asn", - "./102232-3/IPAccessPDU.asn", - "./102232-4/L2AccessPDU.asn", - "./102232-5/IPMultimediaPDU.asn", - "./102232-6/PstnIsdnPDU.asn" - ], - ["./102657/RDMessage.asn"] -] \ No newline at end of file diff --git a/testing/.editorconfig b/testing/deps/.editorconfig similarity index 51% rename from testing/.editorconfig rename to testing/deps/.editorconfig index f1017abbfc8c591987b72d2bceeb1283cc322128..b9936750f79f5c2d6018db76720a6f84b3add978 100644 --- a/testing/.editorconfig +++ b/testing/deps/.editorconfig @@ -1,4 +1,4 @@ -[*] -trim_trailing_whitespace = unset +[**.*] indent_style = unset indent_size = unset +trim_trailing_whitespace = unset \ No newline at end of file diff --git a/testing/dockerfile_asn b/testing/dockerfiles/dockerfile_asn similarity index 100% rename from testing/dockerfile_asn rename to testing/dockerfiles/dockerfile_asn diff --git a/testing/dockerfile_xsd b/testing/dockerfiles/dockerfile_xsd similarity index 100% rename from testing/dockerfile_xsd rename to testing/dockerfiles/dockerfile_xsd diff --git a/testing/dockerfiles/forgeschema b/testing/dockerfiles/forgeschema new file mode 100644 index 0000000000000000000000000000000000000000..d3f9249d2036ee5ba12133410439df7b50ccc3e0 --- /dev/null +++ b/testing/dockerfiles/forgeschema @@ -0,0 +1,19 @@ +# docker login forge.etsi.org:5050 +# docker build -t forge.etsi.org:5050/li/schemas-definitions/forgeschema -f forgeschema . +# docker push forge.etsi.org:5050/li/schemas-definitions/forgeschema + +FROM python:3.12-alpine + +RUN apk add --no-cache \ + jq \ + libxml2-utils \ + libffi-dev \ + openssl-dev \ + gcc \ + musl-dev \ + python3-dev \ + build-base \ + editorconfig + +RUN pip install --upgrade pip && \ + pip install forgeschema \ No newline at end of file diff --git a/testing/json/lint_json.py b/testing/json/lint_json.py new file mode 100644 index 0000000000000000000000000000000000000000..783d53bf20770f3c86f6e270ff50f8c4f507cef5 --- /dev/null +++ b/testing/json/lint_json.py @@ -0,0 +1,81 @@ +from pathlib import Path +from difflib import * +import subprocess + +import colorama +from colorama import Fore, Style + +colorama.init() + +ignore_paths = [Path(x) for x in ["testing/deps"]] + + +def print_colorized_diff_line(line: str): + if line.startswith("-"): + print(f"{Fore.RED}{line}{Style.RESET_ALL}") + elif line.startswith("+"): + print(f"{Fore.GREEN}{line}{Style.RESET_ALL}") + else: + print(line) + + +def lint(file: Path): + completed = subprocess.run( + ["jq", ".", str(file)], capture_output=True, text=True, encoding="utf8" + ) + + if completed.returncode != 0: + print(f" {str(f)}: FAIL") + print(f" jq error code {completed.returncode}") + lines = completed.stderr.splitlines() + for line in lines: + print(f" {line}") + return len(lines) + + linted_xml = completed.stdout + orig_xml = file.read_text(encoding="utf8") + diff = list(unified_diff(orig_xml.splitlines(), linted_xml.splitlines())) + if len(diff) == 0: + print(f"✅ {str(f)}") + return 0 + else: + print(f"❌ {str(f)}: {len(diff)} linting errors") + # for d in diff: + # print("".join(d)) + for d in diff: + print_colorized_diff_line(d) + return len(diff) + + +if __name__ == "__main__": + root = Path("./") + files = list(root.rglob("*.json")) + + files_with_errors = 0 + errors = 0 + + print("───────────────────────────────────────────────────────────────────") + print(f"Linting JSON files...") + + for f in files: + if len(list(set(f.parents) & set(ignore_paths))) > 0: + print(f"(Ignoring {f})") + continue + new_errors = lint(f) + errors += new_errors + files_with_errors += 1 if new_errors > 0 else 0 + + print("───────────────────────────────────────────────────────────────────") + print( + f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}" + ) + if files_with_errors == 0: + print("✅ OK") + else: + print("❌ Fail") + print("───────────────────────────────────────────────────────────────────") + + if files_with_errors > 0: + exit(-1) + else: + exit(0) diff --git a/testing/json/ts_103120._json.json b/testing/json/ts_103120._json.json new file mode 100644 index 0000000000000000000000000000000000000000..bdf4c3985189a9d88f2d2236f06ae0459ee1f2c9 --- /dev/null +++ b/testing/json/ts_103120._json.json @@ -0,0 +1,11 @@ +{ + "coreSchema": "103120/schema/json/ts_103120_Core.schema.json", + "supportingSchemas": [ + "103120/schema/json", + "103120/dictionaries/ts_103120_Dictionaries.schema.json", + "103280/TS_103_280.schema.json" + ], + "instanceDocs": [ + "103120/examples/json" + ] +} diff --git a/testing/json/ts_103280._json.json b/testing/json/ts_103280._json.json new file mode 100644 index 0000000000000000000000000000000000000000..a00d98e54912980b9fe1b21bd45742d167e1b120 --- /dev/null +++ b/testing/json/ts_103280._json.json @@ -0,0 +1,5 @@ +{ + "coreSchema": "103120/dictionaries/ts_103120_Dictionaries.schema.json", + "supportingSchemas": [], + "instanceDocs": [] +} diff --git a/testing/json/ts_103705_ex1_json.json b/testing/json/ts_103705_ex1_json.json new file mode 100644 index 0000000000000000000000000000000000000000..c9ad6211f8bb0d528aef9c3dbb394a95453798cc --- /dev/null +++ b/testing/json/ts_103705_ex1_json.json @@ -0,0 +1,11 @@ +{ + "coreSchema": "103705/schema/response.schema.json", + "supportingSchemas": [ + "103705/schema/", + "103280/TS_103_280.schema.json", + "103705/examples/example1/csp_records.schema.json" + ], + "instanceDocs": [ + "103705/examples/example1/csp_results.json" + ] +} diff --git a/testing/json/ts_103976_json.json b/testing/json/ts_103976_json.json new file mode 100644 index 0000000000000000000000000000000000000000..3d12b02bd2f09effb15e97ed881faca12b9618ad --- /dev/null +++ b/testing/json/ts_103976_json.json @@ -0,0 +1,10 @@ +{ + "coreSchema": "103976/ts_103976.schema.json", + "supportingSchemas": [ + "103280/TS_103_280.schema.json", + "103120/schema/json/ts_103120_Common.schema.json" + ], + "instanceDocs": [ + "103976/examples/" + ] +} diff --git a/testing/lint_xml.py b/testing/lint_xml.py deleted file mode 100644 index abfa1a170ff4d46dc5bfbd7bf50677c7e032c383..0000000000000000000000000000000000000000 --- a/testing/lint_xml.py +++ /dev/null @@ -1,59 +0,0 @@ -from pathlib import Path -from difflib import * -import subprocess - -ignore_paths = [Path(x) for x in [ - 'testing/deps' -]] - -def lint(file : Path): - completed = subprocess.run(["xmllint", str(file)], capture_output=True, text=True, encoding="utf8") - - if completed.returncode != 0: - print (f" {str(f)}: FAIL") - print (f" xmlint error code {completed.returncode}") - lines = completed.stderr.splitlines() - for line in lines: - print (f" {line}") - return len(lines) - - linted_xml = completed.stdout - orig_xml = file.read_text(encoding="utf8") - diff = list(unified_diff(orig_xml.splitlines(), linted_xml.splitlines())) - if len(diff) == 0: - print (f"✅ {str(f)}") - return 0 - else: - print (f"❌ {str(f)}: {len(diff)} linting errors") - for d in diff: - print("".join(d)) - return len(diff) - - - -if __name__ == "__main__": - root = Path("./") - files = list(root.rglob("*.xml")) + list(root.rglob("*.xsd")) - - files_with_errors = 0 - errors = 0 - - print ("-------------------------------------------------") - print (f"Linting XML / XSD files...") - - for f in files: - if len(list(set(f.parents) & set(ignore_paths))) > 0: - print (f"(Ignoring {f})") - continue - new_errors = lint (f) - errors += new_errors - files_with_errors += 1 if new_errors > 0 else 0 - - print ("-------------------------------------------------") - print (f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}") - print ("-------------------------------------------------") - - if (files_with_errors > 0): - exit(-1) - else: - exit(0) \ No newline at end of file diff --git a/testing/merge_test.py b/testing/merge_test.py deleted file mode 100644 index b7a82b39c4958ea30d4ac9a96e100e3041fe73c0..0000000000000000000000000000000000000000 --- a/testing/merge_test.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import pprint -import requests -import json -import subprocess - -crCommitBranch = os.environ.get("CI_COMMIT_REF_NAME", "NOTFOUND") -apiUrl = os.environ.get("CI_API_V4_URL", "https://forge.3gpp.org/rep/api/v4") -projectId = os.environ.get("CI_PROJECT_ID", "13") - -def gapi (query): - url = f"{apiUrl}/projects/{projectId}/{query}" - r = requests.get(url) - return json.loads(r.text) - -def do (commandline): - #print (" Attempting: " + commandline) - completedProc = subprocess.run(commandline, capture_output=True, shell=True) - #print (" STDOUT > " + ("empty" if completedProc.stdout is None else completedProc.stdout.decode('utf-8'))) - #print (" STDERR > " + ("empty" if completedProc.stderr is None else completedProc.stderr.decode('utf-8'))) - #print (f" Completed with code {completedProc.returncode}") - return (completedProc.returncode == 0, completedProc.stdout.decode('utf-8')) - -print ("Searching for corresponding MR...") - -mrs = gapi(f"merge_requests?source_branch={crCommitBranch}&state=opened") -if len(mrs) == 0: - print ("No MR found... aborting") - exit() - -if len(mrs) > 1: - print (f"{len(mrs)} MRs found, 1 expected - aborting") - for m in mrs: - pprint.pprint(m) - exit(-1) - -mr = mrs[0] - -print (f"Found MR {mr['reference']} ({mr['title']})") -print (f"Target branch is {mr['target_branch']}") -print ("Searching for open MRs targeting same branch...") - -mrs = gapi(f"merge_requests?target_branch={mr['target_branch']}&state=opened") -mrs = [m for m in mrs if m['reference'] != mr['reference']] -print (f"{len(mrs)} MRs found") - -mergeConflicts = {} - -for mr in mrs: - source_branch = mr['source_branch'] - print (source_branch) - - try: - do(f"git fetch origin {source_branch}:{source_branch}") - success, errStr = do(f"git merge --no-commit {source_branch}") - if not success: - print ("Merge NOT OK") - mergeConflicts[source_branch] = errStr - else: - print ("Merge OK") - except Exception as ex: - mergeConflicts[source_branch] = str(ex) - raise - finally: - do("git merge --abort") - -print (f"Merge conflicts with following branches: {mergeConflicts}") -exit(len(mergeConflicts.keys())) \ No newline at end of file diff --git a/testing/xml/etsi_dictionaries.json b/testing/xml/etsi_dictionaries.json new file mode 100644 index 0000000000000000000000000000000000000000..d693f9e377d5906cb7ae46d72a8338f1b7c1c2d5 --- /dev/null +++ b/testing/xml/etsi_dictionaries.json @@ -0,0 +1,9 @@ +{ + "coreSchema": "103120/dictionaries/ts_103120_Dictionaries.xsd", + "supportingSchemas": [], + "instanceDocs": [ + "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml", + "104144/dictionaries/", + "103976/ts_103976_ETSIDictionaryDefinitions.xml" + ] +} diff --git a/testing/xml/lint_xml.py b/testing/xml/lint_xml.py new file mode 100644 index 0000000000000000000000000000000000000000..85365e36131fa1a1a492b1fdc5ea0eedc949613c --- /dev/null +++ b/testing/xml/lint_xml.py @@ -0,0 +1,79 @@ +from pathlib import Path +from difflib import * +import subprocess + +import colorama +from colorama import Fore, Style + +colorama.init() + +ignore_paths = [Path(x) for x in ["testing/deps"]] + + +def print_colorized_diff_line(line: str): + if line.startswith("-"): + print(f"{Fore.RED}{line}{Style.RESET_ALL}") + elif line.startswith("+"): + print(f"{Fore.GREEN}{line}{Style.RESET_ALL}") + else: + print(line) + + +def lint(file: Path): + completed = subprocess.run( + ["xmllint", str(file)], capture_output=True, text=True, encoding="utf8" + ) + + if completed.returncode != 0: + print(f" {str(f)}: FAIL") + print(f" xmlint error code {completed.returncode}") + lines = completed.stderr.splitlines() + for line in lines: + print(f" {line}") + return len(lines) + + linted_xml = completed.stdout + orig_xml = file.read_text(encoding="utf8") + diff = list(unified_diff(orig_xml.splitlines(), linted_xml.splitlines())) + if len(diff) == 0: + print(f"✅ {str(f)}") + return 0 + else: + print(f"❌ {str(f)}: {len(diff)} linting errors") + for d in diff: + print("".join(d)) + return len(diff) + + +if __name__ == "__main__": + root = Path("./") + files = list(root.rglob("*.xml")) + list(root.rglob("*.xsd")) + + files_with_errors = 0 + errors = 0 + + print("") + print(f"Linting XML / XSD files...") + + for f in files: + if len(list(set(f.parents) & set(ignore_paths))) > 0: + print(f"(Ignoring {f})") + continue + new_errors = lint(f) + errors += new_errors + files_with_errors += 1 if new_errors > 0 else 0 + + print("───────────────────────────────────────────────────────────────────") + print( + f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}" + ) + if files_with_errors == 0: + print("✅ OK") + else: + print("❌ Fail") + print("───────────────────────────────────────────────────────────────────") + + if files_with_errors > 0: + exit(-1) + else: + exit(0) diff --git a/testing/xml/ts_102657_xsd.json b/testing/xml/ts_102657_xsd.json new file mode 100644 index 0000000000000000000000000000000000000000..2201b45dfb9623a1bbe0f868d8eb372d88a7feeb --- /dev/null +++ b/testing/xml/ts_102657_xsd.json @@ -0,0 +1,5 @@ +{ + "coreSchema": "102657/RDMessage.xsd", + "supportingSchemas": [], + "instanceDocs": [] +} diff --git a/testing/xml/ts_103120_xsd.json b/testing/xml/ts_103120_xsd.json new file mode 100644 index 0000000000000000000000000000000000000000..822b65df217a1d8efb42fa58d05c49f953bd184b --- /dev/null +++ b/testing/xml/ts_103120_xsd.json @@ -0,0 +1,20 @@ +{ + "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", + "supportingSchemas": [ + "103120/schema/xsd/ts_103120_Authorisation.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Document.xsd", + "103120/schema/xsd/ts_103120_Notification.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/xml/FooServiceSchema.xsd" + ], + "instanceDocs": [ + "103120/examples/xml" + ] +} diff --git a/testing/xml/ts_103221-1_xsd.json b/testing/xml/ts_103221-1_xsd.json new file mode 100644 index 0000000000000000000000000000000000000000..37409a36f98e3507f3e2d30d43c00f777d48dc67 --- /dev/null +++ b/testing/xml/ts_103221-1_xsd.json @@ -0,0 +1,22 @@ +{ + "coreSchema": "103221-1/TS_103_221_01.xsd", + "supportingSchemas": [ + "103221-1/TS_103_221_01.xsd", + "103221-1/TS_103_221_01_HashedID.xsd", + "103221-1/TS_103_221_01_Configuration.xsd", + "103221-1/TS_103_221_01_DestinationSet.xsd", + "103221-1/TS_103_221_01_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "103221-1/examples/ExampleGenericObjects.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Core.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "testing/deps/old_400/TS_104_000.xsd" + ], + "instanceDocs": [ + "103221-1/examples" + ] +} diff --git a/testing/xml/ts_103280_xsd.json b/testing/xml/ts_103280_xsd.json new file mode 100644 index 0000000000000000000000000000000000000000..3468dba20f1b1a54c08d3f34e8daedffe9278bdd --- /dev/null +++ b/testing/xml/ts_103280_xsd.json @@ -0,0 +1,5 @@ +{ + "coreSchema": "103280/TS_103_280.xsd", + "supportingSchemas": [], + "instanceDocs": [] +} diff --git a/testing/xml/ts_103707_xsd.json b/testing/xml/ts_103707_xsd.json new file mode 100644 index 0000000000000000000000000000000000000000..7cb681373c67f70b8288edee97d7a0053e319b5e --- /dev/null +++ b/testing/xml/ts_103707_xsd.json @@ -0,0 +1,19 @@ +{ + "coreSchema": "103707/TS_103_707.xsd", + "supportingSchemas": [ + "103280/TS_103_280.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Core.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103707/examples/FooServiceSchema.xsd", + "104112/examples/xml/MessagingServiceSchema.xsd" + ], + "instanceDocs": [ + "103707/examples", + "104112/examples" + ] +} diff --git a/testing/xml/ts_104000_xsd.json b/testing/xml/ts_104000_xsd.json new file mode 100644 index 0000000000000000000000000000000000000000..b23e93d15571bf8a487cdb38c1842033a1dba4ee --- /dev/null +++ b/testing/xml/ts_104000_xsd.json @@ -0,0 +1,9 @@ +{ + "coreSchema": "104000/schema/TS_104_000.xsd", + "supportingSchemas": [ + "103280/TS_103_280.xsd" + ], + "instanceDocs": [ + "104000/examples/" + ] +} diff --git a/testing/xml/ts_104144_xsd.json b/testing/xml/ts_104144_xsd.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae1b3056564b5a3f6e74c2bd0ac27bb062bd0f9 --- /dev/null +++ b/testing/xml/ts_104144_xsd.json @@ -0,0 +1,20 @@ +{ + "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", + "supportingSchemas": [ + "103120/schema/xsd/ts_103120_Authorisation.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Document.xsd", + "103120/schema/xsd/ts_103120_Notification.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/xml/FooServiceSchema.xsd" + ], + "instanceDocs": [ + "104144/examples/" + ] +} diff --git a/testing/xsd_compile_targets.json b/testing/xsd_compile_targets.json deleted file mode 100644 index 00ec73574f5556daf171e3ddc70d473c3c813ba4..0000000000000000000000000000000000000000 --- a/testing/xsd_compile_targets.json +++ /dev/null @@ -1,118 +0,0 @@ -[ - { - "coreSchema" : "102657/RDMessage.xsd", - "supportingSchemas" : [], - "exampleFiles" : [] - }, - { - "coreSchema" : "104000/schema/TS_104_000.xsd", - "supportingSchemas" : [ - "103280/TS_103_280.xsd" - ], - "exampleFiles" : [] - }, - { - "coreSchema" : "103280/TS_103_280.xsd", - "supportingSchemas" : [], - "exampleFiles" : [] - }, - { - "coreSchema" : "103221-1/TS_103_221_01.xsd", - "supportingSchemas" : [ - "103221-1/TS_103_221_01.xsd", - "103221-1/TS_103_221_01_HashedID.xsd", - "103221-1/TS_103_221_01_Configuration.xsd", - "103221-1/TS_103_221_01_DestinationSet.xsd", - "103221-1/TS_103_221_01_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "103221-1/examples/ExampleGenericObjects.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Core.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "testing/deps/old_400/TS_104_000.xsd" - ], - "exampleFiles" : [ - "103221-1/examples" - ] - }, - { - "coreSchema" : "103120/dictionaries/ts_103120_Dictionaries.xsd", - "supportingSchemas" : [], - "exampleFiles" : [ - "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml", - "104144/dictionaries/" - ] - }, - { - "coreSchema" : "103120/schema/xsd/ts_103120_Core.xsd", - "supportingSchemas" : [ - "103120/schema/xsd/ts_103120_Authorisation.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Document.xsd", - "103120/schema/xsd/ts_103120_Notification.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103120/examples/xml/FooServiceSchema.xsd" - ], - "exampleFiles" : [ - "103120/examples/xml" - ] - }, - { - "coreSchema" : "103707/TS_103_707.xsd", - "supportingSchemas" : [ - "103280/TS_103_280.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Core.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103707/examples/FooServiceSchema.xsd", - "104112/examples/xml/MessagingServiceSchema.xsd" - ], - "exampleFiles" : [ - "103707/examples", - "104112/examples" - ] - }, - { - "coreSchema" : "104000/schema/TS_104_000.xsd", - "supportingSchemas" : [ - "103280/TS_103_280.xsd" - ], - "exampleFiles" : [ - "104000/examples" - ] - }, - { - "coreSchema" : "103120/schema/xsd/ts_103120_Core.xsd", - "supportingSchemas" : [ - "103120/schema/xsd/ts_103120_Authorisation.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Document.xsd", - "103120/schema/xsd/ts_103120_Notification.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103120/examples/xml/FooServiceSchema.xsd", - "104144/schema/xsd/ts_104144_EPOCAdditionalInfo.xsd" - ], - "exampleFiles" : [ - "104144/examples" - ] - } -] diff --git a/testing/xsd_ignore.txt b/testing/xsd_ignore.txt deleted file mode 100644 index 90bfe5d59c3b61099ed6685d7ddd6023be91efe5..0000000000000000000000000000000000000000 --- a/testing/xsd_ignore.txt +++ /dev/null @@ -1,3 +0,0 @@ -deps -portal -temp \ No newline at end of file diff --git a/testing/xsd_process.py b/testing/xsd_process.py deleted file mode 100644 index 97dc2a883dd1f344811754a79ebfb328d829aa66..0000000000000000000000000000000000000000 --- a/testing/xsd_process.py +++ /dev/null @@ -1,222 +0,0 @@ -import json -import logging -from pathlib import Path - -from xmlschema import etree_tostring -from xmlschema import XMLSchema, XMLSchemaParseError - - -def BuildSchemaDictonary (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return [] - - logging.info("Schema locations:") - schemaLocations = [] - for schemaFile in fileList: - try: - xs = XMLSchema(schemaFile, validation='skip') - schemaLocations.append((xs.default_namespace, str(Path(schemaFile).resolve()))) - logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, schemaFile)) - except XMLSchemaParseError as ex: - logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) - return schemaLocations - - -def BuildSchema (coreFile, fileList = None): - schemaLocations = [] - if fileList and len(fileList) > 0: - schemaLocations = BuildSchemaDictonary(fileList) - - coreSchema = XMLSchema(str(Path(coreFile)), locations=schemaLocations) - return coreSchema - - -def ValidateXSDFiles (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return {} - - schemaLocations = BuildSchemaDictonary(fileList) - errors = {} - - logging.info("Schema validation:") - for schemaFile in fileList: - try: - schema = XMLSchema(schemaFile, locations = schemaLocations, validation="lax") - logging.info(schemaFile + ": OK") - errors[schemaFile] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in schema.all_errors] - except XMLSchemaParseError as ex: - logging.warning(schemaFile + ": Failed validation ({0})".format(ex.message)) - if (ex.schema_url) and (ex.schema_url != ex.origin_url): - logging.warning(" Error comes from {0}, suppressing".format(ex.schema_url)) - errors[schemaFile] = [] - else: - errors[schemaFile] = [ex] - return errors - - -def ValidateAllXSDFilesInPath (path): - schemaGlob = [str(f) for f in Path(path).rglob("*.xsd")] - return ValidateXSDFiles(schemaGlob) - - -def ValidateInstanceDocuments (coreFile, supportingSchemas, instanceDocs): - if (instanceDocs is None) or len(instanceDocs) == 0: - logging.warning ("No instance documents provided") - return [] - - schema = BuildSchema(coreFile, supportingSchemas) - errors = [] - for instanceDoc in instanceDocs: - try: - schema.validate(instanceDoc) - logging.info ("{0} passed validation".format(instanceDoc)) - except Exception as ex: - logging.error ("{0} failed validation: {1}".format(instanceDoc, ex)) - return errors - - -def processResults (results, stageName): - """ - Counts the number of errors and writes out the output per filename - - :param results: List of filenames (str or Pathlib Path) - :param stageName: Name to decorate the output with - :returns: The number of files which had errors - """ - print("") - errorCount = sum([1 for r in results.values() if not r['ok']]) - logging.info(f"{errorCount} {stageName} errors encountered") - - print(f"{'-':-<60}") - print(f"{stageName} results:") - print(f"{'-':-<60}") - for filename, result in results.items(): - print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") - if not result['ok']: - if isinstance(result['message'], list): - for thing in result['message']: - print(f" {thing['message']}") - else: - print(f" {result['message']}") - - print(f"{'-':-<60}") - print(f"{stageName} errors: {errorCount}") - print(f"{'-':-<60}") - - return errorCount - - -def syntaxCheckXSD (fileList): - results = {} - for file in fileList: - try: - logging.info(f"Syntax checking {str(file)}") - - schema = XMLSchema(str(file), validation="skip") - results[str(file)] = { - 'ok' : len(schema.all_errors) == 0, - 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] - } - except XMLSchemaParseError as ex: - logging.warning(str(file) + ": Failed validation ({0})".format(ex.message)) - results[str(file)] = { - 'ok' : False, - 'message' : f"{ex!r}" - } - return results - - -if __name__ == '__main__': - #logging.basicConfig(level=logging.DEBUG) - - compileTargets = json.loads(Path('testing/xsd_compile_targets.json').read_text()) - results = {} - for target in compileTargets: - coreFile = target['coreSchema'] - logging.info(f"Attempting to compile {coreFile}") - schemaLocations = [] - for supportSchema in target['supportingSchemas']: - logging.debug(f"Adding supporting schema {supportSchema}") - try: - xs = XMLSchema(supportSchema, validation='skip') - schemaLocations.append((xs.target_namespace, str(Path(supportSchema).resolve()))) - logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, supportSchema)) - except Exception as ex: - logging.warning (" [ {0} exception parsing: {1} ]".format(supportSchema, ex)) - results[coreFile] = { - 'ok' : False, - 'message' : f"{ex!r}" - } - break - try: - schema = XMLSchema(coreFile, locations = schemaLocations, validation="strict") - results[coreFile] = { - 'ok' : len(schema.all_errors) == 0, - 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] - } - target["schemaInstance"] = schema - except Exception as ex: - results[coreFile] = { - 'ok' : False, - 'message' : f"{ex!r}" - } - continue - - if (processResults(results, "Compile") > 0): - exit(-1) - - results = {} - - for target in compileTargets: - schema = target["schemaInstance"] - testResults = {} - failureCount = 0 - logging.info (f"Validating example {len(target['exampleFiles'])} entries for {target['coreSchema']}") - for example in target["exampleFiles"]: - examplePath = Path(example) - if examplePath.is_dir(): - logging.debug (f"Expanding {str(examplePath)}") - testFiles = list(examplePath.rglob("./*.xml")) - else: - testFiles = [examplePath] - logging.debug(f"Found {len(testFiles)} test files") - for test in testFiles: - logging.debug(f"Validating {str(test)} against schema") - try: - errors = list(schema.iter_errors(str(test))) - testResults[test] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in errors] - failureCount += len(errors) - except Exception as ex: - testResults[test] = [f"{ex!r}"] - failureCount += 1 - results[target['coreSchema']] = { - 'ok' : failureCount == 0, - 'testResults' : testResults, - 'failureCount' : failureCount - } - - print(f"{'-':-<75}") - print(f"Validation results:") - print(f"{'-':-<75}") - - totalErrors = 0 - for filename, result in results.items(): - if len(result['testResults']) == 0: - print (f"{filename:.<70}SKIP (0)") - continue - else: - print (f"{filename:.<70}{'..OK' if result['ok'] else 'FAIL'} ({len(result['testResults'])})") - totalErrors += result['failureCount'] - if result['failureCount'] > 0: - for testFile, testResult in result['testResults'].items(): - print(f" {str(testFile):.<65}{'..OK' if len(testResult) == 0 else 'FAIL'}") - for tr in testResult: - print(f" {tr}") - - print(f"{'-':-<75}") - print(f"Validation errors: {totalErrors}") - print(f"{'-':-<75}") - - exit(totalErrors > 0) diff --git a/utils/json_to_xml.py b/utils/json_to_xml.py index 17764a7af097aae7a2020522a1ae45adafa7d26a..3f03901c732a010cda374b1ead00664122998141 100644 --- a/utils/json_to_xml.py +++ b/utils/json_to_xml.py @@ -11,8 +11,19 @@ import argparse if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -31,5 +42,7 @@ if __name__ == "__main__": logging.debug(s) j = json.loads(s) - xml = xmltodict.unparse({'HI1Message' : j}, ) - print(xml) \ No newline at end of file + xml = xmltodict.unparse( + {"HI1Message": j}, + ) + print(xml) diff --git a/utils/json_validator.py b/utils/json_validator.py index e9ff0c03a2b121ba5f9a429221288158c0bdcd03..074201ff8d4ae4ac2e81890565babc1780cc4976 100644 --- a/utils/json_validator.py +++ b/utils/json_validator.py @@ -7,10 +7,11 @@ import logging import argparse from itertools import chain + class JsonValidator: - def __init__(self, core_schema: str, other_schemas : dict): + def __init__(self, core_schema: str, other_schemas: dict): self._core_schema = json.load(Path(core_schema).open()) - self._schema_dict = { self._core_schema['$id'] : self._core_schema } + self._schema_dict = {self._core_schema["$id"]: self._core_schema} self._supporting_paths = [] for thing in other_schemas: path = Path(thing) @@ -22,102 +23,186 @@ class JsonValidator: self._supporting_paths.append(path) logging.info(f"Supporting schema paths: {self._supporting_paths}") self._supporting_schemas = [json.load(p.open()) for p in self._supporting_paths] - self._schema_dict = self._schema_dict | { s['$id'] : s for s in self._supporting_schemas } + self._schema_dict = self._schema_dict | { + s["$id"]: s for s in self._supporting_schemas + } logging.info(f"Loaded schema IDs: {[k for k in self._schema_dict.keys()]}") - self._resolver = RefResolver(None, - referrer=None, - store=self._schema_dict) + self._resolver = RefResolver(None, referrer=None, store=self._schema_dict) logging.info("Created RefResolver") - self._validator = Draft202012Validator(self._core_schema, resolver=self._resolver) + self._validator = Draft202012Validator( + self._core_schema, resolver=self._resolver + ) logging.info("Created validator") def validate(self, instance_doc: str): errors = list(self._validator.iter_errors(instance_doc)) return errors - -class TS103120Validator (JsonValidator): - def __init__ (self, path_to_repo): + + +class TS103120Validator(JsonValidator): + def __init__(self, path_to_repo): repo_path = Path(path_to_repo) schema_dirs = [str(repo_path / "103120/schema/json"), str("103280/")] core_schema = str(repo_path / "103120/schema/json/ts_103120_Core.schema.json") JsonValidator.__init__(self, core_schema, schema_dirs) - request_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/RequestPayload" } - self._request_fragment_validator = Draft202012Validator(request_fragment_schema, resolver=self._resolver) - response_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/ResponsePayload" } - self._response_fragment_validator = Draft202012Validator(response_fragment_schema, resolver=self._resolver) - - def expand_request_response_exception (self, ex): - if list(ex.schema_path) == ['properties', 'Payload', 'oneOf']: - logging.info ("Error detected validating payload oneOf - attempting explicit validation...") - if 'RequestPayload' in instance_doc['Payload'].keys(): - ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['RequestPayload'])])) + request_fragment_schema = { + "$ref": "ts_103120_Core_2019_10#/$defs/RequestPayload" + } + self._request_fragment_validator = Draft202012Validator( + request_fragment_schema, resolver=self._resolver + ) + response_fragment_schema = { + "$ref": "ts_103120_Core_2019_10#/$defs/ResponsePayload" + } + self._response_fragment_validator = Draft202012Validator( + response_fragment_schema, resolver=self._resolver + ) + + def expand_request_response_exception(self, ex): + if list(ex.schema_path) == ["properties", "Payload", "oneOf"]: + logging.info( + "Error detected validating payload oneOf - attempting explicit validation..." + ) + if "RequestPayload" in instance_doc["Payload"].keys(): + ret_list = list( + chain( + *[ + self.expand_action_exception(x) + for x in self._request_fragment_validator.iter_errors( + instance_doc["Payload"]["RequestPayload"] + ) + ] + ) + ) for r in ret_list: r.path = ex.path + r.path return ret_list - elif 'ResponsePayload' in instance_doc['Payload'].keys(): - ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['ResponsePayload'])])) + elif "ResponsePayload" in instance_doc["Payload"].keys(): + ret_list = list( + chain( + *[ + self.expand_action_exception(x) + for x in self._request_fragment_validator.iter_errors( + instance_doc["Payload"]["ResponsePayload"] + ) + ] + ) + ) for r in ret_list: r.path = ex.path + r.path return ret_list else: - logging.error("No RequestPayload or ResponsePayload found - is the Payload malformed?") + logging.error( + "No RequestPayload or ResponsePayload found - is the Payload malformed?" + ) return [ex] else: return [ex] - - def expand_action_exception (self, ex): + + def expand_action_exception(self, ex): logging.error("Error detected in ActionRequests/ActionResponses") error_path = list(ex.schema_path) - if error_path != ['properties', 'ActionRequests', 'properties', 'ActionRequest', 'items', 'allOf', 1, 'oneOf'] and error_path != ['properties', 'ActionResponses', 'properties', 'ActionResponse', 'items', 'allOf', 1, 'oneOf']: + if error_path != [ + "properties", + "ActionRequests", + "properties", + "ActionRequest", + "items", + "allOf", + 1, + "oneOf", + ] and error_path != [ + "properties", + "ActionResponses", + "properties", + "ActionResponse", + "items", + "allOf", + 1, + "oneOf", + ]: logging.error("Error not in inner Request/Response allOf/oneOf constraint") - return[ex] + return [ex] j = ex.instance - j.pop('ActionIdentifier') # Remove ActionIdentifier - one remaining key will be the verb + j.pop( + "ActionIdentifier" + ) # Remove ActionIdentifier - one remaining key will be the verb verb = list(j.keys())[0] message = "Request" if error_path[1] == "ActionRequests" else "Response" - v = Draft202012Validator({"$ref" : f"ts_103120_Core_2019_10#/$defs/{verb}{message}"}, resolver=self._resolver) - ret_list = list(chain(*[self.expand_object_exception(x) for x in v.iter_errors(j[verb])])) + v = Draft202012Validator( + {"$ref": f"ts_103120_Core_2019_10#/$defs/{verb}{message}"}, + resolver=self._resolver, + ) + ret_list = list( + chain(*[self.expand_object_exception(x) for x in v.iter_errors(j[verb])]) + ) for r in ret_list: r.path = ex.path + r.path return ret_list - - def expand_object_exception (self, ex): + + def expand_object_exception(self, ex): logging.error("Error detected in verb") # The final level of validation is for the actual HI1Object validation - if list(ex.schema_path) != ['properties', 'HI1Object', 'oneOf']: + if list(ex.schema_path) != ["properties", "HI1Object", "oneOf"]: logging.error("Error not inside HI1Object") return [ex] - object_type = ex.instance['@xsi:type'].split('}')[-1] + object_type = ex.instance["@xsi:type"].split("}")[-1] object_ref = { - 'AuthorisationObject': 'ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject', - 'LITaskObject': 'ts_103120_Task_2020_09#/$defs/LITaskObject', - 'LDTaskObject': 'ts_103120_Task_2020_09#/$defs/LDTaskObject', - 'LPTaskObject': 'ts_103120_Task_2020_09#/$defs/LPTaskObject', - 'DocumentObject': 'ts_103120_Document_2020_09#/$defs/DocumentObject', - 'NotificationObject': 'ts_103120_Notification_2016_02#/$defs/NotificationObject', - 'DeliveryObject': 'ts_103120_Delivery_2019_10#/$defs/DeliveryObject', - 'TrafficPolicyObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject', - 'TrafficRuleObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject', + "AuthorisationObject": "ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject", + "LITaskObject": "ts_103120_Task_2020_09#/$defs/LITaskObject", + "LDTaskObject": "ts_103120_Task_2020_09#/$defs/LDTaskObject", + "LPTaskObject": "ts_103120_Task_2020_09#/$defs/LPTaskObject", + "DocumentObject": "ts_103120_Document_2020_09#/$defs/DocumentObject", + "NotificationObject": "ts_103120_Notification_2016_02#/$defs/NotificationObject", + "DeliveryObject": "ts_103120_Delivery_2019_10#/$defs/DeliveryObject", + "TrafficPolicyObject": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject", + "TrafficRuleObject": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject", }[object_type] - v = Draft202012Validator({"$ref" : object_ref}, resolver=self._resolver) + v = Draft202012Validator({"$ref": object_ref}, resolver=self._resolver) return list(v.iter_errors(ex.instance)) - + def validate(self, instance_doc: str): errors = JsonValidator.validate(self, instance_doc) - out_errors = list(chain(*[self.expand_request_response_exception(ex) for ex in errors])) + out_errors = list( + chain(*[self.expand_request_response_exception(ex) for ex in errors]) + ) return out_errors - if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") - parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") - parser.add_argument('--ts103120', action="store_true", help="Validate a TS 103 120 JSON document") - parser.add_argument('--schema', default=None, help="Primary schema to validate against") - parser.add_argument('-p', '--printerror', action="count", help="Controls how verbose validation error printing is (can be specified multiple times)") + parser.add_argument( + "-s", + "--schemadir", + action="append", + help="Directory containing supporting schema files to use for validation", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) + parser.add_argument( + "--ts103120", action="store_true", help="Validate a TS 103 120 JSON document" + ) + parser.add_argument( + "--schema", default=None, help="Primary schema to validate against" + ) + parser.add_argument( + "-p", + "--printerror", + action="count", + help="Controls how verbose validation error printing is (can be specified multiple times)", + ) args = parser.parse_args() match args.verbose: @@ -130,7 +215,7 @@ if __name__ == "__main__": logging.debug(f"Arguments: {args}") - if (args.ts103120): + if args.ts103120: v = TS103120Validator("./") else: v = JsonValidator(args.schema, args.schemadir) diff --git a/utils/sign_json.py b/utils/sign_json.py index 1ce0bba5b21d00ff86bcab7d680425451d0e24f8..96439d468fc42d0beb29699f4a37ab827f8a6109 100644 --- a/utils/sign_json.py +++ b/utils/sign_json.py @@ -1,4 +1,3 @@ - import argparse import logging import sys @@ -8,18 +7,31 @@ from pathlib import Path import json -def insert_sig_block (j): - j['Signature'] = { - 'protected' : '', - 'signature' : '' - } +def insert_sig_block(j): + j["Signature"] = {"protected": "", "signature": ""} return j + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('--pretty', action="store_true", help='Pretty-print the JSON document before signing') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "--pretty", + action="store_true", + help="Pretty-print the JSON document before signing", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -34,24 +46,22 @@ if __name__ == "__main__": json_text = args.input.read() args.input.close() - + j = json.loads(json_text) j = insert_sig_block(j) - + indent = None if args.pretty: - indent = ' ' + indent = " " presigned_json_text = json.dumps(j, indent=indent) - Path('presigned.json').write_text(presigned_json_text) - presigned_json_bytes = presigned_json_text.encode('utf-8') - - signed = jws.sign(presigned_json_bytes, 'secret_key', algorithm="HS256") - components = signed.split('.') + Path("presigned.json").write_text(presigned_json_text) + presigned_json_bytes = presigned_json_text.encode("utf-8") + + signed = jws.sign(presigned_json_bytes, "secret_key", algorithm="HS256") + components = signed.split(".") - j['Signature']['protected'] = components[0] - j['Signature']['signature'] = components[2] + j["Signature"]["protected"] = components[0] + j["Signature"]["signature"] = components[2] signed_json_text = json.dumps(j, indent=indent) print(signed_json_text) - - diff --git a/utils/translate/ChoiceMapping.py b/utils/translate/ChoiceMapping.py index b477a3360e0a6a7b891f5854951eb4f37dce4b39..16c396247183e28a437e9204e1cacd6c93ab999e 100644 --- a/utils/translate/ChoiceMapping.py +++ b/utils/translate/ChoiceMapping.py @@ -10,38 +10,43 @@ from .ComplexTypeMapping import ComplexTypeMapping log = logging.getLogger() + class ChoiceMapping(ComplexTypeMapping): @classmethod - def process_choice(cls, choice: XsdGroup, current_ns : str, ns_to_id_map): - if choice.model != 'choice': + def process_choice(cls, choice: XsdGroup, current_ns: str, ns_to_id_map): + if choice.model != "choice": raise Exception(f"Wrong group type: {c.model}") oneOf = [] for c in choice.iter_model(): if not (type(c) is XsdElement): - raise Exception (f"Non-element {c} encountered in choice {choice}") + raise Exception(f"Non-element {c} encountered in choice {choice}") element_name = c.local_name if c.target_namespace in ns_to_id_map: ns = ns_to_id_map[c.target_namespace] - if 'prefix' in ns: - element_name = ns['prefix'] + ":" + element_name + if "prefix" in ns: + element_name = ns["prefix"] + ":" + element_name t = TypeMapping.get_type_from_elem(c, current_ns) - oneOf.append({ - "type" : "object", - "properties" : { - element_name : t - }, - "required" : [element_name] - }) - return oneOf + oneOf.append( + { + "type": "object", + "properties": {element_name: t}, + "required": [element_name], + } + ) + return oneOf - def map(self, xst : BaseXsdType): + def map(self, xst: BaseXsdType): log.debug(f"Attempting mapping of {xst} to choice") j = super().map(xst) if j is None: log.debug("Not a complex type, giving up") return None content = xst.content - if (content.model != 'choice'): + if content.model != "choice": log.debug("Not a choice, giving up") return None - return { 'oneOf' : ChoiceMapping.process_choice(content, xst.namespaces[''], self.ns_to_id_map)} + return { + "oneOf": ChoiceMapping.process_choice( + content, xst.namespaces[""], self.ns_to_id_map + ) + } diff --git a/utils/translate/ComplexTypeMapping.py b/utils/translate/ComplexTypeMapping.py index e18190901be2c3e0695c00a772be5364cce591d1..19064e2d5106c4098ac7f26524ed7cd6a12785e1 100644 --- a/utils/translate/ComplexTypeMapping.py +++ b/utils/translate/ComplexTypeMapping.py @@ -2,10 +2,9 @@ from xmlschema.validators.complex_types import * from .TypeMapping import TypeMapping + class ComplexTypeMapping(TypeMapping): def map(self, xst: BaseXsdType): if not (type(xst) is XsdComplexType): return None - return { - "type" : "object" - } + return {"type": "object"} diff --git a/utils/translate/SequenceMapping.py b/utils/translate/SequenceMapping.py index 68f76094121997a2960a9f6a507a542aaaedadce..c70e509bb20c97b721ea95f230e0ed60a3787d52 100644 --- a/utils/translate/SequenceMapping.py +++ b/utils/translate/SequenceMapping.py @@ -20,14 +20,14 @@ class SequenceMapping(ComplexTypeMapping): log.debug("Not a complex type, giving up") return None content = xst.content - if (content.model != 'sequence'): + if content.model != "sequence": log.debug("Not a sequence, giving up") return None mapped_type = { - 'type' : 'object', - 'properties' : {}, - 'required' : [], - 'additionalProperties' : False + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": False, } # Not going to try and do all of this automatically for now @@ -35,11 +35,11 @@ class SequenceMapping(ComplexTypeMapping): # HACK exception for ApproverContactDetails because it isn't abstract if (xst.base_type) and (xst.local_name != "ApproverContactDetails"): # mapped_type['__DESCENDENT_OF__'] = TypeMapping.get_ref_for(xst.base_type, xst.namespaces['']) - mapped_type['properties']['@xsi:type'] = { - "type" : "string", - "enum" : [xst.name] + mapped_type["properties"]["@xsi:type"] = { + "type": "string", + "enum": [xst.name], } - mapped_type['required'].append('@xsi:type') + mapped_type["required"].append("@xsi:type") # if xst.abstract: # mapped_type['__ABSTRACT__'] = True # pass @@ -51,41 +51,50 @@ class SequenceMapping(ComplexTypeMapping): element_name = c.local_name if c.target_namespace in self.ns_to_id_map: ns = self.ns_to_id_map[c.target_namespace] - if 'prefix' in ns: - element_name = ns['prefix'] + ":" + element_name + if "prefix" in ns: + element_name = ns["prefix"] + ":" + element_name if c.effective_max_occurs != 1: - mapped_type['properties'][element_name] = { - "type" : "array", - "items" : TypeMapping.get_type_from_elem(c, xst.namespaces['']) + mapped_type["properties"][element_name] = { + "type": "array", + "items": TypeMapping.get_type_from_elem(c, xst.namespaces[""]), } if c.effective_max_occurs: - mapped_type['properties'][element_name]['maxItems'] = c.effective_max_occurs + mapped_type["properties"][element_name]["maxItems"] = ( + c.effective_max_occurs + ) if c.effective_min_occurs > 0: - mapped_type['properties'][element_name]['minItems'] = c.effective_min_occurs + mapped_type["properties"][element_name]["minItems"] = ( + c.effective_min_occurs + ) else: - mapped_type['properties'][element_name] = TypeMapping.get_type_from_elem(c, xst.namespaces['']) + mapped_type["properties"][element_name] = ( + TypeMapping.get_type_from_elem(c, xst.namespaces[""]) + ) if c.effective_min_occurs == 1: - mapped_type['required'].append(element_name) + mapped_type["required"].append(element_name) elif type(c) is XsdGroup: if inner_choice: - raise Exception (f"Second group '{element_name}' encountered in {xst}") + raise Exception( + f"Second group '{element_name}' encountered in {xst}" + ) if c.model != "choice": - raise Exception (f"Don't know what to do with inner group {c} in {xst} - not a choice") - inner_choice = ChoiceMapping.process_choice(c, xst.namespaces[''], self.ns_to_id_map) + raise Exception( + f"Don't know what to do with inner group {c} in {xst} - not a choice" + ) + inner_choice = ChoiceMapping.process_choice( + c, xst.namespaces[""], self.ns_to_id_map + ) elif type(c) is XsdAnyElement: mapped_type = {} else: raise Exception(f"Unknown element type {c}") - if (inner_choice): + if inner_choice: for inner_thing in inner_choice: - inner_thing.pop('additionalProperties', None) - mapped_type.pop('additionalProperties', None) - return { - 'allOf' : [ - mapped_type, - {'oneOf' : inner_choice} - ], - 'unevaluatedProperties' : False + inner_thing.pop("additionalProperties", None) + mapped_type.pop("additionalProperties", None) + return { + "allOf": [mapped_type, {"oneOf": inner_choice}], + "unevaluatedProperties": False, } else: return mapped_type diff --git a/utils/translate/SimpleTypeMapping.py b/utils/translate/SimpleTypeMapping.py index 2e60f9ca06b321a28992c2a32235707671f95735..70edfb8ce0367c1046482f8056bbe78d67afe6fd 100644 --- a/utils/translate/SimpleTypeMapping.py +++ b/utils/translate/SimpleTypeMapping.py @@ -7,12 +7,11 @@ from .TypeMapping import TypeMapping log = logging.getLogger() + class SimpleTypeMapping(TypeMapping): def map(self, xst: BaseXsdType): log.debug(f"Attempting mapping of {xst} to simple type") if not (type(xst) is XsdAtomicRestriction): log.debug("Type is not an XsdAtomicRestriction, giving up") return None - return { - "$ref" : xst.base_type.name - } \ No newline at end of file + return {"$ref": xst.base_type.name} diff --git a/utils/translate/TypeMapping.py b/utils/translate/TypeMapping.py index 2b4b785ceeedb631c98ecf63df5c8d9754848b29..57a367f6f31935b31234b2d9759569dcf13455ee 100644 --- a/utils/translate/TypeMapping.py +++ b/utils/translate/TypeMapping.py @@ -8,63 +8,52 @@ from xmlschema.validators.facets import * log = logging.getLogger() + class TypeMapping(ABC): ns_to_id_map = {} XSD_NS = "http://www.w3.org/2001/XMLSchema" XSD_TYPE_MAP = { - "string" : { "type" : "string" }, - "normalizedString" : { "type" : "string"}, - "dateTime" : { "type" : "string"}, - "token" : { "type" : "string"}, - "anyURI" : { "type" : "string" }, - - "integer" : { "type" : "integer"}, - "nonNegativeInteger" : { "type" : "integer", "minimum" : 0}, - "positiveInteger" : { "type" : "integer", "minimum" : 1}, - - "boolean" : { "type" : "boolean" }, - - "hexBinary" : { "type" : "string", "pattern" : "^([a-fA-F0-9]{2})*$"}, - "base64Binary" : { "type" : "string", "pattern" : "^[A-Za-z0-9+\/]*={0,3}$"}, - - "anyType" : {} - } + "string": {"type": "string"}, + "normalizedString": {"type": "string"}, + "dateTime": {"type": "string"}, + "token": {"type": "string"}, + "anyURI": {"type": "string"}, + "integer": {"type": "integer"}, + "nonNegativeInteger": {"type": "integer", "minimum": 0}, + "positiveInteger": {"type": "integer", "minimum": 1}, + "boolean": {"type": "boolean"}, + "hexBinary": {"type": "string", "pattern": "^([a-fA-F0-9]{2})*$"}, + "base64Binary": {"type": "string", "pattern": "^[A-Za-z0-9+\/]*={0,3}$"}, + "anyType": {}, + } @abstractmethod - def map(self, xst : BaseXsdType): + def map(self, xst: BaseXsdType): return None @classmethod def extract_namespace(cls, qname: str): - match = re.search(r'^\{([^\{\}]+)\}(([^\{\}]+))$', qname) + match = re.search(r"^\{([^\{\}]+)\}(([^\{\}]+))$", qname) if match is None: return None return match.group(1) @classmethod - def get_ref_for(cls, xsd_type: XsdType, current_ns : str): + def get_ref_for(cls, xsd_type: XsdType, current_ns: str): ns = cls.extract_namespace(xsd_type.name) if ns == current_ns: - return { "$ref" : f"#/$defs/{xsd_type.local_name}" } + return {"$ref": f"#/$defs/{xsd_type.local_name}"} else: mapped_id = cls.ns_to_id_map[ns] - return { "$ref" : f"{mapped_id['id']}#/$defs/{xsd_type.local_name}"} + return {"$ref": f"{mapped_id['id']}#/$defs/{xsd_type.local_name}"} @classmethod - def get_type_from_elem(cls, elem: XsdElement, current_ns : str): + def get_type_from_elem(cls, elem: XsdElement, current_ns: str): ns = cls.extract_namespace(elem.type.name) - if (ns == TypeMapping.XSD_NS): + if ns == TypeMapping.XSD_NS: # this should be an XSD primitive type return dict(TypeMapping.XSD_TYPE_MAP[elem.type.local_name]) else: return cls.get_ref_for(elem.type, current_ns) - - - - - - - - diff --git a/utils/translate/XSDNativeSimpleTypeMapping.py b/utils/translate/XSDNativeSimpleTypeMapping.py index 772ac10b308d4c2128bcf1bac3271117d9bbe80e..bd4d2119bec8c73f8688b8b0c769a6df54ce9d98 100644 --- a/utils/translate/XSDNativeSimpleTypeMapping.py +++ b/utils/translate/XSDNativeSimpleTypeMapping.py @@ -10,8 +10,8 @@ from .SimpleTypeMapping import SimpleTypeMapping log = logging.getLogger() -class XSDNativeSimpleTypeMapping(SimpleTypeMapping): +class XSDNativeSimpleTypeMapping(SimpleTypeMapping): def map(self, xst: BaseXsdType): log.debug(f"Attempting mapping of {xst} to XSD native type") j = super().map(xst) @@ -25,48 +25,50 @@ class XSDNativeSimpleTypeMapping(SimpleTypeMapping): if mapped_type is None: ns = TypeMapping.extract_namespace(xst.base_type.name) if ns == XSDNativeSimpleTypeMapping.XSD_NS: - print (xst) - print (xst.base_type) - raise Exception (f"No mapping for xs:{xst.base_type.local_name}") + print(xst) + print(xst.base_type) + raise Exception(f"No mapping for xs:{xst.base_type.local_name}") if len(xst.facets) == 0: - mapped_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces['']) + mapped_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces[""]) else: - parent_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces['']) + parent_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces[""]) mapped_type = TypeMapping.XSD_TYPE_MAP.get(xst.root_type.local_name) if mapped_type is None: - raise Exception (f"Could not find mapping for root type xs:{xst.root_type.local_name}") + raise Exception( + f"Could not find mapping for root type xs:{xst.root_type.local_name}" + ) mapped_type = dict(mapped_type) - + for k, v in xst.facets.items(): log.debug(f"Mapping facet {v}") if type(v) is XsdMaxLengthFacet: - mapped_type['maxLength'] = v.value + mapped_type["maxLength"] = v.value continue if type(v) is XsdMinLengthFacet: - mapped_type['minLength'] = v.value + mapped_type["minLength"] = v.value continue if type(v) is XsdPatternFacets: if len(v.regexps) > 1: - raise Exception (f"Multiple patterns given in facet {v} of {xst}") + raise Exception(f"Multiple patterns given in facet {v} of {xst}") p = v.regexps[0] - if (not p.startswith('^')) and (not p.endswith('$')): + if (not p.startswith("^")) and (not p.endswith("$")): p = f"^{p}$" - mapped_type['pattern'] = p + mapped_type["pattern"] = p continue - if type (v) is XsdMinInclusiveFacet: - mapped_type['minimum'] = v.value + if type(v) is XsdMinInclusiveFacet: + mapped_type["minimum"] = v.value continue - if type (v) is XsdMaxInclusiveFacet: - mapped_type['maximum'] = v.value + if type(v) is XsdMaxInclusiveFacet: + mapped_type["maximum"] = v.value continue - if type (v) is XsdMinExclusiveFacet: - mapped_type['exclusiveMinimum'] = v.value + if type(v) is XsdMinExclusiveFacet: + mapped_type["exclusiveMinimum"] = v.value continue - if type (v) is XsdMaxExclusiveFacet: - mapped_type['exclusiveMaximum'] = v.value + if type(v) is XsdMaxExclusiveFacet: + mapped_type["exclusiveMaximum"] = v.value continue - raise Exception (f"Unhandled facet {v}") + raise Exception(f"Unhandled facet {v}") if parent_type: - return { 'allOf' : [parent_type, mapped_type] } + return {"allOf": [parent_type, mapped_type]} return mapped_type diff --git a/utils/translate/__init__.py b/utils/translate/__init__.py index 86a33468c8e281b7d9a745261fc96a1b7905e17f..bf2d08da8d0255fdfec576903308fbe701c260b0 100644 --- a/utils/translate/__init__.py +++ b/utils/translate/__init__.py @@ -17,30 +17,33 @@ mappings = [ SequenceMapping(), ] -def translate_schema (schema_path: str, ns_to_id_map: dict, schema_locations = []): + +def translate_schema(schema_path: str, ns_to_id_map: dict, schema_locations=[]): js = { - "$id" : "?", - "$schema" : "https://json-schema.org/draft/2020-12/schema", - "$defs" : {} + "$id": "?", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": {}, } logging.info(f"Translating schema {schema_path}") - xs = XMLSchema(schema_path, validation='lax', locations=schema_locations) - logging.info(f"Schema namespace: {xs.target_namespace}" ) + xs = XMLSchema(schema_path, validation="lax", locations=schema_locations) + logging.info(f"Schema namespace: {xs.target_namespace}") schema_id = ns_to_id_map[xs.target_namespace]["id"] - js['$id'] = schema_id + js["$id"] = schema_id TypeMapping.ns_to_id_map = ns_to_id_map elementList = [] for elementName, element in xs.elements.items(): logging.info(f"Processing element {elementName} : {element}") - elementList.append(TypeMapping.get_ref_for(element.type, element.namespaces[''])) + elementList.append( + TypeMapping.get_ref_for(element.type, element.namespaces[""]) + ) if len(elementList) == 1: - js['$ref'] = elementList[0]['$ref'] + js["$ref"] = elementList[0]["$ref"] elif len(elementList) > 1: - js['oneOf'] = elementList + js["oneOf"] = elementList descendent_types = {} for type_name, xsd_type in xs.types.items(): @@ -57,9 +60,7 @@ def translate_schema (schema_path: str, ns_to_id_map: dict, schema_locations = [ if j is None: raise Exception(f"Unmapped type {type_name} ({xsd_type})") js["$defs"][xsd_type.local_name] = j - logging.debug (f"Mapped {type_name} to {j}") + logging.debug(f"Mapped {type_name} to {j}") - print (descendent_types) + print(descendent_types) return js - - diff --git a/utils/translate_spec.py b/utils/translate_spec.py index f78b8fffdaf6264eb36e3126e6e0a101f08292b3..6ce2f843a17646a88ffde26041a2068805955b8f 100644 --- a/utils/translate_spec.py +++ b/utils/translate_spec.py @@ -8,68 +8,68 @@ from xmlschema import * from translate import * -logging.basicConfig(level = logging.INFO) +logging.basicConfig(level=logging.INFO) json_signature_struct = { - "properties" : { - "protected" : { "type" : "string" }, - "signature" : { "type" : "string" } - }, - "required" : ["protected", "signature" ] + "properties": {"protected": {"type": "string"}, "signature": {"type": "string"}}, + "required": ["protected", "signature"], } -def build_schema_locations (paths): + +def build_schema_locations(paths): schema_locations = [] for schemaFile in paths: try: - xs = XMLSchema(schemaFile, validation='skip') - schema_locations.append((xs.target_namespace, str(Path(schemaFile).resolve()))) - logging.debug (" [ {0} -> {1} ]".format(xs.target_namespace, schemaFile)) + xs = XMLSchema(schemaFile, validation="skip") + schema_locations.append( + (xs.target_namespace, str(Path(schemaFile).resolve())) + ) + logging.debug(" [ {0} -> {1} ]".format(xs.target_namespace, schemaFile)) except XMLSchemaParseError as ex: - logging.debug (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) + logging.debug(" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) return schema_locations + def get_json(filename): with open(filename) as f: j = json.load(f) return j -def convert_ns_to_id (ns): - if ns.startswith('http://uri.etsi.org'): + +def convert_ns_to_id(ns): + if ns.startswith("http://uri.etsi.org"): c = ns.split("/") return f"ts_1{c[3]}{'_' + c[7] if len(c) > 7 else ''}_{c[5]}_{c[6]}" else: - return ns.replace("http://","").replace("/","_") + return ns.replace("http://", "").replace("/", "_") + -def convert_xsd_to_filename (xsd): +def convert_xsd_to_filename(xsd): f = Path(xsd) - return f.name.replace('.xsd', '.schema.json') + return f.name.replace(".xsd", ".schema.json") + if __name__ == "__main__": if len(sys.argv) < 2: - logging.error ("Usage: translate_spec.py path_to_config_file") + logging.error("Usage: translate_spec.py path_to_config_file") exit(-1) config = get_json(sys.argv[1]) - logging.info("Bulding ns map...") ns_map = {} - for location, settings in config['schemas'].items(): - xs = XMLSchema(location, validation='skip') + for location, settings in config["schemas"].items(): + xs = XMLSchema(location, validation="skip") ns = xs.target_namespace id = convert_ns_to_id(ns) - ns_map[ns] = { - "id" : id, - "location" : str(Path(location).resolve()) - } | settings + ns_map[ns] = {"id": id, "location": str(Path(location).resolve())} | settings logging.debug(ns_map) - + logging.info("Building schema locations") - schema_locations = [(k, v["location"]) for k,v in ns_map.items()] + schema_locations = [(k, v["location"]) for k, v in ns_map.items()] logging.debug(schema_locations) - output_path = Path(config['output']) + output_path = Path(config["output"]) if not output_path.exists(): logging.info("Creating output directory") os.mkdir(str(output_path)) @@ -78,45 +78,56 @@ if __name__ == "__main__": json_schemas = {} for schema_tuple in schema_locations: logging.info(f" Translating {schema_tuple}") - if 'skip' in ns_map[schema_tuple[0]]: + if "skip" in ns_map[schema_tuple[0]]: logging.info(f" Skipping {schema_tuple[0]}...") continue js = translate_schema(schema_tuple[1], ns_map, schema_locations) # TODO - Special case, get rid of XML Dsig signature and insert JSON signature - if schema_tuple[0] == 'http://uri.etsi.org/03120/common/2019/10/Core': - logging.info ("Modifying signature elements") - js['$defs']['HI1Message']['patternProperties'] = { "^@" : { "type" : "string"}} - js['$defs']['HI1Message']['properties'].pop('xmldsig:Signature') - js['$defs']['HI1Message']['properties']['Signature'] = json_signature_struct - - - if 'output' in ns_map[schema_tuple[0]]: - js_path = Path(ns_map[schema_tuple[0]]['output']) / convert_xsd_to_filename(schema_tuple[1]) + if schema_tuple[0] == "http://uri.etsi.org/03120/common/2019/10/Core": + logging.info("Modifying signature elements") + js["$defs"]["HI1Message"]["patternProperties"] = {"^@": {"type": "string"}} + js["$defs"]["HI1Message"]["properties"].pop("xmldsig:Signature") + js["$defs"]["HI1Message"]["properties"]["Signature"] = json_signature_struct + + if "output" in ns_map[schema_tuple[0]]: + js_path = Path(ns_map[schema_tuple[0]]["output"]) / convert_xsd_to_filename( + schema_tuple[1] + ) else: js_path = output_path / convert_xsd_to_filename(schema_tuple[1]) # TODO - Special case - abstract HI1Object if "Core" in schema_tuple[1]: - js["$defs"]['ConcreteHI1Object'] = { - 'oneOf' : [ - {'$ref' : 'ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject'}, - {'$ref' : 'ts_103120_Task_2020_09#/$defs/LITaskObject'}, - {'$ref' : 'ts_103120_Task_2020_09#/$defs/LPTaskObject'}, - {'$ref' : 'ts_103120_Task_2020_09#/$defs/LDTaskObject'}, - {'$ref' : 'ts_103120_Document_2020_09#/$defs/DocumentObject'}, - {'$ref' : 'ts_103120_Notification_2016_02#/$defs/NotificationObject'}, - {'$ref' : 'ts_103120_Delivery_2019_10#/$defs/DeliveryObject'}, - {'$ref' : 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject'}, - {'$ref' : 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject'}, + js["$defs"]["ConcreteHI1Object"] = { + "oneOf": [ + { + "$ref": "ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject" + }, + {"$ref": "ts_103120_Task_2020_09#/$defs/LITaskObject"}, + {"$ref": "ts_103120_Task_2020_09#/$defs/LPTaskObject"}, + {"$ref": "ts_103120_Task_2020_09#/$defs/LDTaskObject"}, + {"$ref": "ts_103120_Document_2020_09#/$defs/DocumentObject"}, + { + "$ref": "ts_103120_Notification_2016_02#/$defs/NotificationObject" + }, + {"$ref": "ts_103120_Delivery_2019_10#/$defs/DeliveryObject"}, + { + "$ref": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject" + }, + { + "$ref": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject" + }, ] } json_string = json.dumps(js, indent=2) + "\n" if "Core" in schema_tuple[1]: - json_string = json_string.replace('"$ref": "#/$defs/HI1Object"', '"$ref": "#/$defs/ConcreteHI1Object"') + json_string = json_string.replace( + '"$ref": "#/$defs/HI1Object"', '"$ref": "#/$defs/ConcreteHI1Object"' + ) - with open(str(js_path), 'w', newline='\n') as f: + with open(str(js_path), "w", newline="\n") as f: f.write(json_string) - json_schemas[js['$id']] = json.loads(json_string) + json_schemas[js["$id"]] = json.loads(json_string) diff --git a/utils/ts103120_config.json b/utils/ts103120_config.json index 00043ffc2d594bc62e0304b99e9215364eb94a1a..6a89326662617d392906b85b6719b9203beaae44 100644 --- a/utils/ts103120_config.json +++ b/utils/ts103120_config.json @@ -1,43 +1,42 @@ { - "schemas" : { - "./103120/dictionaries/ts_103120_Dictionaries.xsd" : { - "prefix" : "dictionaries", - "output" : "./103120/dictionaries/" - }, - "./103120/schema/xsd/ts_103120_Authorisation.xsd" : { - "prefix" : "auth" - }, - "./103120/schema/xsd/ts_103120_Common.xsd" : { - "prefix" : "common" - }, - "./103120/schema/xsd/ts_103120_Config.xsd" : { - "prefix" : "config" - }, - "./103120/schema/xsd/ts_103120_Core.xsd" : { - }, - "./103120/schema/xsd/ts_103120_Delivery.xsd" : { - "prefix" : "delivery" - }, - "./103120/schema/xsd/ts_103120_Document.xsd" : { - "prefix" : "doc" - }, - "./103120/schema/xsd/ts_103120_Notification.xsd" : { - "prefix" : "notification" - }, - "./103120/schema/xsd/ts_103120_Task.xsd" : { - "prefix" : "task" - }, - "./103120/schema/xsd/ts_103120_TrafficPolicy.xsd" : { - "prefix" : "tp" - }, - "./103280/TS_103_280.xsd" : { - "prefix" : "etsi280", - "skip" : true - }, - "./testing/deps/xmldsig/xmldsig-core-schema.xsd" : { - "prefix" : "xmldsig", - "skip" : true - } - }, - "output" : "./103120/schema/json/" + "schemas": { + "./103120/dictionaries/ts_103120_Dictionaries.xsd": { + "prefix": "dictionaries", + "output": "./103120/dictionaries/" + }, + "./103120/schema/xsd/ts_103120_Authorisation.xsd": { + "prefix": "auth" + }, + "./103120/schema/xsd/ts_103120_Common.xsd": { + "prefix": "common" + }, + "./103120/schema/xsd/ts_103120_Config.xsd": { + "prefix": "config" + }, + "./103120/schema/xsd/ts_103120_Core.xsd": {}, + "./103120/schema/xsd/ts_103120_Delivery.xsd": { + "prefix": "delivery" + }, + "./103120/schema/xsd/ts_103120_Document.xsd": { + "prefix": "doc" + }, + "./103120/schema/xsd/ts_103120_Notification.xsd": { + "prefix": "notification" + }, + "./103120/schema/xsd/ts_103120_Task.xsd": { + "prefix": "task" + }, + "./103120/schema/xsd/ts_103120_TrafficPolicy.xsd": { + "prefix": "tp" + }, + "./103280/TS_103_280.xsd": { + "prefix": "etsi280", + "skip": true + }, + "./testing/deps/xmldsig/xmldsig-core-schema.xsd": { + "prefix": "xmldsig", + "skip": true + } + }, + "output": "./103120/schema/json/" } diff --git a/utils/ts103280_config.json b/utils/ts103280_config.json index d36a86ebc4c83efe4ce623e60f06d76c4beb7953..cdbb5199766e3457e87dcb9c8e8fad9a7fb41a58 100644 --- a/utils/ts103280_config.json +++ b/utils/ts103280_config.json @@ -1,8 +1,8 @@ { - "schemas" : { - "./103280/TS_103_280.xsd" : { - "prefix" : "etsi280" - } - }, - "output" : "./103280/" + "schemas": { + "./103280/TS_103_280.xsd": { + "prefix": "etsi280" + } + }, + "output": "./103280/" } diff --git a/utils/verify_json.py b/utils/verify_json.py index 329c0692499ee7fd5bc4c36b81e1cebabc3d9d68..5f7408017e872ff6f7c8f9c38ab557b45a41751f 100644 --- a/utils/verify_json.py +++ b/utils/verify_json.py @@ -1,4 +1,3 @@ - import argparse import sys import logging @@ -11,8 +10,19 @@ import json if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -27,28 +37,30 @@ if __name__ == "__main__": signed_json_text = args.input.read() args.input.close() - + j = json.loads(signed_json_text) - - protected_header = j['Signature']['protected'] - signature = j['Signature']['signature'] + + protected_header = j["Signature"]["protected"] + signature = j["Signature"]["signature"] # TODO some safety checks needed here # Remove the newline that appears from the console - if signed_json_text.endswith('\n'): signed_json_text = signed_json_text[:-1] - signed_json_text = signed_json_text.replace(protected_header, "").replace(signature, "") - - payload_bytes = signed_json_text.encode('utf-8') - payload_token = base64.b64encode(payload_bytes).decode('ascii') + if signed_json_text.endswith("\n"): + signed_json_text = signed_json_text[:-1] + signed_json_text = signed_json_text.replace(protected_header, "").replace( + signature, "" + ) + + payload_bytes = signed_json_text.encode("utf-8") + payload_token = base64.b64encode(payload_bytes).decode("ascii") # Un-pad the token, as per RFC7515 annex C - payload_token = payload_token.split('=')[0] - payload_token = payload_token.replace('+','-') - payload_token = payload_token.replace('/','_') + payload_token = payload_token.split("=")[0] + payload_token = payload_token.replace("+", "-") + payload_token = payload_token.replace("/", "_") token = protected_header + "." + payload_token + "." + signature - result = jws.verify(token, key="secret_key", algorithms=['HS256']) - + result = jws.verify(token, key="secret_key", algorithms=["HS256"]) + print("Signature verified") - diff --git a/utils/xml_to_json.py b/utils/xml_to_json.py index 0fc07d79f544df8f8838d02f231433382e350833..74c20f63184286ecd80c59e9b7f004db162bdb48 100644 --- a/utils/xml_to_json.py +++ b/utils/xml_to_json.py @@ -9,99 +9,99 @@ import xmltodict import argparse -def extract_prefixes (d): - return { k.split(':')[1]: d[k] for k in d.keys() if k.startswith("@xmlns:") } +def extract_prefixes(d): + return {k.split(":")[1]: d[k] for k in d.keys() if k.startswith("@xmlns:")} -def removePrefixes (o, prefixes): - if not isinstance(o, dict): return + +def removePrefixes(o, prefixes): + if not isinstance(o, dict): + return replacements = [] - for k,v in o.items(): + for k, v in o.items(): if isinstance(v, dict): removePrefixes(v, prefixes) if isinstance(v, list): for i in v: removePrefixes(i, prefixes) if ":" in k: - prefix = k.split(':')[0] + prefix = k.split(":")[0] if (prefix) in prefixes: - new_key = k.split(':')[1] - replacements.append( (k, new_key) ) + new_key = k.split(":")[1] + replacements.append((k, new_key)) for r in replacements: o[r[1]] = o.pop(r[0]) + object_namespaces = { - 'AuthorisationObject' : 'http://uri.etsi.org/03120/common/2020/09/Authorisation', - 'DeliveryObject' : 'http://uri.etsi.org/03120/common/2019/10/Delivery', - 'DocumentObject' : 'http://uri.etsi.org/03120/common/2020/09/Document', - 'NotificationObject' : 'http://uri.etsi.org/03120/common/2016/02/Notification', - 'LITaskObject' : 'http://uri.etsi.org/03120/common/2020/09/Task', - 'LPTaskObject' : 'http://uri.etsi.org/03120/common/2020/09/Task', - 'LDTaskObject' : 'http://uri.etsi.org/03120/common/2020/09/Task', - 'TrafficPolicyObject' : 'http://uri.etsi.org/03120/common/2022/07/TrafficPolicy', - 'TrafficRuleObject' : 'http://uri.etsi.org/03120/common/2022/07/TrafficPolicy' + "AuthorisationObject": "http://uri.etsi.org/03120/common/2020/09/Authorisation", + "DeliveryObject": "http://uri.etsi.org/03120/common/2019/10/Delivery", + "DocumentObject": "http://uri.etsi.org/03120/common/2020/09/Document", + "NotificationObject": "http://uri.etsi.org/03120/common/2016/02/Notification", + "LITaskObject": "http://uri.etsi.org/03120/common/2020/09/Task", + "LPTaskObject": "http://uri.etsi.org/03120/common/2020/09/Task", + "LDTaskObject": "http://uri.etsi.org/03120/common/2020/09/Task", + "TrafficPolicyObject": "http://uri.etsi.org/03120/common/2022/07/TrafficPolicy", + "TrafficRuleObject": "http://uri.etsi.org/03120/common/2022/07/TrafficPolicy", } coerce_to_list = [ - 'auth:AuthorisationApprovalDetails', - 'auth:AuthorisationFlag', - 'auth:CSPID', - 'common:ApproverContactDetails', - 'ActionRequest', - 'ActionResponse', - 'ListResponseRecord', - 'AssociatedObject', - 'doc:DocumentSignature', - 'doc:DocumentProperty', - 'notification:AssociatedObjectStatus', - 'task:ApprovalDetails', - 'task:TargetIdentifierValue', - 'task:DeliveryDestination', - 'task:TaskFlag', - 'task:AlternativePreservationReference', - 'task:ApprovalDetails', - 'task:ObservedTimes', - 'task:RequestValue', - 'task:RequestSubtype', - 'task:LDDeliveryDestination', - 'task:LDTaskFlag', - 'task:TrafficPolicyReference', - 'tp:TrafficRuleReference', - 'tp:Criteria', - 'common:DictionaryEntry', - 'dictionaries:Dictionary', - 'config:TargetFormatTypeDefinitionEntry', - 'config:SupportedLIWorkflowEndpoint', - 'config:SupportedLPWorkflowEndpoint', + "auth:AuthorisationApprovalDetails", + "auth:AuthorisationFlag", + "auth:CSPID", + "common:ApproverContactDetails", + "ActionRequest", + "ActionResponse", + "ListResponseRecord", + "AssociatedObject", + "doc:DocumentSignature", + "doc:DocumentProperty", + "notification:AssociatedObjectStatus", + "task:ApprovalDetails", + "task:TargetIdentifierValue", + "task:DeliveryDestination", + "task:TaskFlag", + "task:AlternativePreservationReference", + "task:ApprovalDetails", + "task:ObservedTimes", + "task:RequestValue", + "task:RequestSubtype", + "task:LDDeliveryDestination", + "task:LDTaskFlag", + "task:TrafficPolicyReference", + "tp:TrafficRuleReference", + "tp:Criteria", + "common:DictionaryEntry", + "dictionaries:Dictionary", + "config:TargetFormatTypeDefinitionEntry", + "config:SupportedLIWorkflowEndpoint", + "config:SupportedLPWorkflowEndpoint", ] coerce_to_int = [ - 'ActionIdentifier', - 'delivery:SequenceNumber', - 'task:Order', - 'ErrorCode', - 'Generation', - 'tp:Order' + "ActionIdentifier", + "delivery:SequenceNumber", + "task:Order", + "ErrorCode", + "Generation", + "tp:Order", ] -coerce_to_bool = [ - 'delivery:LastSequence' -] +coerce_to_bool = ["delivery:LastSequence"] -coerce_to_empty = [ - 'GETCSPCONFIG' -] +coerce_to_empty = ["GETCSPCONFIG"] coerce_null_to_empty = [ - 'SupportedLIWorkflowEndpoints', - 'SupportedLPWorkflowEndpoints', - 'config:AssociatedLDRequestTypes', - 'config:AssociatedLDRequestSubtypes', - 'config:AssociatedLPRequestTypes', - 'config:AssociatedLPRequestSubtypes', - 'config:AssociatedLIServiceTypes', + "SupportedLIWorkflowEndpoints", + "SupportedLPWorkflowEndpoints", + "config:AssociatedLDRequestTypes", + "config:AssociatedLDRequestSubtypes", + "config:AssociatedLPRequestTypes", + "config:AssociatedLPRequestSubtypes", + "config:AssociatedLIServiceTypes", ] -def postprocessor (path, key, value): + +def postprocessor(path, key, value): if key == "@xsi:type": object_name = value.split(":")[-1] if object_name in object_namespaces.keys(): @@ -118,26 +118,38 @@ def postprocessor (path, key, value): return key, {} return key, value + def recursively_fix_traffic_policy_criteria(d: dict): if isinstance(d, dict): - if ("tp:Criteria" in d + if ( + "tp:Criteria" in d and isinstance(d["tp:Criteria"], list) and len(d["tp:Criteria"]) == 1 - ): - d["tp:Criteria"] = d["tp:Criteria"][0] + ): + d["tp:Criteria"] = d["tp:Criteria"][0] else: - for k,v in d.items(): + for k, v in d.items(): recursively_fix_traffic_policy_criteria(v) elif isinstance(d, list): for d_item in d: recursively_fix_traffic_policy_criteria(d_item) - if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -155,10 +167,9 @@ if __name__ == "__main__": logging.debug(s) - d = xmltodict.parse(s, - force_list=tuple(coerce_to_list), - postprocessor=postprocessor - )['HI1Message'] + d = xmltodict.parse( + s, force_list=tuple(coerce_to_list), postprocessor=postprocessor + )["HI1Message"] # HACK # Needed because TrafficPolicy.xsd has two nested fields both called tp:Criteria