From df0dc5e694b296f492bc38169c6a1b8e60de83f1 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 10:21:04 +0100 Subject: [PATCH 01/33] JSON testing part 1 --- dockerfiles/forgeschema | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 dockerfiles/forgeschema diff --git a/dockerfiles/forgeschema b/dockerfiles/forgeschema new file mode 100644 index 0000000..444c97e --- /dev/null +++ b/dockerfiles/forgeschema @@ -0,0 +1,20 @@ +# docker login forge.etsi.org:5050 +# docker build -t forge.etsi.org:5050/li/schemas-definitions/forgeschema -f forgeschema . +# docker push forge.etsi.org:5050/li/schemas-definitions/forgeschema + +FROM python:3.12-alpine + +RUN apk add --no-cache \ + jq \ + libxml2-utils \ + libffi-dev \ + openssl-dev \ + gcc \ + musl-dev \ + python3-dev \ + build-base + +RUN pip install --upgrade pip && \ + pip install forgeschema + +WORKDIR "/forge/" \ No newline at end of file -- GitLab From d87653265792f7b45b1c32a2fd1a6eeba574ba65 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 10:21:40 +0100 Subject: [PATCH 02/33] JSON testing part 1 --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4c0fc30..12ca416 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,13 @@ process_xsd: script: - python3 testing/xsd_process.py +process_json: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: check + interruptible: true + script: + - forgeschema + generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" stage: build -- GitLab From d15e538674a08d8cf13920be138f869c6c1d5f41 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 10:24:26 +0100 Subject: [PATCH 03/33] Looking for mnt" --- .gitlab-ci.yml | 1 + dockerfiles/forgeschema | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12ca416..f2aa4e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,6 +39,7 @@ process_json: interruptible: true script: - forgeschema + - ls generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" diff --git a/dockerfiles/forgeschema b/dockerfiles/forgeschema index 444c97e..518f240 100644 --- a/dockerfiles/forgeschema +++ b/dockerfiles/forgeschema @@ -15,6 +15,4 @@ RUN apk add --no-cache \ build-base RUN pip install --upgrade pip && \ - pip install forgeschema - -WORKDIR "/forge/" \ No newline at end of file + pip install forgeschema \ No newline at end of file -- GitLab From 647a6e033f134400bd66464c3e88503acf4e622d Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 10:25:36 +0100 Subject: [PATCH 04/33] Looking for mnt" --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2aa4e3..4714f77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,8 +38,8 @@ process_json: stage: check interruptible: true script: - - forgeschema - ls + - forgeschema generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" -- GitLab From 51767ecf3285a7412f8ea19a4a7bd24470473dd3 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 10:27:09 +0100 Subject: [PATCH 05/33] Trying to run validation --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4714f77..e0f4438 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,8 +38,8 @@ process_json: stage: check interruptible: true script: - - ls - - forgeschema + - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json + generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" -- GitLab From be65ae63b1a899d50ba491f8e329684ae1be9129 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 11:19:34 +0100 Subject: [PATCH 06/33] Inserting an error --- 103120/schema/xsd/ts_103120_Core.xsd | 2 +- testing/103120.json | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 testing/103120.json diff --git a/103120/schema/xsd/ts_103120_Core.xsd b/103120/schema/xsd/ts_103120_Core.xsd index 8fca0a8..c9d635e 100644 --- a/103120/schema/xsd/ts_103120_Core.xsd +++ b/103120/schema/xsd/ts_103120_Core.xsd @@ -15,7 +15,7 @@ - + xyz diff --git a/testing/103120.json b/testing/103120.json new file mode 100644 index 0000000..4f28bf2 --- /dev/null +++ b/testing/103120.json @@ -0,0 +1,20 @@ +{ + "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", + "supportingSchemas": [ + "103120/schema/xsd/ts_103120_Authorisation.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Document.xsd", + "103120/schema/xsd/ts_103120_Notification.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/xml/FooServiceSchema.xsd" + ], + "instanceDocs": [ + "103120/examples/xml" + ] +} \ No newline at end of file -- GitLab From 0fb5a01048e198a965d2910bdcdf9a257f7597e4 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 11:28:11 +0100 Subject: [PATCH 07/33] trying env --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0f4438..9ec6464 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,7 @@ process_json: stage: check interruptible: true script: + - export PYCHARM_HOSTED=1 - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json -- GitLab From eee4ea3afdfaddf88a60b635c368365ab60a49f9 Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 11:32:47 +0100 Subject: [PATCH 08/33] Attempting XML too --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ec6464..4d932db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,6 +32,7 @@ process_xsd: interruptible: true script: - python3 testing/xsd_process.py + - forgeschema -c testing/103120.json process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" @@ -41,7 +42,6 @@ process_json: - export PYCHARM_HOSTED=1 - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json - generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" stage: build -- GitLab From dc271ff4d9ca0c513458ebcf572d893033a4d78d Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 11:34:15 +0100 Subject: [PATCH 09/33] Attempting XML too --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d932db..f37e573 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,8 +31,9 @@ process_xsd: stage: check interruptible: true script: - - python3 testing/xsd_process.py - - forgeschema -c testing/103120.json + - export PYCHARM_HOSTED=1 + - forgeschema -c testing/103120.json + - python3 testing/xsd_process.py process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" -- GitLab From d19be47e072d7133fbb3a03d00703d7cfc90d24f Mon Sep 17 00:00:00 2001 From: mark Date: Fri, 4 Jul 2025 11:36:27 +0100 Subject: [PATCH 10/33] Attempting XML too --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f37e573..ed5d465 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,9 +27,10 @@ process_asn: - python3 testing/asn_process.py process_xsd: - image: "forge.etsi.org:5050/li/schemas-definitions/xsdtest:latest" + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: check interruptible: true + allow_failure: true script: - export PYCHARM_HOSTED=1 - forgeschema -c testing/103120.json @@ -39,6 +40,7 @@ process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: check interruptible: true + allow_failure: true script: - export PYCHARM_HOSTED=1 - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json -- GitLab From 8fe795b9c96d4f9f5d892d8b2a344b02552f7e37 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:11:25 +0100 Subject: [PATCH 11/33] Duplicating Steije's bash magic --- .gitlab-ci.yml | 9 +++++---- testing/{103120.json => xml/ts_103120._xsd.json} | 0 2 files changed, 5 insertions(+), 4 deletions(-) rename testing/{103120.json => xml/ts_103120._xsd.json} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed5d465..49de604 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,17 +30,18 @@ process_xsd: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: check interruptible: true - allow_failure: true script: - export PYCHARM_HOSTED=1 - - forgeschema -c testing/103120.json - - python3 testing/xsd_process.py + - | + format_issues = 0 + while IFS= read -r -d file; do + echo $file + done < <(find ./testing/xml -type f name "*.json" -print0) process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: check interruptible: true - allow_failure: true script: - export PYCHARM_HOSTED=1 - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json diff --git a/testing/103120.json b/testing/xml/ts_103120._xsd.json similarity index 100% rename from testing/103120.json rename to testing/xml/ts_103120._xsd.json -- GitLab From ab4efe1a8922b2fb63cc41a64dd8a570768e6a81 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:14:21 +0100 Subject: [PATCH 12/33] Duplicating Steije's bash magic --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49de604..b32d2f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,7 @@ process_xsd: script: - export PYCHARM_HOSTED=1 - | - format_issues = 0 + format_issues=0 while IFS= read -r -d file; do echo $file done < <(find ./testing/xml -type f name "*.json" -print0) -- GitLab From f47e3e98ad1dda8cf581ff5474d2932947672ff2 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:15:50 +0100 Subject: [PATCH 13/33] Duplicating Steije's bash magic --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b32d2f5..9f1b1d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,7 +36,7 @@ process_xsd: format_issues=0 while IFS= read -r -d file; do echo $file - done < <(find ./testing/xml -type f name "*.json" -print0) + done < <(find ./testing/xml -type f -name "*.json" -print0) process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" -- GitLab From b4d413c311ed9abc3a26bb3e69ec63e31473b936 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:19:54 +0100 Subject: [PATCH 14/33] Duplicating Steije's bash magic --- .gitlab-ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f1b1d5..3451c95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,9 +35,17 @@ process_xsd: - | format_issues=0 while IFS= read -r -d file; do - echo $file + if ! forgeschema -c "$file"; then + echo "❌ failed schema checks for $file" + format_issues=1 done < <(find ./testing/xml -type f -name "*.json" -print0) + if [ "$format_issues" -eq 1 ]; then + echo "❌ XSD validation failed" + exit 1 + fi + echo "✅ XSD validation OK" + process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: check -- GitLab From 542d5f688872cda10d75a1ddb8b4f492b8603bbd Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:23:40 +0100 Subject: [PATCH 15/33] Duplicating Steije's bash magic --- .gitlab-ci.yml | 32 ++++++++++++++++++++------------ testing/xml/ts_103280_xsd.json | 7 +++++++ 2 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 testing/xml/ts_103280_xsd.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3451c95..2a3db96 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,7 @@ workflow: stages: - preflight - check + - lint - build preflight: @@ -32,19 +33,19 @@ process_xsd: interruptible: true script: - export PYCHARM_HOSTED=1 - - | - format_issues=0 - while IFS= read -r -d file; do - if ! forgeschema -c "$file"; then - echo "❌ failed schema checks for $file" - format_issues=1 - done < <(find ./testing/xml -type f -name "*.json" -print0) + - | + format_issues=0 - if [ "$format_issues" -eq 1 ]; then - echo "❌ XSD validation failed" - exit 1 - fi - echo "✅ XSD validation OK" + while IFS= read -r -d '' file; do + if ! forgeschema -c "$file"; then + echo "❌ failed schema checks for $file" + format_issues=1 + fi + done < <(find ./testing/xml -type f -name "*.json" -print0) + if [ "$format_issues" -eq 1 ]; then + exit 1 + fi + echo "✅ XSD validation OK" process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" @@ -54,6 +55,13 @@ process_json: - export PYCHARM_HOSTED=1 - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json +lint_xml: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: lint + interruptible: true + script: + - python testing/lint_xml.py + generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" stage: build diff --git a/testing/xml/ts_103280_xsd.json b/testing/xml/ts_103280_xsd.json new file mode 100644 index 0000000..f9e5066 --- /dev/null +++ b/testing/xml/ts_103280_xsd.json @@ -0,0 +1,7 @@ +{ + "coreSchema": "103280/TS_103_280.xsd", + "supportingSchemas": [ + ], + "instanceDocs": [ + ] +} \ No newline at end of file -- GitLab From 2ec9155e0b16a44ceba27fe6abd782ddb0ae5639 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:24:57 +0100 Subject: [PATCH 16/33] Duplicating Steije's bash magic --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a3db96..1625bd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,7 +32,7 @@ process_xsd: stage: check interruptible: true script: - - export PYCHARM_HOSTED=1 + - export PYCHARM_HOSTED=1 - | format_issues=0 -- GitLab From 8f15ba67c55711d45b07f4ae4550cff14d23aa6b Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:32:04 +0100 Subject: [PATCH 17/33] Duplicating Steije's bash magic --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1625bd7..fe36437 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,11 +41,14 @@ process_xsd: echo "❌ failed schema checks for $file" format_issues=1 fi + echo "" + echo "..." + echo "" done < <(find ./testing/xml -type f -name "*.json" -print0) if [ "$format_issues" -eq 1 ]; then exit 1 fi - echo "✅ XSD validation OK" + echo "✅ XSD validation OK for $file" process_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" -- GitLab From fd0b1c42cc31a9712e81ae426685e8cf70d99216 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:38:17 +0100 Subject: [PATCH 18/33] Copying across to JSON testing --- .gitlab-ci.yml | 20 +++++++++++++++++-- 103120/schema/json/ts_103120_Core.schema.json | 2 +- testing/json/ts_103120._json.json | 11 ++++++++++ testing/json/ts_103280._json.json | 5 +++++ testing/{ => xml}/lint_xml.py | 0 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 testing/json/ts_103120._json.json create mode 100644 testing/json/ts_103280._json.json rename testing/{ => xml}/lint_xml.py (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fe36437..369e357 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,8 +55,24 @@ process_json: stage: check interruptible: true script: - - export PYCHARM_HOSTED=1 - - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json + - export PYCHARM_HOSTED=1 + - | + format_issues=0 + + while IFS= read -r -d '' file; do + if ! forgeschema -c "$file"; then + echo "❌ failed schema checks for $file" + format_issues=1 + fi + echo "" + echo "..." + echo "" + done < <(find ./testing/json -type f -name "*.json" -print0) + if [ "$format_issues" -eq 1 ]; then + exit 1 + fi + echo "✅ XSD validation OK for $file" + - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json lint_xml: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" diff --git a/103120/schema/json/ts_103120_Core.schema.json b/103120/schema/json/ts_103120_Core.schema.json index 61d682a..d83dffb 100644 --- a/103120/schema/json/ts_103120_Core.schema.json +++ b/103120/schema/json/ts_103120_Core.schema.json @@ -11,7 +11,7 @@ "Header": { "$ref": "#/$defs/MessageHeader" }, - "Payload": { + "Payload": {this will break stuff "$ref": "#/$defs/MessagePayload" }, "Signature": { diff --git a/testing/json/ts_103120._json.json b/testing/json/ts_103120._json.json new file mode 100644 index 0000000..6873692 --- /dev/null +++ b/testing/json/ts_103120._json.json @@ -0,0 +1,11 @@ +{ + "coreSchema": "103120/schema/json/ts_103120_Core.schema.json", + "supportingSchemas": [ + "103120/schema/json", + "103120/dictionaries/ts_103120_Dictionaries.schema.json", + "103280/TS_103_280.schema.json" + ], + "instanceDocs": [ + "103120/examples/json" + ] +} \ No newline at end of file diff --git a/testing/json/ts_103280._json.json b/testing/json/ts_103280._json.json new file mode 100644 index 0000000..5fe597a --- /dev/null +++ b/testing/json/ts_103280._json.json @@ -0,0 +1,5 @@ +{ + "coreSchema": "103120/dictionaries/ts_103120_Dictionaries.schema.json", + "supportingSchemas": [], + "instanceDocs": [] +} \ No newline at end of file diff --git a/testing/lint_xml.py b/testing/xml/lint_xml.py similarity index 100% rename from testing/lint_xml.py rename to testing/xml/lint_xml.py -- GitLab From c9ba3e0b7742edd6b4800622ad61df675cbbd765 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 08:43:02 +0100 Subject: [PATCH 19/33] Better output? --- .gitlab-ci.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 369e357..cbe62be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,20 +58,27 @@ process_json: - export PYCHARM_HOSTED=1 - | format_issues=0 - + check_count=0 + fail_count=0 while IFS= read -r -d '' file; do + let "check_count=check_count+1" if ! forgeschema -c "$file"; then echo "❌ failed schema checks for $file" - format_issues=1 + let "fail_count=fail_count+1" fi echo "" echo "..." echo "" done < <(find ./testing/json -type f -name "*.json" -print0) - if [ "$format_issues" -eq 1 ]; then + if [ "$fail_count" -gt 0 ]; then + echo "════════════════════════════════════════════════════════" + echo "❌ Failed schema checks for $fail_count of $check_count" exit 1 + echo "════════════════════════════════════════════════════════" fi - echo "✅ XSD validation OK for $file" + echo "════════════════════════════════════════════════════════" + echo "✅ XSD validation OK ($check_count files checked)" + echo "════════════════════════════════════════════════════════" - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json lint_xml: -- GitLab From 75c926318ba090b59e59ca82f7679e3d18e05cb9 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 09:04:40 +0100 Subject: [PATCH 20/33] Updating --- .gitlab-ci.yml | 26 ++++++++++++------- .../dockerfiles}/forgeschema | 0 2 files changed, 16 insertions(+), 10 deletions(-) rename {dockerfiles => testing/dockerfiles}/forgeschema (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbe62be..c933770 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,20 +34,27 @@ process_xsd: script: - export PYCHARM_HOSTED=1 - | - format_issues=0 - + check_count=0 + fail_count=0 while IFS= read -r -d '' file; do + let "check_count=check_count+1" if ! forgeschema -c "$file"; then echo "❌ failed schema checks for $file" - format_issues=1 + let "fail_count=fail_count+1" fi echo "" echo "..." echo "" done < <(find ./testing/xml -type f -name "*.json" -print0) - if [ "$format_issues" -eq 1 ]; then + if [ "$fail_count" -gt 0 ]; then + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "❌ Failed schema checks for $fail_count of $check_count files" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" exit 1 fi + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "✅ XSD validation OK ($check_count files checked)" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" echo "✅ XSD validation OK for $file" process_json: @@ -57,7 +64,6 @@ process_json: script: - export PYCHARM_HOSTED=1 - | - format_issues=0 check_count=0 fail_count=0 while IFS= read -r -d '' file; do @@ -71,14 +77,14 @@ process_json: echo "" done < <(find ./testing/json -type f -name "*.json" -print0) if [ "$fail_count" -gt 0 ]; then - echo "════════════════════════════════════════════════════════" - echo "❌ Failed schema checks for $fail_count of $check_count" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" + echo "❌ Failed schema checks for $fail_count of $check_count files" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" exit 1 - echo "════════════════════════════════════════════════════════" fi - echo "════════════════════════════════════════════════════════" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" echo "✅ XSD validation OK ($check_count files checked)" - echo "════════════════════════════════════════════════════════" + echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json lint_xml: diff --git a/dockerfiles/forgeschema b/testing/dockerfiles/forgeschema similarity index 100% rename from dockerfiles/forgeschema rename to testing/dockerfiles/forgeschema -- GitLab From d1cf3d7640e04540828d66fbe2699316d4d33f1b Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 09:14:05 +0100 Subject: [PATCH 21/33] Ignoring docker build log --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 3e37603..fc05895 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ .vscode/ .idea/ +testing/dockerfiles/build.log + # Vagrant .vagrant/ -- GitLab From 902d23486a77c0d3a89b69eb96dfe741bc87bd5a Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 09:29:52 +0100 Subject: [PATCH 22/33] Moving XSD tests across --- 103120/schema/xsd/ts_103120_Core.xsd | 2 +- testing/xml/etsi_dictionaries.json | 8 +++++++ testing/xml/ts_102657_xsd.json | 5 +++++ ...ts_103120._xsd.json => ts_103120_xsd.json} | 0 testing/xml/ts_103221-1_xsd.json | 22 +++++++++++++++++++ testing/xml/ts_103707_xsd.json | 19 ++++++++++++++++ testing/xml/ts_104000_xsd.json | 9 ++++++++ testing/xml/ts_104144_xsd.json | 20 +++++++++++++++++ 8 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 testing/xml/etsi_dictionaries.json create mode 100644 testing/xml/ts_102657_xsd.json rename testing/xml/{ts_103120._xsd.json => ts_103120_xsd.json} (100%) create mode 100644 testing/xml/ts_103221-1_xsd.json create mode 100644 testing/xml/ts_103707_xsd.json create mode 100644 testing/xml/ts_104000_xsd.json create mode 100644 testing/xml/ts_104144_xsd.json diff --git a/103120/schema/xsd/ts_103120_Core.xsd b/103120/schema/xsd/ts_103120_Core.xsd index c9d635e..8fca0a8 100644 --- a/103120/schema/xsd/ts_103120_Core.xsd +++ b/103120/schema/xsd/ts_103120_Core.xsd @@ -15,7 +15,7 @@ - xyz + diff --git a/testing/xml/etsi_dictionaries.json b/testing/xml/etsi_dictionaries.json new file mode 100644 index 0000000..18e8c6f --- /dev/null +++ b/testing/xml/etsi_dictionaries.json @@ -0,0 +1,8 @@ +{ + "coreSchema" : "103120/dictionaries/ts_103120_Dictionaries.xsd", + "supportingSchemas" : [], + "instanceDocs" : [ + "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml", + "104144/dictionaries/" + ] +} \ No newline at end of file diff --git a/testing/xml/ts_102657_xsd.json b/testing/xml/ts_102657_xsd.json new file mode 100644 index 0000000..d534c1a --- /dev/null +++ b/testing/xml/ts_102657_xsd.json @@ -0,0 +1,5 @@ +{ + "coreSchema": "102657/RDMessage.xsd", + "supportingSchemas": [], + "instanceDocs": [] +} \ No newline at end of file diff --git a/testing/xml/ts_103120._xsd.json b/testing/xml/ts_103120_xsd.json similarity index 100% rename from testing/xml/ts_103120._xsd.json rename to testing/xml/ts_103120_xsd.json diff --git a/testing/xml/ts_103221-1_xsd.json b/testing/xml/ts_103221-1_xsd.json new file mode 100644 index 0000000..2511966 --- /dev/null +++ b/testing/xml/ts_103221-1_xsd.json @@ -0,0 +1,22 @@ +{ + "coreSchema" : "103221-1/TS_103_221_01.xsd", + "supportingSchemas" : [ + "103221-1/TS_103_221_01.xsd", + "103221-1/TS_103_221_01_HashedID.xsd", + "103221-1/TS_103_221_01_Configuration.xsd", + "103221-1/TS_103_221_01_DestinationSet.xsd", + "103221-1/TS_103_221_01_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "103221-1/examples/ExampleGenericObjects.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Core.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "testing/deps/old_400/TS_104_000.xsd" + ], + "instanceDocs": [ + "103221-1/examples" + ] +} \ No newline at end of file diff --git a/testing/xml/ts_103707_xsd.json b/testing/xml/ts_103707_xsd.json new file mode 100644 index 0000000..0b0e601 --- /dev/null +++ b/testing/xml/ts_103707_xsd.json @@ -0,0 +1,19 @@ +{ + "coreSchema": "103707/TS_103_707.xsd", + "supportingSchemas": [ + "103280/TS_103_280.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Core.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103707/examples/FooServiceSchema.xsd", + "104112/examples/xml/MessagingServiceSchema.xsd" + ], + "instanceDocs": [ + "103707/examples", + "104112/examples" + ] +} \ No newline at end of file diff --git a/testing/xml/ts_104000_xsd.json b/testing/xml/ts_104000_xsd.json new file mode 100644 index 0000000..4222fed --- /dev/null +++ b/testing/xml/ts_104000_xsd.json @@ -0,0 +1,9 @@ +{ + "coreSchema": "104000/schema/TS_104_000.xsd", + "supportingSchemas": [ + "103280/TS_103_280.xsd" + ], + "instanceDocs": [ + "104000/examples/" + ] +} \ No newline at end of file diff --git a/testing/xml/ts_104144_xsd.json b/testing/xml/ts_104144_xsd.json new file mode 100644 index 0000000..73e415f --- /dev/null +++ b/testing/xml/ts_104144_xsd.json @@ -0,0 +1,20 @@ +{ + "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", + "supportingSchemas": [ + "103120/schema/xsd/ts_103120_Authorisation.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Document.xsd", + "103120/schema/xsd/ts_103120_Notification.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/xml/FooServiceSchema.xsd" + ], + "instanceDocs": [ + "104144/examples/" + ] +} \ No newline at end of file -- GitLab From 3aa8bdb49fff9e66fd187c958c5b039dceea0754 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 09:58:49 +0100 Subject: [PATCH 23/33] Adding JSON testing --- 103120/schema/json/ts_103120_Core.schema.json | 2 +- 103976/ts_103976.schema.json | 2 +- testing/json/ts_103705_ex1_json.json | 11 +++++++++++ testing/json/ts_103976_json.json | 10 ++++++++++ testing/xml/etsi_dictionaries.json | 3 ++- 5 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 testing/json/ts_103705_ex1_json.json create mode 100644 testing/json/ts_103976_json.json diff --git a/103120/schema/json/ts_103120_Core.schema.json b/103120/schema/json/ts_103120_Core.schema.json index d83dffb..61d682a 100644 --- a/103120/schema/json/ts_103120_Core.schema.json +++ b/103120/schema/json/ts_103120_Core.schema.json @@ -11,7 +11,7 @@ "Header": { "$ref": "#/$defs/MessageHeader" }, - "Payload": {this will break stuff + "Payload": { "$ref": "#/$defs/MessagePayload" }, "Signature": { diff --git a/103976/ts_103976.schema.json b/103976/ts_103976.schema.json index 207bce1..84fe84c 100644 --- a/103976/ts_103976.schema.json +++ b/103976/ts_103976.schema.json @@ -168,7 +168,7 @@ }, "ListOfCommsIDs" : { "type" : "array", - "items" : "#/$defs/CommsID" + "items" : { "$ref" : "#/$defs/CommsID" } }, "ResultRecords" : { "type" : "object", diff --git a/testing/json/ts_103705_ex1_json.json b/testing/json/ts_103705_ex1_json.json new file mode 100644 index 0000000..11b8e11 --- /dev/null +++ b/testing/json/ts_103705_ex1_json.json @@ -0,0 +1,11 @@ +{ + "coreSchema": "103705/schema/response.schema.json", + "supportingSchemas": [ + "103705/schema/", + "103280/TS_103_280.schema.json", + "103705/examples/example1/csp_records.schema.json" + ], + "instanceDocs": [ + "103705/examples/example1/csp_results.json" + ] +} \ No newline at end of file diff --git a/testing/json/ts_103976_json.json b/testing/json/ts_103976_json.json new file mode 100644 index 0000000..af62bbd --- /dev/null +++ b/testing/json/ts_103976_json.json @@ -0,0 +1,10 @@ +{ + "coreSchema": "103976/ts_103976.schema.json", + "supportingSchemas": [ + "103280/TS_103_280.schema.json", + "103120/schema/json/ts_103120_Common.schema.json" + ], + "instanceDocs": [ + "103976/examples/" + ] +} \ No newline at end of file diff --git a/testing/xml/etsi_dictionaries.json b/testing/xml/etsi_dictionaries.json index 18e8c6f..86b29ad 100644 --- a/testing/xml/etsi_dictionaries.json +++ b/testing/xml/etsi_dictionaries.json @@ -3,6 +3,7 @@ "supportingSchemas" : [], "instanceDocs" : [ "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml", - "104144/dictionaries/" + "104144/dictionaries/", + "103976/ts_103976_ETSIDictionaryDefinitions.xml" ] } \ No newline at end of file -- GitLab From a051d5c479aee71863445fc6c944037b7268b131 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 10:01:24 +0100 Subject: [PATCH 24/33] Fixing yml --- .gitlab-ci.yml | 2 +- testing/{ => dockerfiles}/dockerfile_asn | 0 testing/{ => dockerfiles}/dockerfile_xsd | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename testing/{ => dockerfiles}/dockerfile_asn (100%) rename testing/{ => dockerfiles}/dockerfile_xsd (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c933770..ac2ef0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,7 +92,7 @@ lint_xml: stage: lint interruptible: true script: - - python testing/lint_xml.py + - python testing/xml/lint_xml.py generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" diff --git a/testing/dockerfile_asn b/testing/dockerfiles/dockerfile_asn similarity index 100% rename from testing/dockerfile_asn rename to testing/dockerfiles/dockerfile_asn diff --git a/testing/dockerfile_xsd b/testing/dockerfiles/dockerfile_xsd similarity index 100% rename from testing/dockerfile_xsd rename to testing/dockerfiles/dockerfile_xsd -- GitLab From 9c10f5d7d7f09eb07ac750f93e47ddacba2acd16 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 10:22:27 +0100 Subject: [PATCH 25/33] Linting JSON --- .gitlab-ci.yml | 17 +++++++-- testing/json/lint_json.py | 78 +++++++++++++++++++++++++++++++++++++++ testing/xml/lint_xml.py | 23 ++++++++++-- 3 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 testing/json/lint_json.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ac2ef0a..458499b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ workflow: stages: - preflight - - check + - compile - lint - build @@ -20,14 +20,14 @@ preflight: script: - forgelib-preflight https://$CI_SERVER_HOST $CI_PROJECT_ID $CI_MERGE_REQUEST_IID -process_asn: +compile_asn: image: "forge.etsi.org:5050/li/schemas-definitions/asn1test:latest" stage: check interruptible: true script: - python3 testing/asn_process.py -process_xsd: +compile_xsd: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: check interruptible: true @@ -57,7 +57,7 @@ process_xsd: echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" echo "✅ XSD validation OK for $file" -process_json: +compile_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: check interruptible: true @@ -91,9 +91,18 @@ lint_xml: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: lint interruptible: true + allow_failure: true script: - python testing/xml/lint_xml.py +lint_xml: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: lint + interruptible: true + allow_failure: true + script: + - python testing/xml/lint_json.py + generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" stage: build diff --git a/testing/json/lint_json.py b/testing/json/lint_json.py new file mode 100644 index 0000000..f507123 --- /dev/null +++ b/testing/json/lint_json.py @@ -0,0 +1,78 @@ +from pathlib import Path +from difflib import * +import subprocess + +import colorama +from colorama import Fore, Style + +colorama.init() + +ignore_paths = [Path(x) for x in [ + 'testing/deps' +]] + +def print_colorized_diff_line(line: str): + if line.startswith("-"): + print(f"{Fore.RED}{line}{Style.RESET_ALL}") + elif line.startswith("+"): + print(f"{Fore.GREEN}{line}{Style.RESET_ALL}") + else: + print(line) + +def lint(file : Path): + completed = subprocess.run(["jq", ".", str(file)], capture_output=True, text=True, encoding="utf8") + + if completed.returncode != 0: + print (f" {str(f)}: FAIL") + print (f" jq error code {completed.returncode}") + lines = completed.stderr.splitlines() + for line in lines: + print (f" {line}") + return len(lines) + + linted_xml = completed.stdout + orig_xml = file.read_text(encoding="utf8") + diff = list(unified_diff(orig_xml.splitlines(), linted_xml.splitlines())) + if len(diff) == 0: + print (f"✅ {str(f)}") + return 0 + else: + print (f"❌ {str(f)}: {len(diff)} linting errors") + # for d in diff: + # print("".join(d)) + for d in diff: + print_colorized_diff_line(d) + return len(diff) + + + +if __name__ == "__main__": + root = Path("./") + files = list(root.rglob("*.json")) + + files_with_errors = 0 + errors = 0 + + print ("───────────────────────────────────────────────────────────────────") + print (f"Linting JSON files...") + + for f in files: + if len(list(set(f.parents) & set(ignore_paths))) > 0: + print (f"(Ignoring {f})") + continue + new_errors = lint (f) + errors += new_errors + files_with_errors += 1 if new_errors > 0 else 0 + + print ("───────────────────────────────────────────────────────────────────") + print (f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}") + if (files_with_errors == 0): + print ("✅ OK") + else: + print("❌ Fail") + print ("───────────────────────────────────────────────────────────────────") + + if (files_with_errors > 0): + exit(-1) + else: + exit(0) \ No newline at end of file diff --git a/testing/xml/lint_xml.py b/testing/xml/lint_xml.py index abfa1a1..f295e51 100644 --- a/testing/xml/lint_xml.py +++ b/testing/xml/lint_xml.py @@ -2,10 +2,23 @@ from pathlib import Path from difflib import * import subprocess +import colorama +from colorama import Fore, Style + +colorama.init() + ignore_paths = [Path(x) for x in [ 'testing/deps' ]] +def print_colorized_diff_line(line: str): + if line.startswith("-"): + print(f"{Fore.RED}{line}{Style.RESET_ALL}") + elif line.startswith("+"): + print(f"{Fore.GREEN}{line}{Style.RESET_ALL}") + else: + print(line) + def lint(file : Path): completed = subprocess.run(["xmllint", str(file)], capture_output=True, text=True, encoding="utf8") @@ -38,7 +51,7 @@ if __name__ == "__main__": files_with_errors = 0 errors = 0 - print ("-------------------------------------------------") + print ("") print (f"Linting XML / XSD files...") for f in files: @@ -49,9 +62,13 @@ if __name__ == "__main__": errors += new_errors files_with_errors += 1 if new_errors > 0 else 0 - print ("-------------------------------------------------") + print ("───────────────────────────────────────────────────────────────────") print (f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}") - print ("-------------------------------------------------") + if (files_with_errors == 0): + print ("✅ OK") + else: + print("❌ Fail") + print ("───────────────────────────────────────────────────────────────────") if (files_with_errors > 0): exit(-1) -- GitLab From e485f77cbb7223a02d51d1b29abbd49392cd563c Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 10:23:53 +0100 Subject: [PATCH 26/33] Linting JSON --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 458499b..c07b1bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,14 +22,14 @@ preflight: compile_asn: image: "forge.etsi.org:5050/li/schemas-definitions/asn1test:latest" - stage: check + stage: compile interruptible: true script: - python3 testing/asn_process.py compile_xsd: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" - stage: check + stage: compile interruptible: true script: - export PYCHARM_HOSTED=1 @@ -59,7 +59,7 @@ compile_xsd: compile_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" - stage: check + stage: compile interruptible: true script: - export PYCHARM_HOSTED=1 -- GitLab From 7fb43357161f6d35441854f6f851cdc4493b4774 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 10:26:43 +0100 Subject: [PATCH 27/33] Linting JSON --- .editorconfig | 2 +- .gitlab-ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.editorconfig b/.editorconfig index 1473532..117b3f6 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,7 +2,7 @@ root = true # ASN.1, XML: 4 space indents -[**.{asn,asn1,xml,xsd}] +[**.{asn,asn1,xml,xsd,json}] indent_style = space indent_size = 4 trim_trailing_whitespace = true diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c07b1bb..6fc7a95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,13 +95,13 @@ lint_xml: script: - python testing/xml/lint_xml.py -lint_xml: +lint_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: lint interruptible: true allow_failure: true script: - - python testing/xml/lint_json.py + - python testing/json/lint_json.py generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" -- GitLab From 9d7969841e1178882b179b285bff341fd969aa1e Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 10:35:20 +0100 Subject: [PATCH 28/33] Refactoring --- .gitlab-ci.yml | 24 +- temp.asn | 762 +++++++++++++++++++++ testing/{ => asn}/asn_compile_targets.json | 0 testing/{ => asn}/asn_ignore.txt | 0 testing/{ => asn}/asn_ignore_lint.txt | 0 testing/{ => asn}/asn_process.py | 4 +- testing/dockerfiles/forgeschema | 3 +- testing/merge_test.py | 68 -- testing/xsd_compile_targets.json | 118 ---- testing/xsd_ignore.txt | 3 - testing/xsd_process.py | 222 ------ 11 files changed, 787 insertions(+), 417 deletions(-) create mode 100644 temp.asn rename testing/{ => asn}/asn_compile_targets.json (100%) rename testing/{ => asn}/asn_ignore.txt (100%) rename testing/{ => asn}/asn_ignore_lint.txt (100%) rename testing/{ => asn}/asn_process.py (97%) delete mode 100644 testing/merge_test.py delete mode 100644 testing/xsd_compile_targets.json delete mode 100644 testing/xsd_ignore.txt delete mode 100644 testing/xsd_process.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fc7a95..36dcc33 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,10 @@ stages: - lint - build + +# ----------------------------------------------------------- +# Preflight + preflight: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" stage: preflight @@ -20,12 +24,16 @@ preflight: script: - forgelib-preflight https://$CI_SERVER_HOST $CI_PROJECT_ID $CI_MERGE_REQUEST_IID + +# ----------------------------------------------------------- +# Compile + compile_asn: image: "forge.etsi.org:5050/li/schemas-definitions/asn1test:latest" stage: compile interruptible: true script: - - python3 testing/asn_process.py + - python3 testing/asn/asn_process.py compile_xsd: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" @@ -87,13 +95,18 @@ compile_json: echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json + +# ----------------------------------------------------------- +# Lint + lint_xml: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" stage: lint interruptible: true allow_failure: true script: - - python testing/xml/lint_xml.py + - export PYCHARM_HOSTED=1 + - python testing/xml/lint_xml.py lint_json: image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" @@ -101,7 +114,12 @@ lint_json: interruptible: true allow_failure: true script: - - python testing/json/lint_json.py + - export PYCHARM_HOSTED=1 + - python testing/json/lint_json.py + + +# ----------------------------------------------------------- +# Build generate_artefacts: image: "forge.etsi.org:5050/li/schemas-definitions/forgelib" diff --git a/temp.asn b/temp.asn new file mode 100644 index 0000000..e38540a --- /dev/null +++ b/temp.asn @@ -0,0 +1,762 @@ +LI-PS-PDU +{itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) genHeader(1) version40(40)} + +DEFINITIONS IMPLICIT TAGS ::= + +BEGIN + +IMPORTS + -- Any of the IMPORTs may be commented out if they are not used (see clause A.3) + + -- from ETSI TS 103 280 [44] + LIID, + WGS84CoordinateDecimal + FROM Common-Parameters + {itu-t(0) identified-organization(4) etsi(0) common-parameters(3280) version261(261)} + + + -- previously from ETSI TS 101 671 [4]; now provided with this specification + IRI-Parameters, + IRIsContent, + Location, + Network-Element-Identifier + FROM HI2Operations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) hi2(1) version18(18)} + + -- previously from ETSI TS 101 671 [4]; now provided with this specification + HI1-Operation + FROM HI1NotificationOperations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) hi1(0) notificationOperations(1) version7(7)} + + -- from ETSI TS 102 232-2 [5] + EmailCC, + EmailIRI, + MessagingCC, + MessagingMMCC, + MessagingIRI + FROM EmailPDU + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) email(2) version20(20)} + + -- from ETSI TS 102 232-3 [6] + IPCC, + IPIRI, + IPIRIOnly, + IPIRIPacketReport + FROM IPAccessPDU + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPAccess(3) version20(20)} + + -- from ETSI TS 102 232-4 [32] + L2CC, + L2IRI, + L2IRIOnly + FROM L2AccessPDU + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) l2Access(4) version10(10)} + + -- from ETSI TS 102 232-5 [37] + IPMMCC, + IPMMIRI + FROM IPMultimediaPDU + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPMultimedia(5) version17(17)} + + -- from ETSI TS 102 232-6 [36] + PstnIsdnCC, + PstnIsdnIRI + FROM PstnIsdnPDU + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) pstnIsdn(6) version6(6)} + + -- from 3GPP TS 33.108 [9] + IRI-Parameters, + UmtsIRIsContent, + CorrelationValues, + Location + FROM UmtsHI2Operations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2(1)} + -- The relevant module (including the 3GPP release and version number) needs + -- to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + IRI-Parameters, + UmtsCS-IRIsContent + FROM UmtsCS-HI2Operations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2CS(3)} + -- The relevant module (including the 3GPP release and version number) needs + -- to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + IRI-Parameters, + EpsIRIsContent, + EPSLocation + FROM EpsHI2Operations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2eps(8)} + -- The relevant module (including the 3GPP release and version number) needs + -- to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + IRI-Parameters, + ConfIRIsContent + FROM CONFHI2Operations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2conf(10)} + -- The relevant module (including the 3GPP release and version number) needs + -- to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + IRI-Parameters, + ProSeIRIsContent + FROM ProSeHI2Operations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2prose (15)} + -- The relevant module (including the 3GPP release and version number) needs + -- to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + IRI-Parameters, + GcseIRIsContent + FROM GCSEHI2Operations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2gcse(13)} + -- The relevant module (including the 3GPP release and version number) needs + -- to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + CC-PDU + FROM Umts-HI3-PS + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3(2)} + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + CC-PDU + FROM Eps-HI3-PS + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3eps(9)} + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + Conf-CC-PDU + FROM CONF-HI3-IMS + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3conf(11)} + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + Voip-CC-PDU + FROM VoIP-HI3-IMS + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3voip(12)} + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + Gcse-CC-PDU + FROM GCSE-HI3 + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3gcse(14)} + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + + -- from 3GPP TS 33.108 [9] + ThreeGPP-HI1-Operation + FROM ThreeGPP-HI1NotificationOperations + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi1(0) notificationOperations(1)} + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + -- TS 101 671 HI1 and 3GPP HI1 are related to the same functionality but are + -- corresponding to different implementations and exclusive usage each other. + -- The implementation depends of national regulations or LEA/CSP negotiations. + -- 3GPP HI1 may be used with other services/networks than 3GPP's one. + + -- from 3GPP TS 33.108 [9] + CSvoice-CC-PDU + FROM CSvoice-HI3-IP + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi3CSvoice(18)} + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + -- The implementation of the CS domain delivery in IP have to be based on 3GPP TS 33.108 [9]. + + -- from ETSI TS 101 909-20-1 [33] + TARGETACTIVITYMONITOR-1, + TTRAFFIC, + CTTRAFFIC + FROM TS101909201 + {itu-t(0) identified-organization(4) etsi(0) ts101909(1909) part20(20) subpart1(1) interceptVersion(0)} + + -- from ETSI TS 101 909-20-2 [34] + TARGETACTIVITYMONITOR, + TTRAFFIC, + CTTRAFFIC + FROM TS101909202 + {itu-t(0) identified-organization(4) etsi(0) ts101909(1909) part20(20) subpart2(2) interceptVersion(0)} + + -- from ETSI TS 103 462 [45] + ILHIPayload + FROM ILHIPDU + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) informationHandover(4) ilhi(0) ilhiPdu(0) version2(2)} + -- This import is only used for the handover between LEMFs. + + -- from 3GPP TS 33.128 [46] + LINotificationPayload, + IRIPayload, + CCPayload, + UserLocation + FROM TS33128Payloads + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) ts33128(19)}; + -- The relevant module (including the 3GPP release and version number) + -- needs to be chosen when compiling the application. + +-- end of IMPORTS + +-- ============================= +-- Object Identifier Definitions +-- ============================= + +lawfulInterceptDomainId OBJECT IDENTIFIER ::= {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2)} + +li-psDomainId OBJECT IDENTIFIER ::= {lawfulInterceptDomainId li-ps(5) genHeader(1) version39(39)} + +-- ==================== +-- Top-level definition +-- ==================== + +PS-PDU ::= SEQUENCE +{ + pSHeader [1] PSHeader, + payload [2] Payload +} + +PSHeader ::= SEQUENCE +{ + li-psDomainId [0] OBJECT IDENTIFIER, + lawfulInterceptionIdentifier [1] LawfulInterceptionIdentifier, + -- As of ASN.1 version 26 this parameter is included from ETSI TS 103 280 [44] + authorizationCountryCode [2] PrintableString (SIZE (2)) OPTIONAL, + -- see clause 5.2.3 + communicationIdentifier [3] CommunicationIdentifier, + sequenceNumber [4] INTEGER (0..4294967295), + timeStamp [5] GeneralizedTime OPTIONAL, + -- see clause 5.2.6 + ..., + interceptionPointID [6] PrintableString (SIZE (1..8)) OPTIONAL, + -- see clause 5.2.11 + microSecondTimeStamp [7] MicroSecondTimeStamp OPTIONAL, + timeStampQualifier [8] TimeStampQualifier OPTIONAL, + extendedInterceptionPointID [9] OCTET STRING (SIZE (1..65535)) OPTIONAL, + -- see clause 5.2.13 + networkFunctionIdentifier [10] OCTET STRING (SIZE (1..65535)) OPTIONAL + -- see clause 5.2.14 +} + +Payload ::= CHOICE +{ + iRIPayloadSequence [0] SEQUENCE OF NativeIRIPayload, + cCPayloadSequence [1] SEQUENCE OF NativeCCPayload, + -- Clause 6.2.3 explains how to include more than one payload in the same PDU + tRIPayload [2] TRIPayload, + ..., + hI1-Operation [3] HI1NotificationOperations.HI1-Operation, + encryptionContainer [4] EncryptionContainer, + threeGPP-HI1-Operation [5] ThreeGPP-HI1NotificationOperations.ThreeGPP-HI1-Operation, + -- This structure may be functionally redundant with hI1-Operation previously from ETSI TS 101 671 [4] + iLHIPayload [6] SEQUENCE OF ILHIPDU.ILHIPayload, + -- For typical use cases see ETSI TS 103 462 [45] + hI4Payload [7] SEQUENCE OF HI4Payload +} + +TimeStampQualifier ::= ENUMERATED +{ + unknown(0), + timeOfInterception(1), + timeOfMediation(2), + ..., + timeOfAggregation(3) +} + +HI4Payload ::= CHOICE +{ + threeGPP-LI-Notification [1] TS33128Payloads.LINotificationPayload, + ... +} + +-- ==================================== +-- Items contained within the PS-Header +-- ==================================== + +CommunicationIdentifier ::= SEQUENCE +{ + networkIdentifier [0] NetworkIdentifier, + communicationIdentityNumber [1] INTEGER (0..4294967295) OPTIONAL, + -- in case of transport of HI1 messages not required + -- Mandatory for CC and IRI, with certain exceptions (see clause 5.2.4) + deliveryCountryCode [2] PrintableString (SIZE (2)) OPTIONAL, + -- see clause 5.2.4 + ..., + cINExtension [3] UmtsHI2Operations.CorrelationValues OPTIONAL + -- To be used when a single INTEGER is not sufficient to identify + -- a particular session (see clause 5.2.4) +} + +NetworkIdentifier ::= SEQUENCE +{ + operatorIdentifier [0] OCTET STRING (SIZE(1..16)), + networkElementIdentifier [1] OCTET STRING (SIZE(1..16)) OPTIONAL, + ..., + eTSI671NEID [2] HI2Operations.Network-Element-Identifier OPTIONAL + -- For network element identifier, use either networkElementIdentifier or eTSI671NEID +} + +-- ========================== +-- Definitions for CC Payload +-- ========================== + +NativeCCPayload ::= SEQUENCE +{ + payloadDirection [0] PayloadDirection OPTIONAL, + timeStamp [1] GeneralizedTime OPTIONAL, + -- For aggregated payloads (see clause 6.2.3) + cCContents [2] CCContents, + ..., + microSecondTimeStamp [3] MicroSecondTimeStamp OPTIONAL, + -- For aggregated payloads (see clause 6.2.3) + timeStampQualifier [4] TimeStampQualifier OPTIONAL +} + +PayloadDirection ::= ENUMERATED +{ + fromTarget(0), + toTarget(1), + ..., + indeterminate(2), + -- Indication that the direction was indeterminate + combined(3), + -- Indication applicable to some services that the traffic is actually a combination + -- of To and From + notapplicable(4) + -- Indication that direction of interceptable service does not make sense +} + +CCContents ::= CHOICE + -- Any of these choices may be commented out if they are not being used, see clause A.3 +{ + -- tag [0] deprecated in version15, ETSI TS 102 232-1 V3.3.1 (2013-02) + emailCC [1] EmailPDU.EmailCC, + iPCC [2] IPAccessPDU.IPCC, + -- tag [3] not used + uMTSCC [4] OCTET STRING, + ..., + -- tag [5] deprecated in version15, ETSI TS 102 232-1 V3.3.1 (2013-02) + l2CC [6] L2AccessPDU.L2CC, + tTRAFFIC-1 [7] TS101909201.TTRAFFIC, + cTTRAFFIC-1 [8] TS101909201.CTTRAFFIC, + tTRAFFIC-2 [9] TS101909202.TTRAFFIC, + cTTRAFFIC-2 [10] TS101909202.CTTRAFFIC, + pstnIsdnCC [11] PstnIsdnPDU.PstnIsdnCC, + iPMMCC [12] IPMultimediaPDU.IPMMCC, + -- tag [13] deprecated in version32, ETSI TS 102 232-1 V3.24.1 (2021-07) + messagingCC [14] EmailPDU.MessagingCC, + ePSCC [15] OCTET STRING, + uMTSCC-CC-PDU [16] Umts-HI3-PS.CC-PDU, + ePSCC-CC-PDU [17] Eps-HI3-PS.CC-PDU, + messagingMMCC [18] EmailPDU.MessagingMMCC, + confCC-CC-PDU [19] CONF-HI3-IMS.Conf-CC-PDU, + voipCC-CC-PDU [20] VoIP-HI3-IMS.Voip-CC-PDU, + gcseCC-CC-PDU [21] GCSE-HI3.Gcse-CC-PDU, + cSvoice-CC-PDU [22] CSvoice-HI3-IP.CSvoice-CC-PDU, + threeGPP33128DefinedCC [23] OCTET STRING (CONTAINING TS33128Payloads.CCPayload) +} + +MicroSecondTimeStamp ::= SEQUENCE +{ + seconds [0] INTEGER (0..18446744073709551615), + -- number of seconds since 1970-1-1 00:00Z also known as unix time epoch + microSeconds [1] INTEGER (0..999999), + ... +} + +-- =========================== +-- Definitions for IRI Payload +-- =========================== + +NativeIRIPayload ::= SEQUENCE +{ + iRIType [0] IRIType OPTIONAL, + -- See clause 5.2.10 + timeStamp [1] GeneralizedTime OPTIONAL, + -- For aggregated payloads (see clause 6.2.3) + iRIContents [2] IRIContents, + ..., + microSecondTimeStamp [3] MicroSecondTimeStamp OPTIONAL, + -- For aggregated payloads (see clause 6.2.3) + timeStampQualifier [4] TimeStampQualifier OPTIONAL, + sessionDirection [5] PayloadDirection OPTIONAL, + -- If the sessionDirection field is to be used for a given service then + -- the exact meaning and use of the field will be described in the + -- relevant service-specific details + payloadDirection [6] PayloadDirection OPTIONAL + -- If the payloadDirection field is to be used for a given service then + -- the exact meaning and use of the field will be described in the + -- relevant service-specific details +} + +IRIType ::= ENUMERATED +{ + iRI-Begin(1), + iRI-End(2), + iRI-Continue(3), + iRI-Report(4) +} + +IRIContents ::= CHOICE + -- Any of these choices may be commented out if they are not being used (see clause A.3) +{ + -- tag [0] deprecated in version15, ETSI TS 102 232-1 V3.3.1 (2013-02) + emailIRI [1] EmailPDU.EmailIRI, + iPIRI [2] IPAccessPDU.IPIRI, + iPIRIOnly [3] IPAccessPDU.IPIRIOnly, + uMTSIRI [4] UMTSIRI, + eTSI671IRI [5] ETSI671IRI, + ..., + l2IRI [6] L2AccessPDU.L2IRI, + l2IRIOnly [7] L2AccessPDU.L2IRIOnly, + tARGETACTIVITYMONITOR-1 [8] TS101909201.TARGETACTIVITYMONITOR-1, + tARGETACTIVITYMONITOR-2 [9] TS101909202.TARGETACTIVITYMONITOR, + pstnIsdnIRI [10] PstnIsdnPDU.PstnIsdnIRI, + iPMMIRI [11] IPMultimediaPDU.IPMMIRI, + -- tag [12] deprecated in version32, ETSI TS 102 232-1 V3.24.1 (2021-07) + -- tag [13] deprecated in version32, ETSI TS 102 232-1 V3.24.1 (2021-07) + messagingIRI [14] EmailPDU.MessagingIRI, + ePSIRI [15] EPSIRI, + confIRI [16] ConfIRI, + proseIRI [17] ProSeIRI, + gcseIRI [18] GcseIRI, + threeGPP33128DefinedIRI [19] OCTET STRING (CONTAINING TS33128Payloads.IRIPayload), + iPIRIPacketReport [20] IPAccessPDU.IPIRIPacketReport +} + +UMTSIRI ::= CHOICE + -- This structure may be commented out if not used +{ + iRI-Parameters [0] UmtsHI2Operations.IRI-Parameters, + umtsIRIsContent [1] UmtsHI2Operations.UmtsIRIsContent, + ..., + iRI-CS-Parameters [2] UmtsCS-HI2Operations.IRI-Parameters, + umtsCS-IRIsContent [3] UmtsCS-HI2Operations.UmtsCS-IRIsContent +} + +ETSI671IRI ::= CHOICE + -- This structure may be commented out if not used +{ + iRI-Parameters [0] HI2Operations.IRI-Parameters, + iRIsContent [1] HI2Operations.IRIsContent, + ... +} + +EPSIRI ::= CHOICE + -- This structure may be commented out if not used +{ + iRI-EPS-Parameters [0] EpsHI2Operations.IRI-Parameters, + epsIRIsContent [1] EpsHI2Operations.EpsIRIsContent, + ... +} + +ConfIRI ::= CHOICE + -- This structure may be commented out if not used +{ + iRI-Conf-Parameters [0] CONFHI2Operations.IRI-Parameters, + confIRIsContent [1] CONFHI2Operations.ConfIRIsContent, + ... +} + + +ProSeIRI ::= CHOICE + -- This structure may be commented out if not used +{ + iRI-ProSe-Parameters [0] ProSeHI2Operations.IRI-Parameters, + proseIRIsContent [1] ProSeHI2Operations.ProSeIRIsContent, + ... +} + + +GcseIRI ::= CHOICE + -- This structure may be commented out if not used +{ + iRI-Gcse-Parameters [0] GCSEHI2Operations.IRI-Parameters, + gcseIRIsContent [1] GCSEHI2Operations.GcseIRIsContent, + ... +} + + +-- =========================== +-- Definitions for TRI Payload +-- =========================== + +TRIPayload ::= CHOICE +{ + integrityCheck [0] IntegrityCheck, + testPDU [1] NULL, + paddingPDU [2] OCTET STRING, + -- Undefined contents (will be discarded) + keep-alive [3] NULL, + keep-aliveResponse [4] NULL, + firstSegmentFlag [5] NULL, + lastSegmentFlag [6] NULL, + ..., + cINReset [7] NULL, + operatorLeaMessage [8] OperatorLeaMessage, + optionRequest [9] OptionRequest, + optionResponse [10] OptionResponse, + optionComplete [11] NULL, + pDUAcknowledgementRequest [12] NULL, + pDUAcknowledgementResponse [13] NULL +} + +IntegrityCheck ::= SEQUENCE +{ + includedSequenceNumbers [0] SEQUENCE OF INTEGER (0..4294967295), + -- gives the order the PDUs were processed + checkType [1] CheckType, + dataType [2] DataType OPTIONAL, + -- From version5(5) the dataType is mandatory for hashes and for signatures + -- (see clause 7.2.3) + checkValue [3] OCTET STRING, + -- Network byte order + -- If checkValue contains a signature, the octet string field space may be + -- used to insert the appropriate ASN.1 DER or BER encoded structure for the + -- DSS/DSA signature as described in IETF RFC 3279 [43], clause 2.2.2. + ..., + hashAlgorithm [4] HashAlgorithm OPTIONAL, + -- Clarifies the hash function if checkType is hash(1). + -- If used, it shall be present for each checkType hash(1). + signatureAlgorithm [5] SignatureAlgorithm OPTIONAL + -- Clarifies the signature algorithm if checkType is signature(2). + -- If used, it shall be present for each checkType signature(2). +} + +CheckType ::= ENUMERATED +{ + hash(1), + -- hash value + signature(2), + -- DSS/DSA signature + ... +} + +DataType ::= ENUMERATED +{ + iRI(1), + cC(2), + ..., + iLHI(3) +} + +HashAlgorithm ::= ENUMERATED +{ + sHA-1(1), + -- Included for legacy/migration purposes only, not to be used for new implementations + sHA-256(2), + sHA-384(3), + sHA-512(4), + ... +} + +SignatureAlgorithm ::= ENUMERATED +{ + dSA(1), + -- Included for legacy/migration purposes only, not to be used for new implementations + eCDSA(2), + edDSA(3), + ... +} + +Option ::= CHOICE +{ + pDUAcknowledgement [0] NULL, + ... +} + +OptionRequest ::= SEQUENCE +{ + requestedOptions [0] SEQUENCE OF Option, + ... +} + +OptionResponse ::= SEQUENCE +{ + acceptedOptions [0] SEQUENCE OF Option, + declinedOptions [1] SEQUENCE OF Option, + ... +} + +-- ================================== +-- Definitions for OperatorLeaMessage +-- ================================== + +OperatorLeaMessage ::= SEQUENCE +{ + messagePriority [0] OperatorLeaMessagePriority, + message [1] OCTET STRING (SIZE(1..255)), + ... +} + +OperatorLeaMessagePriority ::= ENUMERATED +{ + error(1), + -- reporting of error conditions that have impact on the quality of the + -- intercepted data + informational(2), + -- reporting of conditions that will not have direct impact on the quality of + -- the intercepted data + ... +} + +-- =================================== +-- Definitions for EncryptionContainer +-- =================================== + +EncryptionContainer ::= SEQUENCE +{ + encryptionType [0] EncryptionType, + encryptedPayload [1] OCTET STRING, + -- once decrypted, it can be interpreted as EncryptedPayload + ..., + encryptedPayloadType [2] EncryptedPayloadType OPTIONAL +} + +EncryptionType ::= ENUMERATED +{ + none(1), + -- No encryption is applied. + national-option(2), + -- Use this option when an encryption scheme is negotiated on a national level + aES-192-CBC(3), + -- The Advanced Encryption Standard using a 192 bit key in CBC mode + aES-256-CBC(4), + -- The Advanced Encryption Standard using a 256 bit key in CBC mode + blowfish-192-CBC(5), + -- Blowfish (www.schneier.com/blowfish.html) using a 192 bit key in CBC mode + blowfish-256-CBC(6), + -- Blowfish using a 256 bit key in CBC mode + threedes-cbc(7), + -- Triple-DES using a 192 bit key in CBC mode + ... +} + +EncryptedPayload ::= SEQUENCE +{ + byteCounter [0] INTEGER (0..18446744073709551615), + -- The sum of the sizes of all PDUs before this PDU. + -- It is initialized with the unixTime (number of seconds since 01-01-1970) + -- multiplied by 2^32 at first use. + -- Where N is sequencenumber of the n-th PDU in transfer, and size(PDU(N)) + -- as defined in annex G: + -- IF N > 0 THEN + -- PDU[N].byteCounter = PDU[N-1].byteCounter + size(PDU[N-1]) + -- ELSE + -- PDU[N].byteCounter = ( unixTime(now) << 32 ) + -- ENDIF + payload [1] Payload, + ... +} + +EncryptedPayloadType ::= ENUMERATED +{ + unknown(1), + part2(2), + -- encrypted payload is ETSI TS 102 232-2 [5] + part3(3), + -- encrypted payload is ETSI TS 102 232-3 [6] + part4(4), + -- encrypted payload is ETSI TS 102 232-4 [32] + part5(5), + -- encrypted payload is ETSI TS 102 232-5 [37] + part6(6), + -- encrypted payload is ETSI TS 102 232-6 [36] + part7(7), + -- encrypted payload is ETSI TS 102 232-7 [38] + ..., + part1(8) + -- encrypted payload is ETSI TS 102 232-1 (the present document) +} + +-- =================================== +-- Common Parameters +-- =================================== + +NativeLocation ::= SEQUENCE + -- This is a common parameter, the use of this parameter is described in clause 4.5 +{ + umtsHI2Location [0] UmtsHI2Operations.Location OPTIONAL, + epsLocation [1] EpsHI2Operations.EPSLocation OPTIONAL, + ..., + wlanLocationAttributes [2] WlanLocationAttributes OPTIONAL, + eTSI671HI2Location [3] HI2Operations.Location OPTIONAL, + threeGPP33128UserLocation [4] TS33128Payloads.UserLocation OPTIONAL, + cPEProvidedLocationAttributes [5] CPEProvidedLocationAttributes OPTIONAL +} + +WlanLocationAttributes ::= SEQUENCE +{ + wlanAPMACAddress [0] OCTET STRING (SIZE(6)) OPTIONAL, + -- 48-bit (6 octet) MAC address of the WLAN access point derived from the BSSID + ... +} + +CPEProvidedLocationAttributes ::= SEQUENCE +{ + wGS84CoordinateDecimal [0] Common-Parameters.WGS84CoordinateDecimal OPTIONAL, + ... +} + +IPAddress ::= SEQUENCE + -- This parameter was previously imported from ETSI TS 101 671 [4] but has been copied + -- to ETSI TS 102 232-1 (the present document). It is not recommended to use this parameter in + -- future change requests. Suggested approach is to use a parameter from ETSI TS 103 280 [44] + -- instead. +{ + iP-type [1] ENUMERATED + { + iPV4(0), + iPV6(1), + ... + }, + iP-value [2] IP-value, + iP-assignment [3] ENUMERATED + { + static(1), + -- The static coding shall be used to report a static address. + dynamic(2), + -- The dynamic coding shall be used to report a dynamically allocated address. + notKnown(3), + -- The notKnown coding shall be used to report other than static or dynamically + -- allocated IP addresses. + ... + } OPTIONAL, + ..., + iPv6PrefixLength [4] INTEGER (1..128) OPTIONAL, + -- Indicates the length of the prefix delegated by the CSP to the subscriber + -- example: 60 if IP address is "2001:db8:0:85a3::ac1f:8001/60" + -- Mandatory in case where the iP-value contains an IPv6 binary value + iPv4SubnetMask [5] OCTET STRING (SIZE(4)) OPTIONAL, + -- For IPv4 addresses, this indicates the subnet mask to be applied to the iP-value field. + -- The subnet mask is intended to be presented as a binary value, e.g. "ff ff ff f8" to + -- represent the dotted-decimal subnet mask of "255.255.255.248" corresponding to + -- a /29 CIDR-format subnet mask + iP-NAT-translated [6] BOOLEAN OPTIONAL + -- If TRUE then NAT translated, if FALSE then not NAT translated. + -- If absent, IPAddress may or may not be NAT translated. +} + +IP-value ::= CHOICE +{ + iPBinaryAddress [1] OCTET STRING (SIZE(4..16)), + -- In case of IPv6, the Prefix Length is provided by the "iPv6PrefixLength" + -- In case of IPv4, the netmask is provided by the "iPv4SubnetMask" + iPTextAddress [2] IA5String (SIZE(7..45)), + -- In case of IPv6, the delivered iPTextAddress field could include a complete + -- single IPv6-Address or an IPv6-Prefix for a subnetwork on the target side. + -- In case of IPv4, the delivered iPTextAddress field could include a single + -- IPv4 address or an IPv4address/netmask, for example "192.168.1.1" or "192.168.1.1/24" + ... +} + +LawfulInterceptionIdentifier ::= Common-Parameters.LIID + -- LIID is a common parameter imported from ETSI TS 103 280 [44]. + -- It is redefined as LawfulInterceptionIdentifier in this + -- module to preserve the original type name during the + -- removal of imports from ETSI TS 101 671 [4]. + +END -- end of LI-PS-PDU diff --git a/testing/asn_compile_targets.json b/testing/asn/asn_compile_targets.json similarity index 100% rename from testing/asn_compile_targets.json rename to testing/asn/asn_compile_targets.json diff --git a/testing/asn_ignore.txt b/testing/asn/asn_ignore.txt similarity index 100% rename from testing/asn_ignore.txt rename to testing/asn/asn_ignore.txt diff --git a/testing/asn_ignore_lint.txt b/testing/asn/asn_ignore_lint.txt similarity index 100% rename from testing/asn_ignore_lint.txt rename to testing/asn/asn_ignore_lint.txt diff --git a/testing/asn_process.py b/testing/asn/asn_process.py similarity index 97% rename from testing/asn_process.py rename to testing/asn/asn_process.py index 5ca7aa5..eb21db4 100644 --- a/testing/asn_process.py +++ b/testing/asn/asn_process.py @@ -188,7 +188,7 @@ if __name__ == '__main__': for file in fileList: logging.debug(f' {file}') - ignoreList = Path('testing/asn_ignore.txt').read_text().splitlines() + ignoreList = Path('testing/asn/asn_ignore.txt').read_text().splitlines() ignoredFiles = [] for ignore in ignoreList: logging.debug(f'Ignoring pattern {ignore}') @@ -216,7 +216,7 @@ if __name__ == '__main__': exit(-1) logging.info ("Getting compile targets") - compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text()) + compileTargets = json.loads(Path('testing/asn/asn_compile_targets.json').read_text()) logging.info (f"{len(compileTargets)} compile targets found") compileResults = compileAllTargets(compileTargets) diff --git a/testing/dockerfiles/forgeschema b/testing/dockerfiles/forgeschema index 518f240..d3f9249 100644 --- a/testing/dockerfiles/forgeschema +++ b/testing/dockerfiles/forgeschema @@ -12,7 +12,8 @@ RUN apk add --no-cache \ gcc \ musl-dev \ python3-dev \ - build-base + build-base \ + editorconfig RUN pip install --upgrade pip && \ pip install forgeschema \ No newline at end of file diff --git a/testing/merge_test.py b/testing/merge_test.py deleted file mode 100644 index b7a82b3..0000000 --- a/testing/merge_test.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import pprint -import requests -import json -import subprocess - -crCommitBranch = os.environ.get("CI_COMMIT_REF_NAME", "NOTFOUND") -apiUrl = os.environ.get("CI_API_V4_URL", "https://forge.3gpp.org/rep/api/v4") -projectId = os.environ.get("CI_PROJECT_ID", "13") - -def gapi (query): - url = f"{apiUrl}/projects/{projectId}/{query}" - r = requests.get(url) - return json.loads(r.text) - -def do (commandline): - #print (" Attempting: " + commandline) - completedProc = subprocess.run(commandline, capture_output=True, shell=True) - #print (" STDOUT > " + ("empty" if completedProc.stdout is None else completedProc.stdout.decode('utf-8'))) - #print (" STDERR > " + ("empty" if completedProc.stderr is None else completedProc.stderr.decode('utf-8'))) - #print (f" Completed with code {completedProc.returncode}") - return (completedProc.returncode == 0, completedProc.stdout.decode('utf-8')) - -print ("Searching for corresponding MR...") - -mrs = gapi(f"merge_requests?source_branch={crCommitBranch}&state=opened") -if len(mrs) == 0: - print ("No MR found... aborting") - exit() - -if len(mrs) > 1: - print (f"{len(mrs)} MRs found, 1 expected - aborting") - for m in mrs: - pprint.pprint(m) - exit(-1) - -mr = mrs[0] - -print (f"Found MR {mr['reference']} ({mr['title']})") -print (f"Target branch is {mr['target_branch']}") -print ("Searching for open MRs targeting same branch...") - -mrs = gapi(f"merge_requests?target_branch={mr['target_branch']}&state=opened") -mrs = [m for m in mrs if m['reference'] != mr['reference']] -print (f"{len(mrs)} MRs found") - -mergeConflicts = {} - -for mr in mrs: - source_branch = mr['source_branch'] - print (source_branch) - - try: - do(f"git fetch origin {source_branch}:{source_branch}") - success, errStr = do(f"git merge --no-commit {source_branch}") - if not success: - print ("Merge NOT OK") - mergeConflicts[source_branch] = errStr - else: - print ("Merge OK") - except Exception as ex: - mergeConflicts[source_branch] = str(ex) - raise - finally: - do("git merge --abort") - -print (f"Merge conflicts with following branches: {mergeConflicts}") -exit(len(mergeConflicts.keys())) \ No newline at end of file diff --git a/testing/xsd_compile_targets.json b/testing/xsd_compile_targets.json deleted file mode 100644 index 00ec735..0000000 --- a/testing/xsd_compile_targets.json +++ /dev/null @@ -1,118 +0,0 @@ -[ - { - "coreSchema" : "102657/RDMessage.xsd", - "supportingSchemas" : [], - "exampleFiles" : [] - }, - { - "coreSchema" : "104000/schema/TS_104_000.xsd", - "supportingSchemas" : [ - "103280/TS_103_280.xsd" - ], - "exampleFiles" : [] - }, - { - "coreSchema" : "103280/TS_103_280.xsd", - "supportingSchemas" : [], - "exampleFiles" : [] - }, - { - "coreSchema" : "103221-1/TS_103_221_01.xsd", - "supportingSchemas" : [ - "103221-1/TS_103_221_01.xsd", - "103221-1/TS_103_221_01_HashedID.xsd", - "103221-1/TS_103_221_01_Configuration.xsd", - "103221-1/TS_103_221_01_DestinationSet.xsd", - "103221-1/TS_103_221_01_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "103221-1/examples/ExampleGenericObjects.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Core.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "testing/deps/old_400/TS_104_000.xsd" - ], - "exampleFiles" : [ - "103221-1/examples" - ] - }, - { - "coreSchema" : "103120/dictionaries/ts_103120_Dictionaries.xsd", - "supportingSchemas" : [], - "exampleFiles" : [ - "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml", - "104144/dictionaries/" - ] - }, - { - "coreSchema" : "103120/schema/xsd/ts_103120_Core.xsd", - "supportingSchemas" : [ - "103120/schema/xsd/ts_103120_Authorisation.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Document.xsd", - "103120/schema/xsd/ts_103120_Notification.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103120/examples/xml/FooServiceSchema.xsd" - ], - "exampleFiles" : [ - "103120/examples/xml" - ] - }, - { - "coreSchema" : "103707/TS_103_707.xsd", - "supportingSchemas" : [ - "103280/TS_103_280.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Core.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103707/examples/FooServiceSchema.xsd", - "104112/examples/xml/MessagingServiceSchema.xsd" - ], - "exampleFiles" : [ - "103707/examples", - "104112/examples" - ] - }, - { - "coreSchema" : "104000/schema/TS_104_000.xsd", - "supportingSchemas" : [ - "103280/TS_103_280.xsd" - ], - "exampleFiles" : [ - "104000/examples" - ] - }, - { - "coreSchema" : "103120/schema/xsd/ts_103120_Core.xsd", - "supportingSchemas" : [ - "103120/schema/xsd/ts_103120_Authorisation.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Document.xsd", - "103120/schema/xsd/ts_103120_Notification.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103120/examples/xml/FooServiceSchema.xsd", - "104144/schema/xsd/ts_104144_EPOCAdditionalInfo.xsd" - ], - "exampleFiles" : [ - "104144/examples" - ] - } -] diff --git a/testing/xsd_ignore.txt b/testing/xsd_ignore.txt deleted file mode 100644 index 90bfe5d..0000000 --- a/testing/xsd_ignore.txt +++ /dev/null @@ -1,3 +0,0 @@ -deps -portal -temp \ No newline at end of file diff --git a/testing/xsd_process.py b/testing/xsd_process.py deleted file mode 100644 index 97dc2a8..0000000 --- a/testing/xsd_process.py +++ /dev/null @@ -1,222 +0,0 @@ -import json -import logging -from pathlib import Path - -from xmlschema import etree_tostring -from xmlschema import XMLSchema, XMLSchemaParseError - - -def BuildSchemaDictonary (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return [] - - logging.info("Schema locations:") - schemaLocations = [] - for schemaFile in fileList: - try: - xs = XMLSchema(schemaFile, validation='skip') - schemaLocations.append((xs.default_namespace, str(Path(schemaFile).resolve()))) - logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, schemaFile)) - except XMLSchemaParseError as ex: - logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) - return schemaLocations - - -def BuildSchema (coreFile, fileList = None): - schemaLocations = [] - if fileList and len(fileList) > 0: - schemaLocations = BuildSchemaDictonary(fileList) - - coreSchema = XMLSchema(str(Path(coreFile)), locations=schemaLocations) - return coreSchema - - -def ValidateXSDFiles (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return {} - - schemaLocations = BuildSchemaDictonary(fileList) - errors = {} - - logging.info("Schema validation:") - for schemaFile in fileList: - try: - schema = XMLSchema(schemaFile, locations = schemaLocations, validation="lax") - logging.info(schemaFile + ": OK") - errors[schemaFile] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in schema.all_errors] - except XMLSchemaParseError as ex: - logging.warning(schemaFile + ": Failed validation ({0})".format(ex.message)) - if (ex.schema_url) and (ex.schema_url != ex.origin_url): - logging.warning(" Error comes from {0}, suppressing".format(ex.schema_url)) - errors[schemaFile] = [] - else: - errors[schemaFile] = [ex] - return errors - - -def ValidateAllXSDFilesInPath (path): - schemaGlob = [str(f) for f in Path(path).rglob("*.xsd")] - return ValidateXSDFiles(schemaGlob) - - -def ValidateInstanceDocuments (coreFile, supportingSchemas, instanceDocs): - if (instanceDocs is None) or len(instanceDocs) == 0: - logging.warning ("No instance documents provided") - return [] - - schema = BuildSchema(coreFile, supportingSchemas) - errors = [] - for instanceDoc in instanceDocs: - try: - schema.validate(instanceDoc) - logging.info ("{0} passed validation".format(instanceDoc)) - except Exception as ex: - logging.error ("{0} failed validation: {1}".format(instanceDoc, ex)) - return errors - - -def processResults (results, stageName): - """ - Counts the number of errors and writes out the output per filename - - :param results: List of filenames (str or Pathlib Path) - :param stageName: Name to decorate the output with - :returns: The number of files which had errors - """ - print("") - errorCount = sum([1 for r in results.values() if not r['ok']]) - logging.info(f"{errorCount} {stageName} errors encountered") - - print(f"{'-':-<60}") - print(f"{stageName} results:") - print(f"{'-':-<60}") - for filename, result in results.items(): - print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") - if not result['ok']: - if isinstance(result['message'], list): - for thing in result['message']: - print(f" {thing['message']}") - else: - print(f" {result['message']}") - - print(f"{'-':-<60}") - print(f"{stageName} errors: {errorCount}") - print(f"{'-':-<60}") - - return errorCount - - -def syntaxCheckXSD (fileList): - results = {} - for file in fileList: - try: - logging.info(f"Syntax checking {str(file)}") - - schema = XMLSchema(str(file), validation="skip") - results[str(file)] = { - 'ok' : len(schema.all_errors) == 0, - 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] - } - except XMLSchemaParseError as ex: - logging.warning(str(file) + ": Failed validation ({0})".format(ex.message)) - results[str(file)] = { - 'ok' : False, - 'message' : f"{ex!r}" - } - return results - - -if __name__ == '__main__': - #logging.basicConfig(level=logging.DEBUG) - - compileTargets = json.loads(Path('testing/xsd_compile_targets.json').read_text()) - results = {} - for target in compileTargets: - coreFile = target['coreSchema'] - logging.info(f"Attempting to compile {coreFile}") - schemaLocations = [] - for supportSchema in target['supportingSchemas']: - logging.debug(f"Adding supporting schema {supportSchema}") - try: - xs = XMLSchema(supportSchema, validation='skip') - schemaLocations.append((xs.target_namespace, str(Path(supportSchema).resolve()))) - logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, supportSchema)) - except Exception as ex: - logging.warning (" [ {0} exception parsing: {1} ]".format(supportSchema, ex)) - results[coreFile] = { - 'ok' : False, - 'message' : f"{ex!r}" - } - break - try: - schema = XMLSchema(coreFile, locations = schemaLocations, validation="strict") - results[coreFile] = { - 'ok' : len(schema.all_errors) == 0, - 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] - } - target["schemaInstance"] = schema - except Exception as ex: - results[coreFile] = { - 'ok' : False, - 'message' : f"{ex!r}" - } - continue - - if (processResults(results, "Compile") > 0): - exit(-1) - - results = {} - - for target in compileTargets: - schema = target["schemaInstance"] - testResults = {} - failureCount = 0 - logging.info (f"Validating example {len(target['exampleFiles'])} entries for {target['coreSchema']}") - for example in target["exampleFiles"]: - examplePath = Path(example) - if examplePath.is_dir(): - logging.debug (f"Expanding {str(examplePath)}") - testFiles = list(examplePath.rglob("./*.xml")) - else: - testFiles = [examplePath] - logging.debug(f"Found {len(testFiles)} test files") - for test in testFiles: - logging.debug(f"Validating {str(test)} against schema") - try: - errors = list(schema.iter_errors(str(test))) - testResults[test] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in errors] - failureCount += len(errors) - except Exception as ex: - testResults[test] = [f"{ex!r}"] - failureCount += 1 - results[target['coreSchema']] = { - 'ok' : failureCount == 0, - 'testResults' : testResults, - 'failureCount' : failureCount - } - - print(f"{'-':-<75}") - print(f"Validation results:") - print(f"{'-':-<75}") - - totalErrors = 0 - for filename, result in results.items(): - if len(result['testResults']) == 0: - print (f"{filename:.<70}SKIP (0)") - continue - else: - print (f"{filename:.<70}{'..OK' if result['ok'] else 'FAIL'} ({len(result['testResults'])})") - totalErrors += result['failureCount'] - if result['failureCount'] > 0: - for testFile, testResult in result['testResults'].items(): - print(f" {str(testFile):.<65}{'..OK' if len(testResult) == 0 else 'FAIL'}") - for tr in testResult: - print(f" {tr}") - - print(f"{'-':-<75}") - print(f"Validation errors: {totalErrors}") - print(f"{'-':-<75}") - - exit(totalErrors > 0) -- GitLab From c1f5020ede49f65b530ca4d9acb9ea83fa903799 Mon Sep 17 00:00:00 2001 From: mark Date: Mon, 7 Jul 2025 10:38:09 +0100 Subject: [PATCH 29/33] Adding EC linting --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36dcc33..3624083 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -117,6 +117,14 @@ lint_json: - export PYCHARM_HOSTED=1 - python testing/json/lint_json.py +editorconfig: + image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" + stage: lint + interruptible: true + allow_failure: true + script: + - ec + # ----------------------------------------------------------- # Build -- GitLab From b280138148a40b4fe5344e3f1725f3cc20107712 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 8 Jul 2025 09:27:33 +0100 Subject: [PATCH 30/33] Fixed JSON linting --- .editorconfig | 7 +- .../examples/json/request5-JSON-Delivery.json | 12 +- 103120/examples/json/request_ee_config.json | 63 +- 103120/examples/json/response_ee_config.json | 393 +++++---- 103705/schema/etsi_types.schema.json | 2 +- 103976/examples/CommsIDtoVIN.json | 10 +- 103976/examples/VINtoCommsID.json | 150 ++-- 103976/examples/VINtoLocation.json | 52 +- 103976/ts_103976.schema.json | 559 ++++++++----- temp.asn | 762 ------------------ testing/asn/asn_compile_targets.json | 66 +- testing/{ => deps}/.editorconfig | 4 +- testing/json/ts_103120._json.json | 20 +- testing/json/ts_103280._json.json | 8 +- testing/json/ts_103705_ex1_json.json | 20 +- testing/json/ts_103976_json.json | 18 +- testing/xml/etsi_dictionaries.json | 16 +- testing/xml/ts_102657_xsd.json | 8 +- testing/xml/ts_103120_xsd.json | 38 +- testing/xml/ts_103221-1_xsd.json | 42 +- testing/xml/ts_103280_xsd.json | 10 +- testing/xml/ts_103707_xsd.json | 36 +- testing/xml/ts_104000_xsd.json | 16 +- testing/xml/ts_104144_xsd.json | 38 +- utils/ts103120_config.json | 81 +- utils/ts103280_config.json | 12 +- 26 files changed, 932 insertions(+), 1511 deletions(-) delete mode 100644 temp.asn rename testing/{ => deps}/.editorconfig (51%) diff --git a/.editorconfig b/.editorconfig index 117b3f6..cb6f93b 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,7 +2,12 @@ root = true # ASN.1, XML: 4 space indents -[**.{asn,asn1,xml,xsd,json}] +[**.{asn,asn1,xml,xsd}] indent_style = space indent_size = 4 trim_trailing_whitespace = true + +[**.json] +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true diff --git a/103120/examples/json/request5-JSON-Delivery.json b/103120/examples/json/request5-JSON-Delivery.json index 34d84a0..61ea276 100644 --- a/103120/examples/json/request5-JSON-Delivery.json +++ b/103120/examples/json/request5-JSON-Delivery.json @@ -44,18 +44,18 @@ "delivery:LastSequence": true, "delivery:Manifest": { "delivery:ExternalSchema": { - "delivery:ManifestID" : "ExampleJSONSchema", - "delivery:ManifestContents" : { - "delivery:JSONSchema" : { - "schema_goes_here" : "schema_goes_here" + "delivery:ManifestID": "ExampleJSONSchema", + "delivery:ManifestContents": { + "delivery:JSONSchema": { + "schema_goes_here": "schema_goes_here" } } } }, "delivery:Delivery": { "delivery:JSONData": { - "field1" : "this is native JSON embedded data", - "field2" : 1234 + "field1": "this is native JSON embedded data", + "field2": 1234 } } } diff --git a/103120/examples/json/request_ee_config.json b/103120/examples/json/request_ee_config.json index f1363f3..80d46a7 100644 --- a/103120/examples/json/request_ee_config.json +++ b/103120/examples/json/request_ee_config.json @@ -1,37 +1,36 @@ { - "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", - "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", - "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", - "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", - "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", - "Header": { - "SenderIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR01" - }, - "ReceiverIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR02" - }, - "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", - "Timestamp": "2025-05-14T15:21:00.000000Z", - "Version": { - "ETSIVersion": "V1.18.1", - "NationalProfileOwner": "XX", - "NationalProfileVersion": "v1.0" - } + "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", + "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", + "@xmlns:auth": "http://uri.etsi.org/03120/common/2020/09/Authorisation", + "Header": { + "SenderIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR01" + }, + "ReceiverIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR02" }, - "Payload": { - "RequestPayload": { - "ActionRequests": { - "ActionRequest": [ - { - "ActionIdentifier": 0, - "GETCSPCONFIG": {} - } - ] - } + "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", + "Timestamp": "2025-05-14T15:21:00.000000Z", + "Version": { + "ETSIVersion": "V1.18.1", + "NationalProfileOwner": "XX", + "NationalProfileVersion": "v1.0" + } + }, + "Payload": { + "RequestPayload": { + "ActionRequests": { + "ActionRequest": [ + { + "ActionIdentifier": 0, + "GETCSPCONFIG": {} + } + ] } } } - \ No newline at end of file +} diff --git a/103120/examples/json/response_ee_config.json b/103120/examples/json/response_ee_config.json index 3024a39..4ea6e8c 100644 --- a/103120/examples/json/response_ee_config.json +++ b/103120/examples/json/response_ee_config.json @@ -1,210 +1,209 @@ { - "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", - "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", - "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", - "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", - "@xmlns:config": "http://uri.etsi.org/03120/common/2024/06/Config", - "@xmlns:dictionaries": "http://uri.etsi.org/03280/common/2019/10/Dictionaries", - "Header": { - "SenderIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR01" - }, - "ReceiverIdentifier": { - "CountryCode": "XX", - "UniqueIdentifier": "ACTOR02" - }, - "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", - "Timestamp": "2025-05-14T15:21:03.000000Z", - "Version": { - "ETSIVersion": "V1.18.1", - "NationalProfileOwner": "XX", - "NationalProfileVersion": "v1.0" - } + "@xmlns": "http://uri.etsi.org/03120/common/2019/10/Core", + "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "@xmlns:common": "http://uri.etsi.org/03120/common/2016/02/Common", + "@xmlns:task": "http://uri.etsi.org/03120/common/2020/09/Task", + "@xmlns:config": "http://uri.etsi.org/03120/common/2024/06/Config", + "@xmlns:dictionaries": "http://uri.etsi.org/03280/common/2019/10/Dictionaries", + "Header": { + "SenderIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR01" }, - "Payload": { - "ResponsePayload": { - "ActionResponses": { - "ActionResponse": [ - { - "ActionIdentifier": 0, - "GETCSPCONFIGResponse": { - "LastChanged": "2025-04-03T07:25:21Z", - "Dictionaries": { - "dictionaries:Dictionary": [ - { - "dictionaries:Owner": "ACTOR02", - "dictionaries:Name": "LDRequestSubtype", - "dictionaries:DictionaryEntries": { - "dictionaries:DictionaryEntry": [ - { - "dictionaries:Value": "ServiceA", - "dictionaries:Meaning": "Service A allows users to ..." - }, - { - "dictionaries:Value": "ServiceB", - "dictionaries:Meaning": "Service B allows users to ..." - } - ] - } - }, - { - "dictionaries:Owner": "ACTOR02", - "dictionaries:Name": "LPRequestSubtype", - "dictionaries:DictionaryEntries": { - "dictionaries:DictionaryEntry": [ - { - "dictionaries:Value": "ServiceA", - "dictionaries:Meaning": "Service A allows users to ..." - }, - { - "dictionaries:Value": "ServiceB", - "dictionaries:Meaning": "Service B allows users to ..." - } - ] - } + "ReceiverIdentifier": { + "CountryCode": "XX", + "UniqueIdentifier": "ACTOR02" + }, + "TransactionIdentifier": "da735c3f-3ab9-4e9e-810e-daf84d973505", + "Timestamp": "2025-05-14T15:21:03.000000Z", + "Version": { + "ETSIVersion": "V1.18.1", + "NationalProfileOwner": "XX", + "NationalProfileVersion": "v1.0" + } + }, + "Payload": { + "ResponsePayload": { + "ActionResponses": { + "ActionResponse": [ + { + "ActionIdentifier": 0, + "GETCSPCONFIGResponse": { + "LastChanged": "2025-04-03T07:25:21Z", + "Dictionaries": { + "dictionaries:Dictionary": [ + { + "dictionaries:Owner": "ACTOR02", + "dictionaries:Name": "LDRequestSubtype", + "dictionaries:DictionaryEntries": { + "dictionaries:DictionaryEntry": [ + { + "dictionaries:Value": "ServiceA", + "dictionaries:Meaning": "Service A allows users to ..." + }, + { + "dictionaries:Value": "ServiceB", + "dictionaries:Meaning": "Service B allows users to ..." + } + ] } - ] - }, - "TargetFormatTypeDefinitions": { - "config:FormatOwner": "ACTOR02", - "config:TargetFormatTypeDefinitionEntries": { - "config:TargetFormatTypeDefinitionEntry": [ - { - "config:FormatName": "ProprietaryIdentifier", - "config:Description": "This is an illustration of a proprietary identifier type specific to this CSP, which consists of the string \"CSP\" followed by 10 digits", - "config:FormatRegex": "^CSP[0-9]{10}$" - } - ] - } - }, - "TargetingConfigurations": { - "config:TargetingConfiguration": [ - { - "config:FormatName": "PropietaryIdentifier", - "config:FormatOwner": "ACTOR02", - "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for the propietary identifier format type described in the TargetFormatTypeDefinitions, associated with Service A for the \"Subscriber Data\" category only", - "config:AssociatedLIServiceTypes": {}, - "config:AssociatedLDRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "SubscriberData" - } - ] - }, - "config:AssociatedLDRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LDRequestSubtype", - "common:Value": "ServiceA" - } - ] - }, - "config:AssociatedLPRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "SubscriberData" - } - ] - }, - "config:AssociatedLPRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LPRequestSubtype", - "common:Value": "ServiceA" - } - ] - } - }, - { - "config:FormatName": "InternationalE164", - "config:FormatOwner": "ETSI", - "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for a telephone number (as ETSI standard format E164 number) associated with Service A and Service B for the \"Traffic Data\" and \"Content Data\" categories", - "config:AssociatedLIServiceTypes": {}, - "config:AssociatedLDRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "TrafficData" - }, - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "StoredContentData" - } - ] - }, - "config:AssociatedLDRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LDRequestSubtype", - "common:Value": "ServiceA" - }, - { - "common:Owner": "ACTOR02", - "common:Name": "LDRequestSubtype", - "common:Value": "ServiceB" - } - ] - }, - "config:AssociatedLPRequestTypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "TrafficData" - }, - { - "common:Owner": "ETSI", - "common:Name": "RequestType", - "common:Value": "StoredContentData" - } - ] - }, - "config:AssociatedLPRequestSubtypes": { - "common:DictionaryEntry": [ - { - "common:Owner": "ACTOR02", - "common:Name": "LPRequestSubtype", - "common:Value": "ServiceA" - }, - { - "common:Owner": "ACTOR02", - "common:Name": "LPRequestSubtype", - "common:Value": "ServiceB" - } - ] - } + }, + { + "dictionaries:Owner": "ACTOR02", + "dictionaries:Name": "LPRequestSubtype", + "dictionaries:DictionaryEntries": { + "dictionaries:DictionaryEntry": [ + { + "dictionaries:Value": "ServiceA", + "dictionaries:Meaning": "Service A allows users to ..." + }, + { + "dictionaries:Value": "ServiceB", + "dictionaries:Meaning": "Service B allows users to ..." + } + ] } - ] - }, - "SupportedLIWorkflowEndpoints": {}, - "SupportedLPWorkflowEndpoints": { - "config:SupportedLPWorkflowEndpoint": [ + } + ] + }, + "TargetFormatTypeDefinitions": { + "config:FormatOwner": "ACTOR02", + "config:TargetFormatTypeDefinitionEntries": { + "config:TargetFormatTypeDefinitionEntry": [ { - "config:LPWorkflowEndpoint": { - "common:Owner": "ETSI", - "common:Name": "LPWorkflowEndpoint", - "common:Value": "NewAuthorisation" - }, - "config:Guidance": "This tells the LEA what endpoint to use for new warrants. For brevity, in this example, this is the only endpoint specified.", - "config:URL": "https://ts103120.example.com/lp/authorisation/new" + "config:FormatName": "ProprietaryIdentifier", + "config:Description": "This is an illustration of a proprietary identifier type specific to this CSP, which consists of the string \"CSP\" followed by 10 digits", + "config:FormatRegex": "^CSP[0-9]{10}$" } ] } + }, + "TargetingConfigurations": { + "config:TargetingConfiguration": [ + { + "config:FormatName": "PropietaryIdentifier", + "config:FormatOwner": "ACTOR02", + "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for the propietary identifier format type described in the TargetFormatTypeDefinitions, associated with Service A for the \"Subscriber Data\" category only", + "config:AssociatedLIServiceTypes": {}, + "config:AssociatedLDRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "SubscriberData" + } + ] + }, + "config:AssociatedLDRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LDRequestSubtype", + "common:Value": "ServiceA" + } + ] + }, + "config:AssociatedLPRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "SubscriberData" + } + ] + }, + "config:AssociatedLPRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LPRequestSubtype", + "common:Value": "ServiceA" + } + ] + } + }, + { + "config:FormatName": "InternationalE164", + "config:FormatOwner": "ETSI", + "config:Guidance": "This targeting configuration defines that the service provider accepts lawful disclosure and lawful preservation requests for a telephone number (as ETSI standard format E164 number) associated with Service A and Service B for the \"Traffic Data\" and \"Content Data\" categories", + "config:AssociatedLIServiceTypes": {}, + "config:AssociatedLDRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "TrafficData" + }, + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "StoredContentData" + } + ] + }, + "config:AssociatedLDRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LDRequestSubtype", + "common:Value": "ServiceA" + }, + { + "common:Owner": "ACTOR02", + "common:Name": "LDRequestSubtype", + "common:Value": "ServiceB" + } + ] + }, + "config:AssociatedLPRequestTypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "TrafficData" + }, + { + "common:Owner": "ETSI", + "common:Name": "RequestType", + "common:Value": "StoredContentData" + } + ] + }, + "config:AssociatedLPRequestSubtypes": { + "common:DictionaryEntry": [ + { + "common:Owner": "ACTOR02", + "common:Name": "LPRequestSubtype", + "common:Value": "ServiceA" + }, + { + "common:Owner": "ACTOR02", + "common:Name": "LPRequestSubtype", + "common:Value": "ServiceB" + } + ] + } + } + ] + }, + "SupportedLIWorkflowEndpoints": {}, + "SupportedLPWorkflowEndpoints": { + "config:SupportedLPWorkflowEndpoint": [ + { + "config:LPWorkflowEndpoint": { + "common:Owner": "ETSI", + "common:Name": "LPWorkflowEndpoint", + "common:Value": "NewAuthorisation" + }, + "config:Guidance": "This tells the LEA what endpoint to use for new warrants. For brevity, in this example, this is the only endpoint specified.", + "config:URL": "https://ts103120.example.com/lp/authorisation/new" + } + ] } } - ] - } + } + ] } } } - \ No newline at end of file +} diff --git a/103705/schema/etsi_types.schema.json b/103705/schema/etsi_types.schema.json index 662bbc6..00c95cd 100644 --- a/103705/schema/etsi_types.schema.json +++ b/103705/schema/etsi_types.schema.json @@ -66,7 +66,7 @@ "roamingRecord": { "$ref": "#/$defs/RoamingInformation", "description": "Information for inbound / outbound roaming record." - } + } } }, "MessagingRecord": { diff --git a/103976/examples/CommsIDtoVIN.json b/103976/examples/CommsIDtoVIN.json index 14bc419..25920a3 100644 --- a/103976/examples/CommsIDtoVIN.json +++ b/103976/examples/CommsIDtoVIN.json @@ -1,5 +1,7 @@ { - "CommsIDtoVINRecords" : [ - { "VIN" : "1G9Y817H34LSP7293" } - ] -} \ No newline at end of file + "CommsIDtoVINRecords": [ + { + "VIN": "1G9Y817H34LSP7293" + } + ] +} diff --git a/103976/examples/VINtoCommsID.json b/103976/examples/VINtoCommsID.json index 471523a..b130c84 100644 --- a/103976/examples/VINtoCommsID.json +++ b/103976/examples/VINtoCommsID.json @@ -1,67 +1,87 @@ { - "VINtoCommsIDRecords" : [ - { - "CommsID" : { "IMEI" : "00440123456789" }, - "AssociationTime" : { - "PeriodInTime" : { - "StartTime" : "2022-01-16T15:57:00Z", - "EndTime" : "2022-01-16T15:57:00Z" - } - } - }, - { - "CommsID" : { "IMSI" : "999990123456789" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "ICCID" : "89999012345678901234" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "PEIIMEI" : "00440123456789" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "SUPIIMSI" : "999990123456789" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "SUPINAI" : "example@example.com" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "MSISDN" : "491713920000" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "GPSIMSISDN" : "491713920000" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "MACAddress" : "00:00:5e:00:53:00" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "CommsID" : { "EUI164" : "00:00:5e:ef:10:00:00:00" }, - "AssociationTime" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } + "VINtoCommsIDRecords": [ + { + "CommsID": { + "IMEI": "00440123456789" + }, + "AssociationTime": { + "PeriodInTime": { + "StartTime": "2022-01-16T15:57:00Z", + "EndTime": "2022-01-16T15:57:00Z" } - ] -} \ No newline at end of file + } + }, + { + "CommsID": { + "IMSI": "999990123456789" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "ICCID": "89999012345678901234" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "PEIIMEI": "00440123456789" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "SUPIIMSI": "999990123456789" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "SUPINAI": "example@example.com" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "MSISDN": "491713920000" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "GPSIMSISDN": "491713920000" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "MACAddress": "00:00:5e:00:53:00" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "CommsID": { + "EUI164": "00:00:5e:ef:10:00:00:00" + }, + "AssociationTime": { + "PointInTime": "2022-01-16T15:57:00Z" + } + } + ] +} diff --git a/103976/examples/VINtoLocation.json b/103976/examples/VINtoLocation.json index 708d23b..832fb2a 100644 --- a/103976/examples/VINtoLocation.json +++ b/103976/examples/VINtoLocation.json @@ -1,28 +1,28 @@ { - "VINtoLocationRecords" : [ - { - "Location" : { - "WGS84CoordinateDecimal" : { - "etsi280:latitude" : "N43.617003", - "etsi280:longitude" : "E007.053222" - } - }, - "TimeOfLocation" : { - "PointInTime" : "2022-01-16T15:57:00Z" - } - }, - { - "Location" : { - "WGS84CoordinateDecimal" : { - "etsi280:latitude" : "N43.617003", - "etsi280:longitude" : "E007.053222" - } - }, - "TimeOfLocation" : { - "PointInTime" : "2022-01-16T15:57:00Z" - }, - "SourceOfLocation" : "GNSS", - "LocationRecordReason" : "Location record reason" + "VINtoLocationRecords": [ + { + "Location": { + "WGS84CoordinateDecimal": { + "etsi280:latitude": "N43.617003", + "etsi280:longitude": "E007.053222" } - ] -} \ No newline at end of file + }, + "TimeOfLocation": { + "PointInTime": "2022-01-16T15:57:00Z" + } + }, + { + "Location": { + "WGS84CoordinateDecimal": { + "etsi280:latitude": "N43.617003", + "etsi280:longitude": "E007.053222" + } + }, + "TimeOfLocation": { + "PointInTime": "2022-01-16T15:57:00Z" + }, + "SourceOfLocation": "GNSS", + "LocationRecordReason": "Location record reason" + } + ] +} diff --git a/103976/ts_103976.schema.json b/103976/ts_103976.schema.json index 84fe84c..2f45d02 100644 --- a/103976/ts_103976.schema.json +++ b/103976/ts_103976.schema.json @@ -1,204 +1,363 @@ { - "$id": "ts_103976_core", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$defs": { - "AssociationPeriod" : { - "type" : "object", - "properties" : { - "StartTime" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "EndTime" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" } - }, - "required" : ["StartTime"] - }, - "AssociationTime" : { - "oneOf" : [ - { - "type" : "object", - "properties" : { - "PointInTime" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" } - }, - "required" : ["PointInTime"] - }, - { - "type" : "object", - "properties" : { - "PeriodInTime" : { "$ref": "#/$defs/AssociationPeriod" } - }, - "required" : ["PeriodInTime"] - } - ] - }, - "CommsID" : { - "oneOf" : [ - { - "type" : "object", - "properties" : { "IMEI" : { "$ref" : "ts_103280_2017_07#/$defs/IMEI"} }, - "required" : ["IMEI"] - }, - { - "type" : "object", - "properties" : { "IMSI" : { "$ref" : "ts_103280_2017_07#/$defs/IMSI"} }, - "required" : ["IMSI"] - }, - { - "type" : "object", - "properties" : { "ICCID" : { "$ref" : "ts_103280_2017_07#/$defs/ICCID"} }, - "required" : ["ICCID"] - }, - { - "type" : "object", - "properties" : { "PEIIMEI" : { "$ref" : "ts_103280_2017_07#/$defs/PEIIMEI"} }, - "required" : ["PEIIMEI"] - }, - { - "type" : "object", - "properties" : { "SUPIIMSI" : { "$ref" : "ts_103280_2017_07#/$defs/SUPIIMSI"} }, - "required" : ["SUPIIMSI"] - }, - { - "type" : "object", - "properties" : { "SUPINAI" : { "$ref" : "ts_103280_2017_07#/$defs/SUPINAI"} }, - "required" : ["SUPINAI"] - }, - { - "type" : "object", - "properties" : { "MSISDN" : { "$ref" : "ts_103280_2017_07#/$defs/InternationalE164"} }, - "required" : ["MSISDN"] - }, - { - "type" : "object", - "properties" : { "GPSIMSISDN" : { "$ref" : "ts_103280_2017_07#/$defs/GPSIMSISDN"} }, - "required" : ["GPSIMSISDN"] - }, - { - "type" : "object", - "properties" : { "GPSINAI" : { "$ref" : "ts_103280_2017_07#/$defs/GPSINAI"} }, - "required" : ["GPSINAI"] - }, - { - "type" : "object", - "properties" : { "MACAddress" : { "$ref" : "ts_103280_2017_07#/$defs/MACAddress"} }, - "required" : ["MACAddress"] - }, - { - "type" : "object", - "properties" : { "EUI164" : { "$ref" : "ts_103280_2017_07#/$defs/EUI64"} }, - "required" : ["EUI164"] - } - ] - }, - "Location" : { - "oneOf" : [ - { - "type" : "object", - "properties" : { - "WGS84CoordinateDecimal" : { "$ref" : "ts_103280_2017_07#/$defs/WGS84CoordinateDecimal"} - }, - "required" : ["WGS84CoordinateDecimal"] - } - ] - }, - "SourceOfLocation" : { - "enum" : ["GNSS"] - }, - "VINtoCommsIDRecord" : { - "type" : "object", - "properties" : { - "CommsID" : { "$ref" : "#/$defs/CommsID" }, - "AssociationTime" : { "$ref" : "#/$defs/AssociationTime"} - }, - "required" : ["CommsID"] - }, - "CommsIDToVINRecord" : { - "type" : "object", - "properties" : { - "VIN" : { "$ref" : "ts_103280_2017_07#/$defs/VIN" }, - "AssociationTime" : { "$ref" : "#/$defs/AssociationTime"} - }, - "required" : ["VIN"] - }, - "VINtoLocationRecord" : { - "type" : "object", - "properties" : { - "Location" : { "$ref" : "#/$defs/Location" }, - "TimeOfLocation" : { "$ref" : "#/$defs/AssociationTime" }, - "SourceOfLocation" : { "$ref" : "#/$defs/SourceOfLocation" }, - "LocationRecordReason" : { "$ref" : "ts_103280_2017_07#/$defs/LongString" } - }, - "required" : ["Location", "TimeOfLocation"] - }, - "VINtoUniquePartNumberRecord" : { - "type" : "object", - "properties" : { - "UniquePartNumber" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "PartType" : { "$ref" : "ts_103120_Common_2016_02#/$defs/DictionaryEntry"}, - "PartTypeFreeText" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" } - }, - "required" : ["UniquePartNumber"] - }, - "UniquePartNumbertoVINRecord" : { - "type" : "object", - "properties" : { - "VIN" : { "$ref" : "ts_103280_2017_07#/$defs/VIN" } - }, - "required" : ["VIN"] - }, - "VINtoSubscribedServicesRecord" : { - "type" : "object", - "properties" : { - "NameOfService" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "FirstPaymentDate" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "MostRecentPaymentDate" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "TerminationDate" : { "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" }, - "UserID" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "CustomerName" : { "$ref" : "ts_103280_2017_07#/$defs/ShortString" }, - "CustomerAddress": { "$ref" : "ts_103280_2017_07#/$defs/LongString" }, - "TypeOfPaymentMethod" : { "$ref" : "ts_103120_Common_2016_02#/$defs/DictionaryEntry"} - }, - "required" : [] - }, - "VINtoConnectedDevicesRecord" : { - "type" : "object", - "properties" : { - "ConnectionType" : { "$ref" : "ts_103120_Common_2016_02#/$defs/DictionaryEntry"}, - "DeviceIdentifiers" : { "$ref" : "#/$defs/ListOfCommsIDs"}, - "ConnectionTime" : { "$ref" : "#/$defs/AssociationTime" } - }, - "required" : ["ConnectionTime"] - }, - "ListOfCommsIDs" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/CommsID" } - }, - "ResultRecords" : { - "type" : "object", - "properties" : { - "VINtoCommsIDRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoCommsIDRecord"} - }, - "CommsIDtoVINRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/CommsIDToVINRecord"} - }, - "VINtoLocationRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoLocationRecord"} - }, - "VINtoUniquePartNumberRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoUniquePartNumberRecord"} - }, - "UniquePartNumbertoVINRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/UniquePartNumbertoVINRecord"} - }, - "VINtoSubscribedServicesRecords" : { - "type" : "array", - "items" : { "$ref" : "#/$defs/VINtoSubscribedServicesRecord"} - } + "$id": "ts_103976_core", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "AssociationPeriod": { + "type": "object", + "properties": { + "StartTime": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "EndTime": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + } + }, + "required": [ + "StartTime" + ] + }, + "AssociationTime": { + "oneOf": [ + { + "type": "object", + "properties": { + "PointInTime": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + } + }, + "required": [ + "PointInTime" + ] + }, + { + "type": "object", + "properties": { + "PeriodInTime": { + "$ref": "#/$defs/AssociationPeriod" + } + }, + "required": [ + "PeriodInTime" + ] + } + ] + }, + "CommsID": { + "oneOf": [ + { + "type": "object", + "properties": { + "IMEI": { + "$ref": "ts_103280_2017_07#/$defs/IMEI" + } + }, + "required": [ + "IMEI" + ] + }, + { + "type": "object", + "properties": { + "IMSI": { + "$ref": "ts_103280_2017_07#/$defs/IMSI" + } + }, + "required": [ + "IMSI" + ] + }, + { + "type": "object", + "properties": { + "ICCID": { + "$ref": "ts_103280_2017_07#/$defs/ICCID" + } + }, + "required": [ + "ICCID" + ] + }, + { + "type": "object", + "properties": { + "PEIIMEI": { + "$ref": "ts_103280_2017_07#/$defs/PEIIMEI" + } + }, + "required": [ + "PEIIMEI" + ] + }, + { + "type": "object", + "properties": { + "SUPIIMSI": { + "$ref": "ts_103280_2017_07#/$defs/SUPIIMSI" + } + }, + "required": [ + "SUPIIMSI" + ] + }, + { + "type": "object", + "properties": { + "SUPINAI": { + "$ref": "ts_103280_2017_07#/$defs/SUPINAI" } + }, + "required": [ + "SUPINAI" + ] + }, + { + "type": "object", + "properties": { + "MSISDN": { + "$ref": "ts_103280_2017_07#/$defs/InternationalE164" + } + }, + "required": [ + "MSISDN" + ] + }, + { + "type": "object", + "properties": { + "GPSIMSISDN": { + "$ref": "ts_103280_2017_07#/$defs/GPSIMSISDN" + } + }, + "required": [ + "GPSIMSISDN" + ] + }, + { + "type": "object", + "properties": { + "GPSINAI": { + "$ref": "ts_103280_2017_07#/$defs/GPSINAI" + } + }, + "required": [ + "GPSINAI" + ] + }, + { + "type": "object", + "properties": { + "MACAddress": { + "$ref": "ts_103280_2017_07#/$defs/MACAddress" + } + }, + "required": [ + "MACAddress" + ] + }, + { + "type": "object", + "properties": { + "EUI164": { + "$ref": "ts_103280_2017_07#/$defs/EUI64" + } + }, + "required": [ + "EUI164" + ] + } + ] + }, + "Location": { + "oneOf": [ + { + "type": "object", + "properties": { + "WGS84CoordinateDecimal": { + "$ref": "ts_103280_2017_07#/$defs/WGS84CoordinateDecimal" + } + }, + "required": [ + "WGS84CoordinateDecimal" + ] + } + ] + }, + "SourceOfLocation": { + "enum": [ + "GNSS" + ] + }, + "VINtoCommsIDRecord": { + "type": "object", + "properties": { + "CommsID": { + "$ref": "#/$defs/CommsID" + }, + "AssociationTime": { + "$ref": "#/$defs/AssociationTime" } + }, + "required": [ + "CommsID" + ] }, - "$ref": "#/$defs/ResultRecords" -} \ No newline at end of file + "CommsIDToVINRecord": { + "type": "object", + "properties": { + "VIN": { + "$ref": "ts_103280_2017_07#/$defs/VIN" + }, + "AssociationTime": { + "$ref": "#/$defs/AssociationTime" + } + }, + "required": [ + "VIN" + ] + }, + "VINtoLocationRecord": { + "type": "object", + "properties": { + "Location": { + "$ref": "#/$defs/Location" + }, + "TimeOfLocation": { + "$ref": "#/$defs/AssociationTime" + }, + "SourceOfLocation": { + "$ref": "#/$defs/SourceOfLocation" + }, + "LocationRecordReason": { + "$ref": "ts_103280_2017_07#/$defs/LongString" + } + }, + "required": [ + "Location", + "TimeOfLocation" + ] + }, + "VINtoUniquePartNumberRecord": { + "type": "object", + "properties": { + "UniquePartNumber": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "PartType": { + "$ref": "ts_103120_Common_2016_02#/$defs/DictionaryEntry" + }, + "PartTypeFreeText": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + } + }, + "required": [ + "UniquePartNumber" + ] + }, + "UniquePartNumbertoVINRecord": { + "type": "object", + "properties": { + "VIN": { + "$ref": "ts_103280_2017_07#/$defs/VIN" + } + }, + "required": [ + "VIN" + ] + }, + "VINtoSubscribedServicesRecord": { + "type": "object", + "properties": { + "NameOfService": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "FirstPaymentDate": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "MostRecentPaymentDate": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "TerminationDate": { + "$ref": "ts_103280_2017_07#/$defs/QualifiedDateTime" + }, + "UserID": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "CustomerName": { + "$ref": "ts_103280_2017_07#/$defs/ShortString" + }, + "CustomerAddress": { + "$ref": "ts_103280_2017_07#/$defs/LongString" + }, + "TypeOfPaymentMethod": { + "$ref": "ts_103120_Common_2016_02#/$defs/DictionaryEntry" + } + }, + "required": [] + }, + "VINtoConnectedDevicesRecord": { + "type": "object", + "properties": { + "ConnectionType": { + "$ref": "ts_103120_Common_2016_02#/$defs/DictionaryEntry" + }, + "DeviceIdentifiers": { + "$ref": "#/$defs/ListOfCommsIDs" + }, + "ConnectionTime": { + "$ref": "#/$defs/AssociationTime" + } + }, + "required": [ + "ConnectionTime" + ] + }, + "ListOfCommsIDs": { + "type": "array", + "items": { + "$ref": "#/$defs/CommsID" + } + }, + "ResultRecords": { + "type": "object", + "properties": { + "VINtoCommsIDRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoCommsIDRecord" + } + }, + "CommsIDtoVINRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/CommsIDToVINRecord" + } + }, + "VINtoLocationRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoLocationRecord" + } + }, + "VINtoUniquePartNumberRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoUniquePartNumberRecord" + } + }, + "UniquePartNumbertoVINRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/UniquePartNumbertoVINRecord" + } + }, + "VINtoSubscribedServicesRecords": { + "type": "array", + "items": { + "$ref": "#/$defs/VINtoSubscribedServicesRecord" + } + } + } + } + }, + "$ref": "#/$defs/ResultRecords" +} diff --git a/temp.asn b/temp.asn deleted file mode 100644 index e38540a..0000000 --- a/temp.asn +++ /dev/null @@ -1,762 +0,0 @@ -LI-PS-PDU -{itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) genHeader(1) version40(40)} - -DEFINITIONS IMPLICIT TAGS ::= - -BEGIN - -IMPORTS - -- Any of the IMPORTs may be commented out if they are not used (see clause A.3) - - -- from ETSI TS 103 280 [44] - LIID, - WGS84CoordinateDecimal - FROM Common-Parameters - {itu-t(0) identified-organization(4) etsi(0) common-parameters(3280) version261(261)} - - - -- previously from ETSI TS 101 671 [4]; now provided with this specification - IRI-Parameters, - IRIsContent, - Location, - Network-Element-Identifier - FROM HI2Operations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) hi2(1) version18(18)} - - -- previously from ETSI TS 101 671 [4]; now provided with this specification - HI1-Operation - FROM HI1NotificationOperations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) hi1(0) notificationOperations(1) version7(7)} - - -- from ETSI TS 102 232-2 [5] - EmailCC, - EmailIRI, - MessagingCC, - MessagingMMCC, - MessagingIRI - FROM EmailPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) email(2) version20(20)} - - -- from ETSI TS 102 232-3 [6] - IPCC, - IPIRI, - IPIRIOnly, - IPIRIPacketReport - FROM IPAccessPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPAccess(3) version20(20)} - - -- from ETSI TS 102 232-4 [32] - L2CC, - L2IRI, - L2IRIOnly - FROM L2AccessPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) l2Access(4) version10(10)} - - -- from ETSI TS 102 232-5 [37] - IPMMCC, - IPMMIRI - FROM IPMultimediaPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPMultimedia(5) version17(17)} - - -- from ETSI TS 102 232-6 [36] - PstnIsdnCC, - PstnIsdnIRI - FROM PstnIsdnPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) pstnIsdn(6) version6(6)} - - -- from 3GPP TS 33.108 [9] - IRI-Parameters, - UmtsIRIsContent, - CorrelationValues, - Location - FROM UmtsHI2Operations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2(1)} - -- The relevant module (including the 3GPP release and version number) needs - -- to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - IRI-Parameters, - UmtsCS-IRIsContent - FROM UmtsCS-HI2Operations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2CS(3)} - -- The relevant module (including the 3GPP release and version number) needs - -- to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - IRI-Parameters, - EpsIRIsContent, - EPSLocation - FROM EpsHI2Operations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2eps(8)} - -- The relevant module (including the 3GPP release and version number) needs - -- to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - IRI-Parameters, - ConfIRIsContent - FROM CONFHI2Operations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2conf(10)} - -- The relevant module (including the 3GPP release and version number) needs - -- to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - IRI-Parameters, - ProSeIRIsContent - FROM ProSeHI2Operations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2prose (15)} - -- The relevant module (including the 3GPP release and version number) needs - -- to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - IRI-Parameters, - GcseIRIsContent - FROM GCSEHI2Operations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi2gcse(13)} - -- The relevant module (including the 3GPP release and version number) needs - -- to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - CC-PDU - FROM Umts-HI3-PS - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3(2)} - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - CC-PDU - FROM Eps-HI3-PS - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3eps(9)} - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - Conf-CC-PDU - FROM CONF-HI3-IMS - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3conf(11)} - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - Voip-CC-PDU - FROM VoIP-HI3-IMS - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3voip(12)} - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - Gcse-CC-PDU - FROM GCSE-HI3 - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulintercept(2) threeGPP(4) hi3gcse(14)} - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - - -- from 3GPP TS 33.108 [9] - ThreeGPP-HI1-Operation - FROM ThreeGPP-HI1NotificationOperations - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi1(0) notificationOperations(1)} - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - -- TS 101 671 HI1 and 3GPP HI1 are related to the same functionality but are - -- corresponding to different implementations and exclusive usage each other. - -- The implementation depends of national regulations or LEA/CSP negotiations. - -- 3GPP HI1 may be used with other services/networks than 3GPP's one. - - -- from 3GPP TS 33.108 [9] - CSvoice-CC-PDU - FROM CSvoice-HI3-IP - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) hi3CSvoice(18)} - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - -- The implementation of the CS domain delivery in IP have to be based on 3GPP TS 33.108 [9]. - - -- from ETSI TS 101 909-20-1 [33] - TARGETACTIVITYMONITOR-1, - TTRAFFIC, - CTTRAFFIC - FROM TS101909201 - {itu-t(0) identified-organization(4) etsi(0) ts101909(1909) part20(20) subpart1(1) interceptVersion(0)} - - -- from ETSI TS 101 909-20-2 [34] - TARGETACTIVITYMONITOR, - TTRAFFIC, - CTTRAFFIC - FROM TS101909202 - {itu-t(0) identified-organization(4) etsi(0) ts101909(1909) part20(20) subpart2(2) interceptVersion(0)} - - -- from ETSI TS 103 462 [45] - ILHIPayload - FROM ILHIPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) informationHandover(4) ilhi(0) ilhiPdu(0) version2(2)} - -- This import is only used for the handover between LEMFs. - - -- from 3GPP TS 33.128 [46] - LINotificationPayload, - IRIPayload, - CCPayload, - UserLocation - FROM TS33128Payloads - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) threeGPP(4) ts33128(19)}; - -- The relevant module (including the 3GPP release and version number) - -- needs to be chosen when compiling the application. - --- end of IMPORTS - --- ============================= --- Object Identifier Definitions --- ============================= - -lawfulInterceptDomainId OBJECT IDENTIFIER ::= {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2)} - -li-psDomainId OBJECT IDENTIFIER ::= {lawfulInterceptDomainId li-ps(5) genHeader(1) version39(39)} - --- ==================== --- Top-level definition --- ==================== - -PS-PDU ::= SEQUENCE -{ - pSHeader [1] PSHeader, - payload [2] Payload -} - -PSHeader ::= SEQUENCE -{ - li-psDomainId [0] OBJECT IDENTIFIER, - lawfulInterceptionIdentifier [1] LawfulInterceptionIdentifier, - -- As of ASN.1 version 26 this parameter is included from ETSI TS 103 280 [44] - authorizationCountryCode [2] PrintableString (SIZE (2)) OPTIONAL, - -- see clause 5.2.3 - communicationIdentifier [3] CommunicationIdentifier, - sequenceNumber [4] INTEGER (0..4294967295), - timeStamp [5] GeneralizedTime OPTIONAL, - -- see clause 5.2.6 - ..., - interceptionPointID [6] PrintableString (SIZE (1..8)) OPTIONAL, - -- see clause 5.2.11 - microSecondTimeStamp [7] MicroSecondTimeStamp OPTIONAL, - timeStampQualifier [8] TimeStampQualifier OPTIONAL, - extendedInterceptionPointID [9] OCTET STRING (SIZE (1..65535)) OPTIONAL, - -- see clause 5.2.13 - networkFunctionIdentifier [10] OCTET STRING (SIZE (1..65535)) OPTIONAL - -- see clause 5.2.14 -} - -Payload ::= CHOICE -{ - iRIPayloadSequence [0] SEQUENCE OF NativeIRIPayload, - cCPayloadSequence [1] SEQUENCE OF NativeCCPayload, - -- Clause 6.2.3 explains how to include more than one payload in the same PDU - tRIPayload [2] TRIPayload, - ..., - hI1-Operation [3] HI1NotificationOperations.HI1-Operation, - encryptionContainer [4] EncryptionContainer, - threeGPP-HI1-Operation [5] ThreeGPP-HI1NotificationOperations.ThreeGPP-HI1-Operation, - -- This structure may be functionally redundant with hI1-Operation previously from ETSI TS 101 671 [4] - iLHIPayload [6] SEQUENCE OF ILHIPDU.ILHIPayload, - -- For typical use cases see ETSI TS 103 462 [45] - hI4Payload [7] SEQUENCE OF HI4Payload -} - -TimeStampQualifier ::= ENUMERATED -{ - unknown(0), - timeOfInterception(1), - timeOfMediation(2), - ..., - timeOfAggregation(3) -} - -HI4Payload ::= CHOICE -{ - threeGPP-LI-Notification [1] TS33128Payloads.LINotificationPayload, - ... -} - --- ==================================== --- Items contained within the PS-Header --- ==================================== - -CommunicationIdentifier ::= SEQUENCE -{ - networkIdentifier [0] NetworkIdentifier, - communicationIdentityNumber [1] INTEGER (0..4294967295) OPTIONAL, - -- in case of transport of HI1 messages not required - -- Mandatory for CC and IRI, with certain exceptions (see clause 5.2.4) - deliveryCountryCode [2] PrintableString (SIZE (2)) OPTIONAL, - -- see clause 5.2.4 - ..., - cINExtension [3] UmtsHI2Operations.CorrelationValues OPTIONAL - -- To be used when a single INTEGER is not sufficient to identify - -- a particular session (see clause 5.2.4) -} - -NetworkIdentifier ::= SEQUENCE -{ - operatorIdentifier [0] OCTET STRING (SIZE(1..16)), - networkElementIdentifier [1] OCTET STRING (SIZE(1..16)) OPTIONAL, - ..., - eTSI671NEID [2] HI2Operations.Network-Element-Identifier OPTIONAL - -- For network element identifier, use either networkElementIdentifier or eTSI671NEID -} - --- ========================== --- Definitions for CC Payload --- ========================== - -NativeCCPayload ::= SEQUENCE -{ - payloadDirection [0] PayloadDirection OPTIONAL, - timeStamp [1] GeneralizedTime OPTIONAL, - -- For aggregated payloads (see clause 6.2.3) - cCContents [2] CCContents, - ..., - microSecondTimeStamp [3] MicroSecondTimeStamp OPTIONAL, - -- For aggregated payloads (see clause 6.2.3) - timeStampQualifier [4] TimeStampQualifier OPTIONAL -} - -PayloadDirection ::= ENUMERATED -{ - fromTarget(0), - toTarget(1), - ..., - indeterminate(2), - -- Indication that the direction was indeterminate - combined(3), - -- Indication applicable to some services that the traffic is actually a combination - -- of To and From - notapplicable(4) - -- Indication that direction of interceptable service does not make sense -} - -CCContents ::= CHOICE - -- Any of these choices may be commented out if they are not being used, see clause A.3 -{ - -- tag [0] deprecated in version15, ETSI TS 102 232-1 V3.3.1 (2013-02) - emailCC [1] EmailPDU.EmailCC, - iPCC [2] IPAccessPDU.IPCC, - -- tag [3] not used - uMTSCC [4] OCTET STRING, - ..., - -- tag [5] deprecated in version15, ETSI TS 102 232-1 V3.3.1 (2013-02) - l2CC [6] L2AccessPDU.L2CC, - tTRAFFIC-1 [7] TS101909201.TTRAFFIC, - cTTRAFFIC-1 [8] TS101909201.CTTRAFFIC, - tTRAFFIC-2 [9] TS101909202.TTRAFFIC, - cTTRAFFIC-2 [10] TS101909202.CTTRAFFIC, - pstnIsdnCC [11] PstnIsdnPDU.PstnIsdnCC, - iPMMCC [12] IPMultimediaPDU.IPMMCC, - -- tag [13] deprecated in version32, ETSI TS 102 232-1 V3.24.1 (2021-07) - messagingCC [14] EmailPDU.MessagingCC, - ePSCC [15] OCTET STRING, - uMTSCC-CC-PDU [16] Umts-HI3-PS.CC-PDU, - ePSCC-CC-PDU [17] Eps-HI3-PS.CC-PDU, - messagingMMCC [18] EmailPDU.MessagingMMCC, - confCC-CC-PDU [19] CONF-HI3-IMS.Conf-CC-PDU, - voipCC-CC-PDU [20] VoIP-HI3-IMS.Voip-CC-PDU, - gcseCC-CC-PDU [21] GCSE-HI3.Gcse-CC-PDU, - cSvoice-CC-PDU [22] CSvoice-HI3-IP.CSvoice-CC-PDU, - threeGPP33128DefinedCC [23] OCTET STRING (CONTAINING TS33128Payloads.CCPayload) -} - -MicroSecondTimeStamp ::= SEQUENCE -{ - seconds [0] INTEGER (0..18446744073709551615), - -- number of seconds since 1970-1-1 00:00Z also known as unix time epoch - microSeconds [1] INTEGER (0..999999), - ... -} - --- =========================== --- Definitions for IRI Payload --- =========================== - -NativeIRIPayload ::= SEQUENCE -{ - iRIType [0] IRIType OPTIONAL, - -- See clause 5.2.10 - timeStamp [1] GeneralizedTime OPTIONAL, - -- For aggregated payloads (see clause 6.2.3) - iRIContents [2] IRIContents, - ..., - microSecondTimeStamp [3] MicroSecondTimeStamp OPTIONAL, - -- For aggregated payloads (see clause 6.2.3) - timeStampQualifier [4] TimeStampQualifier OPTIONAL, - sessionDirection [5] PayloadDirection OPTIONAL, - -- If the sessionDirection field is to be used for a given service then - -- the exact meaning and use of the field will be described in the - -- relevant service-specific details - payloadDirection [6] PayloadDirection OPTIONAL - -- If the payloadDirection field is to be used for a given service then - -- the exact meaning and use of the field will be described in the - -- relevant service-specific details -} - -IRIType ::= ENUMERATED -{ - iRI-Begin(1), - iRI-End(2), - iRI-Continue(3), - iRI-Report(4) -} - -IRIContents ::= CHOICE - -- Any of these choices may be commented out if they are not being used (see clause A.3) -{ - -- tag [0] deprecated in version15, ETSI TS 102 232-1 V3.3.1 (2013-02) - emailIRI [1] EmailPDU.EmailIRI, - iPIRI [2] IPAccessPDU.IPIRI, - iPIRIOnly [3] IPAccessPDU.IPIRIOnly, - uMTSIRI [4] UMTSIRI, - eTSI671IRI [5] ETSI671IRI, - ..., - l2IRI [6] L2AccessPDU.L2IRI, - l2IRIOnly [7] L2AccessPDU.L2IRIOnly, - tARGETACTIVITYMONITOR-1 [8] TS101909201.TARGETACTIVITYMONITOR-1, - tARGETACTIVITYMONITOR-2 [9] TS101909202.TARGETACTIVITYMONITOR, - pstnIsdnIRI [10] PstnIsdnPDU.PstnIsdnIRI, - iPMMIRI [11] IPMultimediaPDU.IPMMIRI, - -- tag [12] deprecated in version32, ETSI TS 102 232-1 V3.24.1 (2021-07) - -- tag [13] deprecated in version32, ETSI TS 102 232-1 V3.24.1 (2021-07) - messagingIRI [14] EmailPDU.MessagingIRI, - ePSIRI [15] EPSIRI, - confIRI [16] ConfIRI, - proseIRI [17] ProSeIRI, - gcseIRI [18] GcseIRI, - threeGPP33128DefinedIRI [19] OCTET STRING (CONTAINING TS33128Payloads.IRIPayload), - iPIRIPacketReport [20] IPAccessPDU.IPIRIPacketReport -} - -UMTSIRI ::= CHOICE - -- This structure may be commented out if not used -{ - iRI-Parameters [0] UmtsHI2Operations.IRI-Parameters, - umtsIRIsContent [1] UmtsHI2Operations.UmtsIRIsContent, - ..., - iRI-CS-Parameters [2] UmtsCS-HI2Operations.IRI-Parameters, - umtsCS-IRIsContent [3] UmtsCS-HI2Operations.UmtsCS-IRIsContent -} - -ETSI671IRI ::= CHOICE - -- This structure may be commented out if not used -{ - iRI-Parameters [0] HI2Operations.IRI-Parameters, - iRIsContent [1] HI2Operations.IRIsContent, - ... -} - -EPSIRI ::= CHOICE - -- This structure may be commented out if not used -{ - iRI-EPS-Parameters [0] EpsHI2Operations.IRI-Parameters, - epsIRIsContent [1] EpsHI2Operations.EpsIRIsContent, - ... -} - -ConfIRI ::= CHOICE - -- This structure may be commented out if not used -{ - iRI-Conf-Parameters [0] CONFHI2Operations.IRI-Parameters, - confIRIsContent [1] CONFHI2Operations.ConfIRIsContent, - ... -} - - -ProSeIRI ::= CHOICE - -- This structure may be commented out if not used -{ - iRI-ProSe-Parameters [0] ProSeHI2Operations.IRI-Parameters, - proseIRIsContent [1] ProSeHI2Operations.ProSeIRIsContent, - ... -} - - -GcseIRI ::= CHOICE - -- This structure may be commented out if not used -{ - iRI-Gcse-Parameters [0] GCSEHI2Operations.IRI-Parameters, - gcseIRIsContent [1] GCSEHI2Operations.GcseIRIsContent, - ... -} - - --- =========================== --- Definitions for TRI Payload --- =========================== - -TRIPayload ::= CHOICE -{ - integrityCheck [0] IntegrityCheck, - testPDU [1] NULL, - paddingPDU [2] OCTET STRING, - -- Undefined contents (will be discarded) - keep-alive [3] NULL, - keep-aliveResponse [4] NULL, - firstSegmentFlag [5] NULL, - lastSegmentFlag [6] NULL, - ..., - cINReset [7] NULL, - operatorLeaMessage [8] OperatorLeaMessage, - optionRequest [9] OptionRequest, - optionResponse [10] OptionResponse, - optionComplete [11] NULL, - pDUAcknowledgementRequest [12] NULL, - pDUAcknowledgementResponse [13] NULL -} - -IntegrityCheck ::= SEQUENCE -{ - includedSequenceNumbers [0] SEQUENCE OF INTEGER (0..4294967295), - -- gives the order the PDUs were processed - checkType [1] CheckType, - dataType [2] DataType OPTIONAL, - -- From version5(5) the dataType is mandatory for hashes and for signatures - -- (see clause 7.2.3) - checkValue [3] OCTET STRING, - -- Network byte order - -- If checkValue contains a signature, the octet string field space may be - -- used to insert the appropriate ASN.1 DER or BER encoded structure for the - -- DSS/DSA signature as described in IETF RFC 3279 [43], clause 2.2.2. - ..., - hashAlgorithm [4] HashAlgorithm OPTIONAL, - -- Clarifies the hash function if checkType is hash(1). - -- If used, it shall be present for each checkType hash(1). - signatureAlgorithm [5] SignatureAlgorithm OPTIONAL - -- Clarifies the signature algorithm if checkType is signature(2). - -- If used, it shall be present for each checkType signature(2). -} - -CheckType ::= ENUMERATED -{ - hash(1), - -- hash value - signature(2), - -- DSS/DSA signature - ... -} - -DataType ::= ENUMERATED -{ - iRI(1), - cC(2), - ..., - iLHI(3) -} - -HashAlgorithm ::= ENUMERATED -{ - sHA-1(1), - -- Included for legacy/migration purposes only, not to be used for new implementations - sHA-256(2), - sHA-384(3), - sHA-512(4), - ... -} - -SignatureAlgorithm ::= ENUMERATED -{ - dSA(1), - -- Included for legacy/migration purposes only, not to be used for new implementations - eCDSA(2), - edDSA(3), - ... -} - -Option ::= CHOICE -{ - pDUAcknowledgement [0] NULL, - ... -} - -OptionRequest ::= SEQUENCE -{ - requestedOptions [0] SEQUENCE OF Option, - ... -} - -OptionResponse ::= SEQUENCE -{ - acceptedOptions [0] SEQUENCE OF Option, - declinedOptions [1] SEQUENCE OF Option, - ... -} - --- ================================== --- Definitions for OperatorLeaMessage --- ================================== - -OperatorLeaMessage ::= SEQUENCE -{ - messagePriority [0] OperatorLeaMessagePriority, - message [1] OCTET STRING (SIZE(1..255)), - ... -} - -OperatorLeaMessagePriority ::= ENUMERATED -{ - error(1), - -- reporting of error conditions that have impact on the quality of the - -- intercepted data - informational(2), - -- reporting of conditions that will not have direct impact on the quality of - -- the intercepted data - ... -} - --- =================================== --- Definitions for EncryptionContainer --- =================================== - -EncryptionContainer ::= SEQUENCE -{ - encryptionType [0] EncryptionType, - encryptedPayload [1] OCTET STRING, - -- once decrypted, it can be interpreted as EncryptedPayload - ..., - encryptedPayloadType [2] EncryptedPayloadType OPTIONAL -} - -EncryptionType ::= ENUMERATED -{ - none(1), - -- No encryption is applied. - national-option(2), - -- Use this option when an encryption scheme is negotiated on a national level - aES-192-CBC(3), - -- The Advanced Encryption Standard using a 192 bit key in CBC mode - aES-256-CBC(4), - -- The Advanced Encryption Standard using a 256 bit key in CBC mode - blowfish-192-CBC(5), - -- Blowfish (www.schneier.com/blowfish.html) using a 192 bit key in CBC mode - blowfish-256-CBC(6), - -- Blowfish using a 256 bit key in CBC mode - threedes-cbc(7), - -- Triple-DES using a 192 bit key in CBC mode - ... -} - -EncryptedPayload ::= SEQUENCE -{ - byteCounter [0] INTEGER (0..18446744073709551615), - -- The sum of the sizes of all PDUs before this PDU. - -- It is initialized with the unixTime (number of seconds since 01-01-1970) - -- multiplied by 2^32 at first use. - -- Where N is sequencenumber of the n-th PDU in transfer, and size(PDU(N)) - -- as defined in annex G: - -- IF N > 0 THEN - -- PDU[N].byteCounter = PDU[N-1].byteCounter + size(PDU[N-1]) - -- ELSE - -- PDU[N].byteCounter = ( unixTime(now) << 32 ) - -- ENDIF - payload [1] Payload, - ... -} - -EncryptedPayloadType ::= ENUMERATED -{ - unknown(1), - part2(2), - -- encrypted payload is ETSI TS 102 232-2 [5] - part3(3), - -- encrypted payload is ETSI TS 102 232-3 [6] - part4(4), - -- encrypted payload is ETSI TS 102 232-4 [32] - part5(5), - -- encrypted payload is ETSI TS 102 232-5 [37] - part6(6), - -- encrypted payload is ETSI TS 102 232-6 [36] - part7(7), - -- encrypted payload is ETSI TS 102 232-7 [38] - ..., - part1(8) - -- encrypted payload is ETSI TS 102 232-1 (the present document) -} - --- =================================== --- Common Parameters --- =================================== - -NativeLocation ::= SEQUENCE - -- This is a common parameter, the use of this parameter is described in clause 4.5 -{ - umtsHI2Location [0] UmtsHI2Operations.Location OPTIONAL, - epsLocation [1] EpsHI2Operations.EPSLocation OPTIONAL, - ..., - wlanLocationAttributes [2] WlanLocationAttributes OPTIONAL, - eTSI671HI2Location [3] HI2Operations.Location OPTIONAL, - threeGPP33128UserLocation [4] TS33128Payloads.UserLocation OPTIONAL, - cPEProvidedLocationAttributes [5] CPEProvidedLocationAttributes OPTIONAL -} - -WlanLocationAttributes ::= SEQUENCE -{ - wlanAPMACAddress [0] OCTET STRING (SIZE(6)) OPTIONAL, - -- 48-bit (6 octet) MAC address of the WLAN access point derived from the BSSID - ... -} - -CPEProvidedLocationAttributes ::= SEQUENCE -{ - wGS84CoordinateDecimal [0] Common-Parameters.WGS84CoordinateDecimal OPTIONAL, - ... -} - -IPAddress ::= SEQUENCE - -- This parameter was previously imported from ETSI TS 101 671 [4] but has been copied - -- to ETSI TS 102 232-1 (the present document). It is not recommended to use this parameter in - -- future change requests. Suggested approach is to use a parameter from ETSI TS 103 280 [44] - -- instead. -{ - iP-type [1] ENUMERATED - { - iPV4(0), - iPV6(1), - ... - }, - iP-value [2] IP-value, - iP-assignment [3] ENUMERATED - { - static(1), - -- The static coding shall be used to report a static address. - dynamic(2), - -- The dynamic coding shall be used to report a dynamically allocated address. - notKnown(3), - -- The notKnown coding shall be used to report other than static or dynamically - -- allocated IP addresses. - ... - } OPTIONAL, - ..., - iPv6PrefixLength [4] INTEGER (1..128) OPTIONAL, - -- Indicates the length of the prefix delegated by the CSP to the subscriber - -- example: 60 if IP address is "2001:db8:0:85a3::ac1f:8001/60" - -- Mandatory in case where the iP-value contains an IPv6 binary value - iPv4SubnetMask [5] OCTET STRING (SIZE(4)) OPTIONAL, - -- For IPv4 addresses, this indicates the subnet mask to be applied to the iP-value field. - -- The subnet mask is intended to be presented as a binary value, e.g. "ff ff ff f8" to - -- represent the dotted-decimal subnet mask of "255.255.255.248" corresponding to - -- a /29 CIDR-format subnet mask - iP-NAT-translated [6] BOOLEAN OPTIONAL - -- If TRUE then NAT translated, if FALSE then not NAT translated. - -- If absent, IPAddress may or may not be NAT translated. -} - -IP-value ::= CHOICE -{ - iPBinaryAddress [1] OCTET STRING (SIZE(4..16)), - -- In case of IPv6, the Prefix Length is provided by the "iPv6PrefixLength" - -- In case of IPv4, the netmask is provided by the "iPv4SubnetMask" - iPTextAddress [2] IA5String (SIZE(7..45)), - -- In case of IPv6, the delivered iPTextAddress field could include a complete - -- single IPv6-Address or an IPv6-Prefix for a subnetwork on the target side. - -- In case of IPv4, the delivered iPTextAddress field could include a single - -- IPv4 address or an IPv4address/netmask, for example "192.168.1.1" or "192.168.1.1/24" - ... -} - -LawfulInterceptionIdentifier ::= Common-Parameters.LIID - -- LIID is a common parameter imported from ETSI TS 103 280 [44]. - -- It is redefined as LawfulInterceptionIdentifier in this - -- module to preserve the original type name during the - -- removal of imports from ETSI TS 101 671 [4]. - -END -- end of LI-PS-PDU diff --git a/testing/asn/asn_compile_targets.json b/testing/asn/asn_compile_targets.json index e186add..f12b657 100644 --- a/testing/asn/asn_compile_targets.json +++ b/testing/asn/asn_compile_targets.json @@ -1,33 +1,35 @@ [ - [ - "./102232-1/LI-PS-PDU.asn", - "./103280/TS_103_280.asn1", - "./testing/deps/33128/TS33128Payloads.asn", - "./testing/deps/33108/Three3gppHI1Notifications.asn", - "./testing/deps/33108/UMTSHI2Operations.asn", - "./testing/deps/33108/UMTSHI3PS.asn", - "./testing/deps/33108/EpsHI3PS.asn", - "./testing/deps/33108/ConfHI3IMS.asn", - "./testing/deps/33108/VoipHI3IMS.asn", - "./testing/deps/33108/GCSEHI3.asn", - "./testing/deps/33108/CSVoiceHI3IP.asn", - "./testing/deps/33108/UMTSCSHI2Operations.asn", - "./testing/deps/33108/EpsHI2Operations.asn", - "./testing/deps/33108/ConfHI2Operations.asn", - "./testing/deps/33108/ProSeHI2Operations.asn", - "./testing/deps/33108/GCSEHI2Operations.asn", - "./testing/deps/101671/HI1NotificationOperations,ver7.asn", - "./testing/deps/101671/HI2Operations,ver18.asn", - "./testing/deps/101909/TS101909201.asn", - "./testing/deps/101909/TS101909202.asn", - "./testing/deps/101909/PCESP.asn", - "./testing/deps/301040/06132v203_C01.asn", - "./103462/ILHIPDU.asn", - "./102232-2/EmailPDU.asn", - "./102232-3/IPAccessPDU.asn", - "./102232-4/L2AccessPDU.asn", - "./102232-5/IPMultimediaPDU.asn", - "./102232-6/PstnIsdnPDU.asn" - ], - ["./102657/RDMessage.asn"] -] \ No newline at end of file + [ + "./102232-1/LI-PS-PDU.asn", + "./103280/TS_103_280.asn1", + "./testing/deps/33128/TS33128Payloads.asn", + "./testing/deps/33108/Three3gppHI1Notifications.asn", + "./testing/deps/33108/UMTSHI2Operations.asn", + "./testing/deps/33108/UMTSHI3PS.asn", + "./testing/deps/33108/EpsHI3PS.asn", + "./testing/deps/33108/ConfHI3IMS.asn", + "./testing/deps/33108/VoipHI3IMS.asn", + "./testing/deps/33108/GCSEHI3.asn", + "./testing/deps/33108/CSVoiceHI3IP.asn", + "./testing/deps/33108/UMTSCSHI2Operations.asn", + "./testing/deps/33108/EpsHI2Operations.asn", + "./testing/deps/33108/ConfHI2Operations.asn", + "./testing/deps/33108/ProSeHI2Operations.asn", + "./testing/deps/33108/GCSEHI2Operations.asn", + "./testing/deps/101671/HI1NotificationOperations,ver7.asn", + "./testing/deps/101671/HI2Operations,ver18.asn", + "./testing/deps/101909/TS101909201.asn", + "./testing/deps/101909/TS101909202.asn", + "./testing/deps/101909/PCESP.asn", + "./testing/deps/301040/06132v203_C01.asn", + "./103462/ILHIPDU.asn", + "./102232-2/EmailPDU.asn", + "./102232-3/IPAccessPDU.asn", + "./102232-4/L2AccessPDU.asn", + "./102232-5/IPMultimediaPDU.asn", + "./102232-6/PstnIsdnPDU.asn" + ], + [ + "./102657/RDMessage.asn" + ] +] diff --git a/testing/.editorconfig b/testing/deps/.editorconfig similarity index 51% rename from testing/.editorconfig rename to testing/deps/.editorconfig index f1017ab..b993675 100644 --- a/testing/.editorconfig +++ b/testing/deps/.editorconfig @@ -1,4 +1,4 @@ -[*] -trim_trailing_whitespace = unset +[**.*] indent_style = unset indent_size = unset +trim_trailing_whitespace = unset \ No newline at end of file diff --git a/testing/json/ts_103120._json.json b/testing/json/ts_103120._json.json index 6873692..bdf4c39 100644 --- a/testing/json/ts_103120._json.json +++ b/testing/json/ts_103120._json.json @@ -1,11 +1,11 @@ { - "coreSchema": "103120/schema/json/ts_103120_Core.schema.json", - "supportingSchemas": [ - "103120/schema/json", - "103120/dictionaries/ts_103120_Dictionaries.schema.json", - "103280/TS_103_280.schema.json" - ], - "instanceDocs": [ - "103120/examples/json" - ] -} \ No newline at end of file + "coreSchema": "103120/schema/json/ts_103120_Core.schema.json", + "supportingSchemas": [ + "103120/schema/json", + "103120/dictionaries/ts_103120_Dictionaries.schema.json", + "103280/TS_103_280.schema.json" + ], + "instanceDocs": [ + "103120/examples/json" + ] +} diff --git a/testing/json/ts_103280._json.json b/testing/json/ts_103280._json.json index 5fe597a..a00d98e 100644 --- a/testing/json/ts_103280._json.json +++ b/testing/json/ts_103280._json.json @@ -1,5 +1,5 @@ { - "coreSchema": "103120/dictionaries/ts_103120_Dictionaries.schema.json", - "supportingSchemas": [], - "instanceDocs": [] -} \ No newline at end of file + "coreSchema": "103120/dictionaries/ts_103120_Dictionaries.schema.json", + "supportingSchemas": [], + "instanceDocs": [] +} diff --git a/testing/json/ts_103705_ex1_json.json b/testing/json/ts_103705_ex1_json.json index 11b8e11..c9ad621 100644 --- a/testing/json/ts_103705_ex1_json.json +++ b/testing/json/ts_103705_ex1_json.json @@ -1,11 +1,11 @@ { - "coreSchema": "103705/schema/response.schema.json", - "supportingSchemas": [ - "103705/schema/", - "103280/TS_103_280.schema.json", - "103705/examples/example1/csp_records.schema.json" - ], - "instanceDocs": [ - "103705/examples/example1/csp_results.json" - ] -} \ No newline at end of file + "coreSchema": "103705/schema/response.schema.json", + "supportingSchemas": [ + "103705/schema/", + "103280/TS_103_280.schema.json", + "103705/examples/example1/csp_records.schema.json" + ], + "instanceDocs": [ + "103705/examples/example1/csp_results.json" + ] +} diff --git a/testing/json/ts_103976_json.json b/testing/json/ts_103976_json.json index af62bbd..3d12b02 100644 --- a/testing/json/ts_103976_json.json +++ b/testing/json/ts_103976_json.json @@ -1,10 +1,10 @@ { - "coreSchema": "103976/ts_103976.schema.json", - "supportingSchemas": [ - "103280/TS_103_280.schema.json", - "103120/schema/json/ts_103120_Common.schema.json" - ], - "instanceDocs": [ - "103976/examples/" - ] -} \ No newline at end of file + "coreSchema": "103976/ts_103976.schema.json", + "supportingSchemas": [ + "103280/TS_103_280.schema.json", + "103120/schema/json/ts_103120_Common.schema.json" + ], + "instanceDocs": [ + "103976/examples/" + ] +} diff --git a/testing/xml/etsi_dictionaries.json b/testing/xml/etsi_dictionaries.json index 86b29ad..d693f9e 100644 --- a/testing/xml/etsi_dictionaries.json +++ b/testing/xml/etsi_dictionaries.json @@ -1,9 +1,9 @@ { - "coreSchema" : "103120/dictionaries/ts_103120_Dictionaries.xsd", - "supportingSchemas" : [], - "instanceDocs" : [ - "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml", - "104144/dictionaries/", - "103976/ts_103976_ETSIDictionaryDefinitions.xml" - ] -} \ No newline at end of file + "coreSchema": "103120/dictionaries/ts_103120_Dictionaries.xsd", + "supportingSchemas": [], + "instanceDocs": [ + "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml", + "104144/dictionaries/", + "103976/ts_103976_ETSIDictionaryDefinitions.xml" + ] +} diff --git a/testing/xml/ts_102657_xsd.json b/testing/xml/ts_102657_xsd.json index d534c1a..2201b45 100644 --- a/testing/xml/ts_102657_xsd.json +++ b/testing/xml/ts_102657_xsd.json @@ -1,5 +1,5 @@ { - "coreSchema": "102657/RDMessage.xsd", - "supportingSchemas": [], - "instanceDocs": [] -} \ No newline at end of file + "coreSchema": "102657/RDMessage.xsd", + "supportingSchemas": [], + "instanceDocs": [] +} diff --git a/testing/xml/ts_103120_xsd.json b/testing/xml/ts_103120_xsd.json index 4f28bf2..822b65d 100644 --- a/testing/xml/ts_103120_xsd.json +++ b/testing/xml/ts_103120_xsd.json @@ -1,20 +1,20 @@ { - "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", - "supportingSchemas": [ - "103120/schema/xsd/ts_103120_Authorisation.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Document.xsd", - "103120/schema/xsd/ts_103120_Notification.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103120/examples/xml/FooServiceSchema.xsd" - ], - "instanceDocs": [ - "103120/examples/xml" - ] -} \ No newline at end of file + "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", + "supportingSchemas": [ + "103120/schema/xsd/ts_103120_Authorisation.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Document.xsd", + "103120/schema/xsd/ts_103120_Notification.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/xml/FooServiceSchema.xsd" + ], + "instanceDocs": [ + "103120/examples/xml" + ] +} diff --git a/testing/xml/ts_103221-1_xsd.json b/testing/xml/ts_103221-1_xsd.json index 2511966..37409a3 100644 --- a/testing/xml/ts_103221-1_xsd.json +++ b/testing/xml/ts_103221-1_xsd.json @@ -1,22 +1,22 @@ { - "coreSchema" : "103221-1/TS_103_221_01.xsd", - "supportingSchemas" : [ - "103221-1/TS_103_221_01.xsd", - "103221-1/TS_103_221_01_HashedID.xsd", - "103221-1/TS_103_221_01_Configuration.xsd", - "103221-1/TS_103_221_01_DestinationSet.xsd", - "103221-1/TS_103_221_01_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "103221-1/examples/ExampleGenericObjects.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Core.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "testing/deps/old_400/TS_104_000.xsd" - ], - "instanceDocs": [ - "103221-1/examples" - ] -} \ No newline at end of file + "coreSchema": "103221-1/TS_103_221_01.xsd", + "supportingSchemas": [ + "103221-1/TS_103_221_01.xsd", + "103221-1/TS_103_221_01_HashedID.xsd", + "103221-1/TS_103_221_01_Configuration.xsd", + "103221-1/TS_103_221_01_DestinationSet.xsd", + "103221-1/TS_103_221_01_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "103221-1/examples/ExampleGenericObjects.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Core.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "testing/deps/old_400/TS_104_000.xsd" + ], + "instanceDocs": [ + "103221-1/examples" + ] +} diff --git a/testing/xml/ts_103280_xsd.json b/testing/xml/ts_103280_xsd.json index f9e5066..3468dba 100644 --- a/testing/xml/ts_103280_xsd.json +++ b/testing/xml/ts_103280_xsd.json @@ -1,7 +1,5 @@ { - "coreSchema": "103280/TS_103_280.xsd", - "supportingSchemas": [ - ], - "instanceDocs": [ - ] -} \ No newline at end of file + "coreSchema": "103280/TS_103_280.xsd", + "supportingSchemas": [], + "instanceDocs": [] +} diff --git a/testing/xml/ts_103707_xsd.json b/testing/xml/ts_103707_xsd.json index 0b0e601..7cb6813 100644 --- a/testing/xml/ts_103707_xsd.json +++ b/testing/xml/ts_103707_xsd.json @@ -1,19 +1,19 @@ { - "coreSchema": "103707/TS_103_707.xsd", - "supportingSchemas": [ - "103280/TS_103_280.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Core.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103707/examples/FooServiceSchema.xsd", - "104112/examples/xml/MessagingServiceSchema.xsd" - ], - "instanceDocs": [ - "103707/examples", - "104112/examples" - ] -} \ No newline at end of file + "coreSchema": "103707/TS_103_707.xsd", + "supportingSchemas": [ + "103280/TS_103_280.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Core.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103707/examples/FooServiceSchema.xsd", + "104112/examples/xml/MessagingServiceSchema.xsd" + ], + "instanceDocs": [ + "103707/examples", + "104112/examples" + ] +} diff --git a/testing/xml/ts_104000_xsd.json b/testing/xml/ts_104000_xsd.json index 4222fed..b23e93d 100644 --- a/testing/xml/ts_104000_xsd.json +++ b/testing/xml/ts_104000_xsd.json @@ -1,9 +1,9 @@ { - "coreSchema": "104000/schema/TS_104_000.xsd", - "supportingSchemas": [ - "103280/TS_103_280.xsd" - ], - "instanceDocs": [ - "104000/examples/" - ] -} \ No newline at end of file + "coreSchema": "104000/schema/TS_104_000.xsd", + "supportingSchemas": [ + "103280/TS_103_280.xsd" + ], + "instanceDocs": [ + "104000/examples/" + ] +} diff --git a/testing/xml/ts_104144_xsd.json b/testing/xml/ts_104144_xsd.json index 73e415f..2ae1b30 100644 --- a/testing/xml/ts_104144_xsd.json +++ b/testing/xml/ts_104144_xsd.json @@ -1,20 +1,20 @@ { - "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", - "supportingSchemas": [ - "103120/schema/xsd/ts_103120_Authorisation.xsd", - "103120/schema/xsd/ts_103120_Common.xsd", - "103120/schema/xsd/ts_103120_Config.xsd", - "103120/schema/xsd/ts_103120_Delivery.xsd", - "103120/dictionaries/ts_103120_Dictionaries.xsd", - "103120/schema/xsd/ts_103120_Document.xsd", - "103120/schema/xsd/ts_103120_Notification.xsd", - "103120/schema/xsd/ts_103120_Task.xsd", - "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", - "103280/TS_103_280.xsd", - "testing/deps/xmldsig/xmldsig-core-schema.xsd", - "103120/examples/xml/FooServiceSchema.xsd" - ], - "instanceDocs": [ - "104144/examples/" - ] -} \ No newline at end of file + "coreSchema": "103120/schema/xsd/ts_103120_Core.xsd", + "supportingSchemas": [ + "103120/schema/xsd/ts_103120_Authorisation.xsd", + "103120/schema/xsd/ts_103120_Common.xsd", + "103120/schema/xsd/ts_103120_Config.xsd", + "103120/schema/xsd/ts_103120_Delivery.xsd", + "103120/dictionaries/ts_103120_Dictionaries.xsd", + "103120/schema/xsd/ts_103120_Document.xsd", + "103120/schema/xsd/ts_103120_Notification.xsd", + "103120/schema/xsd/ts_103120_Task.xsd", + "103120/schema/xsd/ts_103120_TrafficPolicy.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/xml/FooServiceSchema.xsd" + ], + "instanceDocs": [ + "104144/examples/" + ] +} diff --git a/utils/ts103120_config.json b/utils/ts103120_config.json index 00043ff..6a89326 100644 --- a/utils/ts103120_config.json +++ b/utils/ts103120_config.json @@ -1,43 +1,42 @@ { - "schemas" : { - "./103120/dictionaries/ts_103120_Dictionaries.xsd" : { - "prefix" : "dictionaries", - "output" : "./103120/dictionaries/" - }, - "./103120/schema/xsd/ts_103120_Authorisation.xsd" : { - "prefix" : "auth" - }, - "./103120/schema/xsd/ts_103120_Common.xsd" : { - "prefix" : "common" - }, - "./103120/schema/xsd/ts_103120_Config.xsd" : { - "prefix" : "config" - }, - "./103120/schema/xsd/ts_103120_Core.xsd" : { - }, - "./103120/schema/xsd/ts_103120_Delivery.xsd" : { - "prefix" : "delivery" - }, - "./103120/schema/xsd/ts_103120_Document.xsd" : { - "prefix" : "doc" - }, - "./103120/schema/xsd/ts_103120_Notification.xsd" : { - "prefix" : "notification" - }, - "./103120/schema/xsd/ts_103120_Task.xsd" : { - "prefix" : "task" - }, - "./103120/schema/xsd/ts_103120_TrafficPolicy.xsd" : { - "prefix" : "tp" - }, - "./103280/TS_103_280.xsd" : { - "prefix" : "etsi280", - "skip" : true - }, - "./testing/deps/xmldsig/xmldsig-core-schema.xsd" : { - "prefix" : "xmldsig", - "skip" : true - } - }, - "output" : "./103120/schema/json/" + "schemas": { + "./103120/dictionaries/ts_103120_Dictionaries.xsd": { + "prefix": "dictionaries", + "output": "./103120/dictionaries/" + }, + "./103120/schema/xsd/ts_103120_Authorisation.xsd": { + "prefix": "auth" + }, + "./103120/schema/xsd/ts_103120_Common.xsd": { + "prefix": "common" + }, + "./103120/schema/xsd/ts_103120_Config.xsd": { + "prefix": "config" + }, + "./103120/schema/xsd/ts_103120_Core.xsd": {}, + "./103120/schema/xsd/ts_103120_Delivery.xsd": { + "prefix": "delivery" + }, + "./103120/schema/xsd/ts_103120_Document.xsd": { + "prefix": "doc" + }, + "./103120/schema/xsd/ts_103120_Notification.xsd": { + "prefix": "notification" + }, + "./103120/schema/xsd/ts_103120_Task.xsd": { + "prefix": "task" + }, + "./103120/schema/xsd/ts_103120_TrafficPolicy.xsd": { + "prefix": "tp" + }, + "./103280/TS_103_280.xsd": { + "prefix": "etsi280", + "skip": true + }, + "./testing/deps/xmldsig/xmldsig-core-schema.xsd": { + "prefix": "xmldsig", + "skip": true + } + }, + "output": "./103120/schema/json/" } diff --git a/utils/ts103280_config.json b/utils/ts103280_config.json index d36a86e..cdbb519 100644 --- a/utils/ts103280_config.json +++ b/utils/ts103280_config.json @@ -1,8 +1,8 @@ { - "schemas" : { - "./103280/TS_103_280.xsd" : { - "prefix" : "etsi280" - } - }, - "output" : "./103280/" + "schemas": { + "./103280/TS_103_280.xsd": { + "prefix": "etsi280" + } + }, + "output": "./103280/" } -- GitLab From c28dfd8a8d58762a1cc6ef9724dd36e6344037cb Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 8 Jul 2025 09:43:59 +0100 Subject: [PATCH 31/33] Removing ec step --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3624083..36dcc33 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -117,14 +117,6 @@ lint_json: - export PYCHARM_HOSTED=1 - python testing/json/lint_json.py -editorconfig: - image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest" - stage: lint - interruptible: true - allow_failure: true - script: - - ec - # ----------------------------------------------------------- # Build -- GitLab From 0a1fa744f6a3b9c1708fa56f964f60c619060263 Mon Sep 17 00:00:00 2001 From: Mark Canterbury Date: Tue, 8 Jul 2025 09:50:42 +0000 Subject: [PATCH 32/33] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Steije van Schelt <239-svs@users.noreply.forge.etsi.org> --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36dcc33..0305e9e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,7 +91,7 @@ compile_json: exit 1 fi echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" - echo "✅ XSD validation OK ($check_count files checked)" + echo "✅ JSON validation OK ($check_count files checked)" echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅" - forgeschema -s 103120/schema/json/ts_103120_Core.schema.json -u 103120/schema/json -u 103120/dictionaries/ts_103120_Dictionaries.schema.json -u 103280/TS_103_280.schema.json -i 103120/examples/json -- GitLab From fbe011522322b185fdf1598e3622387295af7f1a Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 8 Jul 2025 10:54:56 +0100 Subject: [PATCH 33/33] Ruff! --- testing/asn/asn_process.py | 160 +++++++-------- testing/json/lint_json.py | 47 +++-- testing/xml/lint_xml.py | 47 +++-- utils/json_to_xml.py | 21 +- utils/json_validator.py | 193 +++++++++++++----- utils/sign_json.py | 52 +++-- utils/translate/ChoiceMapping.py | 37 ++-- utils/translate/ComplexTypeMapping.py | 5 +- utils/translate/SequenceMapping.py | 69 ++++--- utils/translate/SimpleTypeMapping.py | 5 +- utils/translate/TypeMapping.py | 53 ++--- utils/translate/XSDNativeSimpleTypeMapping.py | 48 ++--- utils/translate/__init__.py | 29 +-- utils/translate_spec.py | 113 +++++----- utils/verify_json.py | 48 +++-- utils/xml_to_json.py | 167 ++++++++------- 16 files changed, 621 insertions(+), 473 deletions(-) diff --git a/testing/asn/asn_process.py b/testing/asn/asn_process.py index eb21db4..5085691 100644 --- a/testing/asn/asn_process.py +++ b/testing/asn/asn_process.py @@ -6,26 +6,32 @@ from shutil import which from pycrate_asn1c.asnproc import * -def reconstrainInteger (filename): - Path('temp.asn').write_text(Path(filename).read_text().replace("18446744073709551615", "65536")) - return 'temp.asn' + +def reconstrainInteger(filename): + Path("temp.asn").write_text( + Path(filename).read_text().replace("18446744073709551615", "65536") + ) + return "temp.asn" + filesWithBigInts = [ - '102232-1/LI-PS-PDU.asn', - '102232-3/IPAccessPDU.asn', - '102232-4/L2AccessPDU.asn' + "102232-1/LI-PS-PDU.asn", + "102232-3/IPAccessPDU.asn", + "102232-4/L2AccessPDU.asn", ] asn1c_path = "" change_path_to_unix = False + def fix_path(path): if change_path_to_unix: - return "./" + path.replace("\\","/") + return "./" + path.replace("\\", "/") else: return path -def syntaxCheckASN (fileList): + +def syntaxCheckASN(fileList): """ Performs ASN syntax checking on a list of filenames (or pathlib Paths) @@ -40,56 +46,48 @@ def syntaxCheckASN (fileList): try: if file.as_posix() in filesWithBigInts: newFile = reconstrainInteger(str(file)) - p = run([asn1c_path, '-E', fix_path(newFile)], capture_output=True) + p = run([asn1c_path, "-E", fix_path(newFile)], capture_output=True) Path(newFile).unlink() else: - p = run([asn1c_path, '-E', fix_path(str(file))], capture_output=True) - if (p.returncode != 0): + p = run([asn1c_path, "-E", fix_path(str(file))], capture_output=True) + if p.returncode != 0: errorMessage = p.stderr.decode().splitlines()[0] if errorMessage.startswith(' Value "18446744073709551615" at line'): - results[str(file)] = { 'ok' : True} + results[str(file)] = {"ok": True} continue results[str(file)] = { - 'ok' : False, - 'code' : p.returncode, - 'message' : p.stderr.decode().splitlines()[0] + "ok": False, + "code": p.returncode, + "message": p.stderr.decode().splitlines()[0], } else: - results[str(file)] = { - 'ok' : True - } + results[str(file)] = {"ok": True} except Exception as ex: raise ex - results[str(file)] = { - 'ok' : False, - 'code' : -1, - 'message' : f"{ex!r}" - } + results[str(file)] = {"ok": False, "code": -1, "message": f"{ex!r}"} return results duplicateObjects = { - '102232-1/LI-PS-PDU.asn' : [ - 'CCPayload', - 'IRIPayload', - 'Location' - ], - 'testing/mod1.asn' : [ - 'ClashField' - ] + "102232-1/LI-PS-PDU.asn": ["CCPayload", "IRIPayload", "Location"], + "testing/mod1.asn": ["ClashField"], } + + def fixDuplicateObjects(filename): stringContent = filename.read_text() for object in duplicateObjects[filename.as_posix()]: - stringContent = stringContent.replace(f'{object} ::=', f'Native{object} ::=', 1) - stringContent = stringContent.replace(f'SEQUENCE OF {object}', f'SEQUENCE OF Native{object}') - #stringContent = sub(f"]\\w{object}", f"] Native{object}", stringContent) + stringContent = stringContent.replace(f"{object} ::=", f"Native{object} ::=", 1) + stringContent = stringContent.replace( + f"SEQUENCE OF {object}", f"SEQUENCE OF Native{object}" + ) + # stringContent = sub(f"]\\w{object}", f"] Native{object}", stringContent) - Path('temp.asn').write_text(stringContent) - return 'temp.asn' + Path("temp.asn").write_text(stringContent) + return "temp.asn" -def compileAllTargets (compileTargets): +def compileAllTargets(compileTargets): """ Attempts to compile a set of compile targets using the pycrate ASN1 tools @@ -100,10 +98,10 @@ def compileAllTargets (compileTargets): to be the "primary" file. This doesn't have any relavance to the compilation, but will be used as the identifier when reporting any compile errors. The compilation is performed by the pycrate ASN compile functions; errors - are caught as exceptions and rendered into a list. - + are caught as exceptions and rendered into a list. + Unfortunately, the pycrate compiler doesn't report line numbers. - The asn1c compiler does, but doesn't properly handle identifiers with the + The asn1c compiler does, but doesn't properly handle identifiers with the same name in different modules; as this occurs multiple times in TS 33.108, we can't use it. """ @@ -120,107 +118,105 @@ def compileAllTargets (compileTargets): if pFile.as_posix() in duplicateObjects: tmpFile = Path(fixDuplicateObjects(pFile)) fileTexts.append(tmpFile.read_text()) - #tmpFile.unlink() + # tmpFile.unlink() else: fileTexts.append(pFile.read_text()) fileNames.append(filename) - logging.debug (f" Loading {filename}") - compile_text(fileTexts, filenames = fileNames) + logging.debug(f" Loading {filename}") + compile_text(fileTexts, filenames=fileNames) results[str(firstTarget)] = { - 'ok' : True, + "ok": True, } except Exception as ex: - results[str(firstTarget)] = { - 'ok' : False, - 'code' : -1, - 'message' : f"{ex!r}" - } + results[str(firstTarget)] = {"ok": False, "code": -1, "message": f"{ex!r}"} continue return results - -def processResults (results, stageName): +def processResults(results, stageName): """ Counts the number of errors and writes out the output per filename :param results: List of filenames (str or Pathlib Path) :param stageName: Name to decorate the output with :returns: The number of files which had errors - """ + """ print("") - errorCount = sum([1 for r in results.values() if not r['ok']]) + errorCount = sum([1 for r in results.values() if not r["ok"]]) logging.info(f"{errorCount} {stageName} errors encountered") - + print(f"{'-':-<60}") print(f"{stageName} results:") print(f"{'-':-<60}") for filename, result in results.items(): print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") - if not result['ok']: - if isinstance(result['message'], list): - for thing in result['message']: + if not result["ok"]: + if isinstance(result["message"], list): + for thing in result["message"]: print(f" {thing['message']}") else: print(f" {result['message']}") - + print(f"{'-':-<60}") print(f"{stageName} errors: {errorCount}") print(f"{'-':-<60}") - + return errorCount -if __name__ == '__main__': - logging.info ('Searching for ASN1C') +if __name__ == "__main__": + logging.info("Searching for ASN1C") asn1c_path = which("asn1c") if asn1c_path is None: - raise Exception ("No asn1c executable found. Please install asn1c") - logging.info (f"asn1c found at {asn1c_path}") + raise Exception("No asn1c executable found. Please install asn1c") + logging.info(f"asn1c found at {asn1c_path}") if asn1c_path.lower().endswith("bat"): - logging.info (f"asn1c is a batch file, so assume path separators need to be changed") + logging.info( + f"asn1c is a batch file, so assume path separators need to be changed" + ) change_path_to_unix = True - - logging.info('Searching for ASN.1 files') + logging.info("Searching for ASN.1 files") fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) - logging.info(f'{len(fileList)} ASN.1 files found') + logging.info(f"{len(fileList)} ASN.1 files found") for file in fileList: - logging.debug(f' {file}') - - ignoreList = Path('testing/asn/asn_ignore.txt').read_text().splitlines() + logging.debug(f" {file}") + + ignoreList = Path("testing/asn/asn_ignore.txt").read_text().splitlines() ignoredFiles = [] for ignore in ignoreList: - logging.debug(f'Ignoring pattern {ignore}') + logging.debug(f"Ignoring pattern {ignore}") for file in fileList: if ignore in str(file): ignoredFiles.append(file) logging.debug(f" Ignoring {str(file)} as contains {ignore}") ignoredFiles = list(set(ignoredFiles)) - logging.info(f'{len(ignoredFiles)} files ignored') + logging.info(f"{len(ignoredFiles)} files ignored") for file in ignoredFiles: - logging.debug(f' {file}') - + logging.debug(f" {file}") + fileList = [file for file in fileList if file not in ignoredFiles] - logging.info(f'{len(fileList)} files to process') + logging.info(f"{len(fileList)} files to process") for file in fileList: - logging.debug(f' {file}') + logging.debug(f" {file}") if len(fileList) == 0: - logging.warning ("No files specified") + logging.warning("No files specified") exit(0) - + logging.info("Parsing ASN1 files") parseResults = syntaxCheckASN(fileList) if processResults(parseResults, "Parsing") > 0: exit(-1) - logging.info ("Getting compile targets") - compileTargets = json.loads(Path('testing/asn/asn_compile_targets.json').read_text()) - logging.info (f"{len(compileTargets)} compile targets found") + logging.info("Getting compile targets") + compileTargets = json.loads( + Path("testing/asn/asn_compile_targets.json").read_text() + ) + logging.info(f"{len(compileTargets)} compile targets found") compileResults = compileAllTargets(compileTargets) if processResults(compileResults, "Compiling") > 0: exit(-1) - + exit(0) diff --git a/testing/json/lint_json.py b/testing/json/lint_json.py index f507123..783d53b 100644 --- a/testing/json/lint_json.py +++ b/testing/json/lint_json.py @@ -7,9 +7,8 @@ from colorama import Fore, Style colorama.init() -ignore_paths = [Path(x) for x in [ - 'testing/deps' -]] +ignore_paths = [Path(x) for x in ["testing/deps"]] + def print_colorized_diff_line(line: str): if line.startswith("-"): @@ -19,31 +18,33 @@ def print_colorized_diff_line(line: str): else: print(line) -def lint(file : Path): - completed = subprocess.run(["jq", ".", str(file)], capture_output=True, text=True, encoding="utf8") + +def lint(file: Path): + completed = subprocess.run( + ["jq", ".", str(file)], capture_output=True, text=True, encoding="utf8" + ) if completed.returncode != 0: - print (f" {str(f)}: FAIL") - print (f" jq error code {completed.returncode}") + print(f" {str(f)}: FAIL") + print(f" jq error code {completed.returncode}") lines = completed.stderr.splitlines() for line in lines: - print (f" {line}") + print(f" {line}") return len(lines) linted_xml = completed.stdout orig_xml = file.read_text(encoding="utf8") diff = list(unified_diff(orig_xml.splitlines(), linted_xml.splitlines())) if len(diff) == 0: - print (f"✅ {str(f)}") + print(f"✅ {str(f)}") return 0 else: - print (f"❌ {str(f)}: {len(diff)} linting errors") + print(f"❌ {str(f)}: {len(diff)} linting errors") # for d in diff: # print("".join(d)) for d in diff: print_colorized_diff_line(d) return len(diff) - if __name__ == "__main__": @@ -53,26 +54,28 @@ if __name__ == "__main__": files_with_errors = 0 errors = 0 - print ("───────────────────────────────────────────────────────────────────") - print (f"Linting JSON files...") + print("───────────────────────────────────────────────────────────────────") + print(f"Linting JSON files...") for f in files: if len(list(set(f.parents) & set(ignore_paths))) > 0: - print (f"(Ignoring {f})") + print(f"(Ignoring {f})") continue - new_errors = lint (f) + new_errors = lint(f) errors += new_errors files_with_errors += 1 if new_errors > 0 else 0 - print ("───────────────────────────────────────────────────────────────────") - print (f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}") - if (files_with_errors == 0): - print ("✅ OK") + print("───────────────────────────────────────────────────────────────────") + print( + f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}" + ) + if files_with_errors == 0: + print("✅ OK") else: print("❌ Fail") - print ("───────────────────────────────────────────────────────────────────") + print("───────────────────────────────────────────────────────────────────") - if (files_with_errors > 0): + if files_with_errors > 0: exit(-1) else: - exit(0) \ No newline at end of file + exit(0) diff --git a/testing/xml/lint_xml.py b/testing/xml/lint_xml.py index f295e51..85365e3 100644 --- a/testing/xml/lint_xml.py +++ b/testing/xml/lint_xml.py @@ -7,9 +7,8 @@ from colorama import Fore, Style colorama.init() -ignore_paths = [Path(x) for x in [ - 'testing/deps' -]] +ignore_paths = [Path(x) for x in ["testing/deps"]] + def print_colorized_diff_line(line: str): if line.startswith("-"): @@ -19,29 +18,31 @@ def print_colorized_diff_line(line: str): else: print(line) -def lint(file : Path): - completed = subprocess.run(["xmllint", str(file)], capture_output=True, text=True, encoding="utf8") + +def lint(file: Path): + completed = subprocess.run( + ["xmllint", str(file)], capture_output=True, text=True, encoding="utf8" + ) if completed.returncode != 0: - print (f" {str(f)}: FAIL") - print (f" xmlint error code {completed.returncode}") + print(f" {str(f)}: FAIL") + print(f" xmlint error code {completed.returncode}") lines = completed.stderr.splitlines() for line in lines: - print (f" {line}") + print(f" {line}") return len(lines) linted_xml = completed.stdout orig_xml = file.read_text(encoding="utf8") diff = list(unified_diff(orig_xml.splitlines(), linted_xml.splitlines())) if len(diff) == 0: - print (f"✅ {str(f)}") + print(f"✅ {str(f)}") return 0 else: - print (f"❌ {str(f)}: {len(diff)} linting errors") + print(f"❌ {str(f)}: {len(diff)} linting errors") for d in diff: print("".join(d)) return len(diff) - if __name__ == "__main__": @@ -51,26 +52,28 @@ if __name__ == "__main__": files_with_errors = 0 errors = 0 - print ("") - print (f"Linting XML / XSD files...") + print("") + print(f"Linting XML / XSD files...") for f in files: if len(list(set(f.parents) & set(ignore_paths))) > 0: - print (f"(Ignoring {f})") + print(f"(Ignoring {f})") continue - new_errors = lint (f) + new_errors = lint(f) errors += new_errors files_with_errors += 1 if new_errors > 0 else 0 - print ("───────────────────────────────────────────────────────────────────") - print (f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}") - if (files_with_errors == 0): - print ("✅ OK") + print("───────────────────────────────────────────────────────────────────") + print( + f"Files: {len(files)} ({files_with_errors} with errors) Total errors: {errors}" + ) + if files_with_errors == 0: + print("✅ OK") else: print("❌ Fail") - print ("───────────────────────────────────────────────────────────────────") + print("───────────────────────────────────────────────────────────────────") - if (files_with_errors > 0): + if files_with_errors > 0: exit(-1) else: - exit(0) \ No newline at end of file + exit(0) diff --git a/utils/json_to_xml.py b/utils/json_to_xml.py index 17764a7..3f03901 100644 --- a/utils/json_to_xml.py +++ b/utils/json_to_xml.py @@ -11,8 +11,19 @@ import argparse if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -31,5 +42,7 @@ if __name__ == "__main__": logging.debug(s) j = json.loads(s) - xml = xmltodict.unparse({'HI1Message' : j}, ) - print(xml) \ No newline at end of file + xml = xmltodict.unparse( + {"HI1Message": j}, + ) + print(xml) diff --git a/utils/json_validator.py b/utils/json_validator.py index e9ff0c0..074201f 100644 --- a/utils/json_validator.py +++ b/utils/json_validator.py @@ -7,10 +7,11 @@ import logging import argparse from itertools import chain + class JsonValidator: - def __init__(self, core_schema: str, other_schemas : dict): + def __init__(self, core_schema: str, other_schemas: dict): self._core_schema = json.load(Path(core_schema).open()) - self._schema_dict = { self._core_schema['$id'] : self._core_schema } + self._schema_dict = {self._core_schema["$id"]: self._core_schema} self._supporting_paths = [] for thing in other_schemas: path = Path(thing) @@ -22,102 +23,186 @@ class JsonValidator: self._supporting_paths.append(path) logging.info(f"Supporting schema paths: {self._supporting_paths}") self._supporting_schemas = [json.load(p.open()) for p in self._supporting_paths] - self._schema_dict = self._schema_dict | { s['$id'] : s for s in self._supporting_schemas } + self._schema_dict = self._schema_dict | { + s["$id"]: s for s in self._supporting_schemas + } logging.info(f"Loaded schema IDs: {[k for k in self._schema_dict.keys()]}") - self._resolver = RefResolver(None, - referrer=None, - store=self._schema_dict) + self._resolver = RefResolver(None, referrer=None, store=self._schema_dict) logging.info("Created RefResolver") - self._validator = Draft202012Validator(self._core_schema, resolver=self._resolver) + self._validator = Draft202012Validator( + self._core_schema, resolver=self._resolver + ) logging.info("Created validator") def validate(self, instance_doc: str): errors = list(self._validator.iter_errors(instance_doc)) return errors - -class TS103120Validator (JsonValidator): - def __init__ (self, path_to_repo): + + +class TS103120Validator(JsonValidator): + def __init__(self, path_to_repo): repo_path = Path(path_to_repo) schema_dirs = [str(repo_path / "103120/schema/json"), str("103280/")] core_schema = str(repo_path / "103120/schema/json/ts_103120_Core.schema.json") JsonValidator.__init__(self, core_schema, schema_dirs) - request_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/RequestPayload" } - self._request_fragment_validator = Draft202012Validator(request_fragment_schema, resolver=self._resolver) - response_fragment_schema = { "$ref" : "ts_103120_Core_2019_10#/$defs/ResponsePayload" } - self._response_fragment_validator = Draft202012Validator(response_fragment_schema, resolver=self._resolver) - - def expand_request_response_exception (self, ex): - if list(ex.schema_path) == ['properties', 'Payload', 'oneOf']: - logging.info ("Error detected validating payload oneOf - attempting explicit validation...") - if 'RequestPayload' in instance_doc['Payload'].keys(): - ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['RequestPayload'])])) + request_fragment_schema = { + "$ref": "ts_103120_Core_2019_10#/$defs/RequestPayload" + } + self._request_fragment_validator = Draft202012Validator( + request_fragment_schema, resolver=self._resolver + ) + response_fragment_schema = { + "$ref": "ts_103120_Core_2019_10#/$defs/ResponsePayload" + } + self._response_fragment_validator = Draft202012Validator( + response_fragment_schema, resolver=self._resolver + ) + + def expand_request_response_exception(self, ex): + if list(ex.schema_path) == ["properties", "Payload", "oneOf"]: + logging.info( + "Error detected validating payload oneOf - attempting explicit validation..." + ) + if "RequestPayload" in instance_doc["Payload"].keys(): + ret_list = list( + chain( + *[ + self.expand_action_exception(x) + for x in self._request_fragment_validator.iter_errors( + instance_doc["Payload"]["RequestPayload"] + ) + ] + ) + ) for r in ret_list: r.path = ex.path + r.path return ret_list - elif 'ResponsePayload' in instance_doc['Payload'].keys(): - ret_list = list(chain(*[self.expand_action_exception(x) for x in self._request_fragment_validator.iter_errors(instance_doc['Payload']['ResponsePayload'])])) + elif "ResponsePayload" in instance_doc["Payload"].keys(): + ret_list = list( + chain( + *[ + self.expand_action_exception(x) + for x in self._request_fragment_validator.iter_errors( + instance_doc["Payload"]["ResponsePayload"] + ) + ] + ) + ) for r in ret_list: r.path = ex.path + r.path return ret_list else: - logging.error("No RequestPayload or ResponsePayload found - is the Payload malformed?") + logging.error( + "No RequestPayload or ResponsePayload found - is the Payload malformed?" + ) return [ex] else: return [ex] - - def expand_action_exception (self, ex): + + def expand_action_exception(self, ex): logging.error("Error detected in ActionRequests/ActionResponses") error_path = list(ex.schema_path) - if error_path != ['properties', 'ActionRequests', 'properties', 'ActionRequest', 'items', 'allOf', 1, 'oneOf'] and error_path != ['properties', 'ActionResponses', 'properties', 'ActionResponse', 'items', 'allOf', 1, 'oneOf']: + if error_path != [ + "properties", + "ActionRequests", + "properties", + "ActionRequest", + "items", + "allOf", + 1, + "oneOf", + ] and error_path != [ + "properties", + "ActionResponses", + "properties", + "ActionResponse", + "items", + "allOf", + 1, + "oneOf", + ]: logging.error("Error not in inner Request/Response allOf/oneOf constraint") - return[ex] + return [ex] j = ex.instance - j.pop('ActionIdentifier') # Remove ActionIdentifier - one remaining key will be the verb + j.pop( + "ActionIdentifier" + ) # Remove ActionIdentifier - one remaining key will be the verb verb = list(j.keys())[0] message = "Request" if error_path[1] == "ActionRequests" else "Response" - v = Draft202012Validator({"$ref" : f"ts_103120_Core_2019_10#/$defs/{verb}{message}"}, resolver=self._resolver) - ret_list = list(chain(*[self.expand_object_exception(x) for x in v.iter_errors(j[verb])])) + v = Draft202012Validator( + {"$ref": f"ts_103120_Core_2019_10#/$defs/{verb}{message}"}, + resolver=self._resolver, + ) + ret_list = list( + chain(*[self.expand_object_exception(x) for x in v.iter_errors(j[verb])]) + ) for r in ret_list: r.path = ex.path + r.path return ret_list - - def expand_object_exception (self, ex): + + def expand_object_exception(self, ex): logging.error("Error detected in verb") # The final level of validation is for the actual HI1Object validation - if list(ex.schema_path) != ['properties', 'HI1Object', 'oneOf']: + if list(ex.schema_path) != ["properties", "HI1Object", "oneOf"]: logging.error("Error not inside HI1Object") return [ex] - object_type = ex.instance['@xsi:type'].split('}')[-1] + object_type = ex.instance["@xsi:type"].split("}")[-1] object_ref = { - 'AuthorisationObject': 'ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject', - 'LITaskObject': 'ts_103120_Task_2020_09#/$defs/LITaskObject', - 'LDTaskObject': 'ts_103120_Task_2020_09#/$defs/LDTaskObject', - 'LPTaskObject': 'ts_103120_Task_2020_09#/$defs/LPTaskObject', - 'DocumentObject': 'ts_103120_Document_2020_09#/$defs/DocumentObject', - 'NotificationObject': 'ts_103120_Notification_2016_02#/$defs/NotificationObject', - 'DeliveryObject': 'ts_103120_Delivery_2019_10#/$defs/DeliveryObject', - 'TrafficPolicyObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject', - 'TrafficRuleObject': 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject', + "AuthorisationObject": "ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject", + "LITaskObject": "ts_103120_Task_2020_09#/$defs/LITaskObject", + "LDTaskObject": "ts_103120_Task_2020_09#/$defs/LDTaskObject", + "LPTaskObject": "ts_103120_Task_2020_09#/$defs/LPTaskObject", + "DocumentObject": "ts_103120_Document_2020_09#/$defs/DocumentObject", + "NotificationObject": "ts_103120_Notification_2016_02#/$defs/NotificationObject", + "DeliveryObject": "ts_103120_Delivery_2019_10#/$defs/DeliveryObject", + "TrafficPolicyObject": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject", + "TrafficRuleObject": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject", }[object_type] - v = Draft202012Validator({"$ref" : object_ref}, resolver=self._resolver) + v = Draft202012Validator({"$ref": object_ref}, resolver=self._resolver) return list(v.iter_errors(ex.instance)) - + def validate(self, instance_doc: str): errors = JsonValidator.validate(self, instance_doc) - out_errors = list(chain(*[self.expand_request_response_exception(ex) for ex in errors])) + out_errors = list( + chain(*[self.expand_request_response_exception(ex) for ex in errors]) + ) return out_errors - if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") - parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") - parser.add_argument('--ts103120', action="store_true", help="Validate a TS 103 120 JSON document") - parser.add_argument('--schema', default=None, help="Primary schema to validate against") - parser.add_argument('-p', '--printerror', action="count", help="Controls how verbose validation error printing is (can be specified multiple times)") + parser.add_argument( + "-s", + "--schemadir", + action="append", + help="Directory containing supporting schema files to use for validation", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) + parser.add_argument( + "--ts103120", action="store_true", help="Validate a TS 103 120 JSON document" + ) + parser.add_argument( + "--schema", default=None, help="Primary schema to validate against" + ) + parser.add_argument( + "-p", + "--printerror", + action="count", + help="Controls how verbose validation error printing is (can be specified multiple times)", + ) args = parser.parse_args() match args.verbose: @@ -130,7 +215,7 @@ if __name__ == "__main__": logging.debug(f"Arguments: {args}") - if (args.ts103120): + if args.ts103120: v = TS103120Validator("./") else: v = JsonValidator(args.schema, args.schemadir) diff --git a/utils/sign_json.py b/utils/sign_json.py index 1ce0bba..96439d4 100644 --- a/utils/sign_json.py +++ b/utils/sign_json.py @@ -1,4 +1,3 @@ - import argparse import logging import sys @@ -8,18 +7,31 @@ from pathlib import Path import json -def insert_sig_block (j): - j['Signature'] = { - 'protected' : '', - 'signature' : '' - } +def insert_sig_block(j): + j["Signature"] = {"protected": "", "signature": ""} return j + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('--pretty', action="store_true", help='Pretty-print the JSON document before signing') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "--pretty", + action="store_true", + help="Pretty-print the JSON document before signing", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -34,24 +46,22 @@ if __name__ == "__main__": json_text = args.input.read() args.input.close() - + j = json.loads(json_text) j = insert_sig_block(j) - + indent = None if args.pretty: - indent = ' ' + indent = " " presigned_json_text = json.dumps(j, indent=indent) - Path('presigned.json').write_text(presigned_json_text) - presigned_json_bytes = presigned_json_text.encode('utf-8') - - signed = jws.sign(presigned_json_bytes, 'secret_key', algorithm="HS256") - components = signed.split('.') + Path("presigned.json").write_text(presigned_json_text) + presigned_json_bytes = presigned_json_text.encode("utf-8") + + signed = jws.sign(presigned_json_bytes, "secret_key", algorithm="HS256") + components = signed.split(".") - j['Signature']['protected'] = components[0] - j['Signature']['signature'] = components[2] + j["Signature"]["protected"] = components[0] + j["Signature"]["signature"] = components[2] signed_json_text = json.dumps(j, indent=indent) print(signed_json_text) - - diff --git a/utils/translate/ChoiceMapping.py b/utils/translate/ChoiceMapping.py index b477a33..16c3962 100644 --- a/utils/translate/ChoiceMapping.py +++ b/utils/translate/ChoiceMapping.py @@ -10,38 +10,43 @@ from .ComplexTypeMapping import ComplexTypeMapping log = logging.getLogger() + class ChoiceMapping(ComplexTypeMapping): @classmethod - def process_choice(cls, choice: XsdGroup, current_ns : str, ns_to_id_map): - if choice.model != 'choice': + def process_choice(cls, choice: XsdGroup, current_ns: str, ns_to_id_map): + if choice.model != "choice": raise Exception(f"Wrong group type: {c.model}") oneOf = [] for c in choice.iter_model(): if not (type(c) is XsdElement): - raise Exception (f"Non-element {c} encountered in choice {choice}") + raise Exception(f"Non-element {c} encountered in choice {choice}") element_name = c.local_name if c.target_namespace in ns_to_id_map: ns = ns_to_id_map[c.target_namespace] - if 'prefix' in ns: - element_name = ns['prefix'] + ":" + element_name + if "prefix" in ns: + element_name = ns["prefix"] + ":" + element_name t = TypeMapping.get_type_from_elem(c, current_ns) - oneOf.append({ - "type" : "object", - "properties" : { - element_name : t - }, - "required" : [element_name] - }) - return oneOf + oneOf.append( + { + "type": "object", + "properties": {element_name: t}, + "required": [element_name], + } + ) + return oneOf - def map(self, xst : BaseXsdType): + def map(self, xst: BaseXsdType): log.debug(f"Attempting mapping of {xst} to choice") j = super().map(xst) if j is None: log.debug("Not a complex type, giving up") return None content = xst.content - if (content.model != 'choice'): + if content.model != "choice": log.debug("Not a choice, giving up") return None - return { 'oneOf' : ChoiceMapping.process_choice(content, xst.namespaces[''], self.ns_to_id_map)} + return { + "oneOf": ChoiceMapping.process_choice( + content, xst.namespaces[""], self.ns_to_id_map + ) + } diff --git a/utils/translate/ComplexTypeMapping.py b/utils/translate/ComplexTypeMapping.py index e181909..19064e2 100644 --- a/utils/translate/ComplexTypeMapping.py +++ b/utils/translate/ComplexTypeMapping.py @@ -2,10 +2,9 @@ from xmlschema.validators.complex_types import * from .TypeMapping import TypeMapping + class ComplexTypeMapping(TypeMapping): def map(self, xst: BaseXsdType): if not (type(xst) is XsdComplexType): return None - return { - "type" : "object" - } + return {"type": "object"} diff --git a/utils/translate/SequenceMapping.py b/utils/translate/SequenceMapping.py index 68f7609..c70e509 100644 --- a/utils/translate/SequenceMapping.py +++ b/utils/translate/SequenceMapping.py @@ -20,14 +20,14 @@ class SequenceMapping(ComplexTypeMapping): log.debug("Not a complex type, giving up") return None content = xst.content - if (content.model != 'sequence'): + if content.model != "sequence": log.debug("Not a sequence, giving up") return None mapped_type = { - 'type' : 'object', - 'properties' : {}, - 'required' : [], - 'additionalProperties' : False + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": False, } # Not going to try and do all of this automatically for now @@ -35,11 +35,11 @@ class SequenceMapping(ComplexTypeMapping): # HACK exception for ApproverContactDetails because it isn't abstract if (xst.base_type) and (xst.local_name != "ApproverContactDetails"): # mapped_type['__DESCENDENT_OF__'] = TypeMapping.get_ref_for(xst.base_type, xst.namespaces['']) - mapped_type['properties']['@xsi:type'] = { - "type" : "string", - "enum" : [xst.name] + mapped_type["properties"]["@xsi:type"] = { + "type": "string", + "enum": [xst.name], } - mapped_type['required'].append('@xsi:type') + mapped_type["required"].append("@xsi:type") # if xst.abstract: # mapped_type['__ABSTRACT__'] = True # pass @@ -51,41 +51,50 @@ class SequenceMapping(ComplexTypeMapping): element_name = c.local_name if c.target_namespace in self.ns_to_id_map: ns = self.ns_to_id_map[c.target_namespace] - if 'prefix' in ns: - element_name = ns['prefix'] + ":" + element_name + if "prefix" in ns: + element_name = ns["prefix"] + ":" + element_name if c.effective_max_occurs != 1: - mapped_type['properties'][element_name] = { - "type" : "array", - "items" : TypeMapping.get_type_from_elem(c, xst.namespaces['']) + mapped_type["properties"][element_name] = { + "type": "array", + "items": TypeMapping.get_type_from_elem(c, xst.namespaces[""]), } if c.effective_max_occurs: - mapped_type['properties'][element_name]['maxItems'] = c.effective_max_occurs + mapped_type["properties"][element_name]["maxItems"] = ( + c.effective_max_occurs + ) if c.effective_min_occurs > 0: - mapped_type['properties'][element_name]['minItems'] = c.effective_min_occurs + mapped_type["properties"][element_name]["minItems"] = ( + c.effective_min_occurs + ) else: - mapped_type['properties'][element_name] = TypeMapping.get_type_from_elem(c, xst.namespaces['']) + mapped_type["properties"][element_name] = ( + TypeMapping.get_type_from_elem(c, xst.namespaces[""]) + ) if c.effective_min_occurs == 1: - mapped_type['required'].append(element_name) + mapped_type["required"].append(element_name) elif type(c) is XsdGroup: if inner_choice: - raise Exception (f"Second group '{element_name}' encountered in {xst}") + raise Exception( + f"Second group '{element_name}' encountered in {xst}" + ) if c.model != "choice": - raise Exception (f"Don't know what to do with inner group {c} in {xst} - not a choice") - inner_choice = ChoiceMapping.process_choice(c, xst.namespaces[''], self.ns_to_id_map) + raise Exception( + f"Don't know what to do with inner group {c} in {xst} - not a choice" + ) + inner_choice = ChoiceMapping.process_choice( + c, xst.namespaces[""], self.ns_to_id_map + ) elif type(c) is XsdAnyElement: mapped_type = {} else: raise Exception(f"Unknown element type {c}") - if (inner_choice): + if inner_choice: for inner_thing in inner_choice: - inner_thing.pop('additionalProperties', None) - mapped_type.pop('additionalProperties', None) - return { - 'allOf' : [ - mapped_type, - {'oneOf' : inner_choice} - ], - 'unevaluatedProperties' : False + inner_thing.pop("additionalProperties", None) + mapped_type.pop("additionalProperties", None) + return { + "allOf": [mapped_type, {"oneOf": inner_choice}], + "unevaluatedProperties": False, } else: return mapped_type diff --git a/utils/translate/SimpleTypeMapping.py b/utils/translate/SimpleTypeMapping.py index 2e60f9c..70edfb8 100644 --- a/utils/translate/SimpleTypeMapping.py +++ b/utils/translate/SimpleTypeMapping.py @@ -7,12 +7,11 @@ from .TypeMapping import TypeMapping log = logging.getLogger() + class SimpleTypeMapping(TypeMapping): def map(self, xst: BaseXsdType): log.debug(f"Attempting mapping of {xst} to simple type") if not (type(xst) is XsdAtomicRestriction): log.debug("Type is not an XsdAtomicRestriction, giving up") return None - return { - "$ref" : xst.base_type.name - } \ No newline at end of file + return {"$ref": xst.base_type.name} diff --git a/utils/translate/TypeMapping.py b/utils/translate/TypeMapping.py index 2b4b785..57a367f 100644 --- a/utils/translate/TypeMapping.py +++ b/utils/translate/TypeMapping.py @@ -8,63 +8,52 @@ from xmlschema.validators.facets import * log = logging.getLogger() + class TypeMapping(ABC): ns_to_id_map = {} XSD_NS = "http://www.w3.org/2001/XMLSchema" XSD_TYPE_MAP = { - "string" : { "type" : "string" }, - "normalizedString" : { "type" : "string"}, - "dateTime" : { "type" : "string"}, - "token" : { "type" : "string"}, - "anyURI" : { "type" : "string" }, - - "integer" : { "type" : "integer"}, - "nonNegativeInteger" : { "type" : "integer", "minimum" : 0}, - "positiveInteger" : { "type" : "integer", "minimum" : 1}, - - "boolean" : { "type" : "boolean" }, - - "hexBinary" : { "type" : "string", "pattern" : "^([a-fA-F0-9]{2})*$"}, - "base64Binary" : { "type" : "string", "pattern" : "^[A-Za-z0-9+\/]*={0,3}$"}, - - "anyType" : {} - } + "string": {"type": "string"}, + "normalizedString": {"type": "string"}, + "dateTime": {"type": "string"}, + "token": {"type": "string"}, + "anyURI": {"type": "string"}, + "integer": {"type": "integer"}, + "nonNegativeInteger": {"type": "integer", "minimum": 0}, + "positiveInteger": {"type": "integer", "minimum": 1}, + "boolean": {"type": "boolean"}, + "hexBinary": {"type": "string", "pattern": "^([a-fA-F0-9]{2})*$"}, + "base64Binary": {"type": "string", "pattern": "^[A-Za-z0-9+\/]*={0,3}$"}, + "anyType": {}, + } @abstractmethod - def map(self, xst : BaseXsdType): + def map(self, xst: BaseXsdType): return None @classmethod def extract_namespace(cls, qname: str): - match = re.search(r'^\{([^\{\}]+)\}(([^\{\}]+))$', qname) + match = re.search(r"^\{([^\{\}]+)\}(([^\{\}]+))$", qname) if match is None: return None return match.group(1) @classmethod - def get_ref_for(cls, xsd_type: XsdType, current_ns : str): + def get_ref_for(cls, xsd_type: XsdType, current_ns: str): ns = cls.extract_namespace(xsd_type.name) if ns == current_ns: - return { "$ref" : f"#/$defs/{xsd_type.local_name}" } + return {"$ref": f"#/$defs/{xsd_type.local_name}"} else: mapped_id = cls.ns_to_id_map[ns] - return { "$ref" : f"{mapped_id['id']}#/$defs/{xsd_type.local_name}"} + return {"$ref": f"{mapped_id['id']}#/$defs/{xsd_type.local_name}"} @classmethod - def get_type_from_elem(cls, elem: XsdElement, current_ns : str): + def get_type_from_elem(cls, elem: XsdElement, current_ns: str): ns = cls.extract_namespace(elem.type.name) - if (ns == TypeMapping.XSD_NS): + if ns == TypeMapping.XSD_NS: # this should be an XSD primitive type return dict(TypeMapping.XSD_TYPE_MAP[elem.type.local_name]) else: return cls.get_ref_for(elem.type, current_ns) - - - - - - - - diff --git a/utils/translate/XSDNativeSimpleTypeMapping.py b/utils/translate/XSDNativeSimpleTypeMapping.py index 772ac10..bd4d211 100644 --- a/utils/translate/XSDNativeSimpleTypeMapping.py +++ b/utils/translate/XSDNativeSimpleTypeMapping.py @@ -10,8 +10,8 @@ from .SimpleTypeMapping import SimpleTypeMapping log = logging.getLogger() -class XSDNativeSimpleTypeMapping(SimpleTypeMapping): +class XSDNativeSimpleTypeMapping(SimpleTypeMapping): def map(self, xst: BaseXsdType): log.debug(f"Attempting mapping of {xst} to XSD native type") j = super().map(xst) @@ -25,48 +25,50 @@ class XSDNativeSimpleTypeMapping(SimpleTypeMapping): if mapped_type is None: ns = TypeMapping.extract_namespace(xst.base_type.name) if ns == XSDNativeSimpleTypeMapping.XSD_NS: - print (xst) - print (xst.base_type) - raise Exception (f"No mapping for xs:{xst.base_type.local_name}") + print(xst) + print(xst.base_type) + raise Exception(f"No mapping for xs:{xst.base_type.local_name}") if len(xst.facets) == 0: - mapped_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces['']) + mapped_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces[""]) else: - parent_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces['']) + parent_type = TypeMapping.get_ref_for(xst.base_type, xst.namespaces[""]) mapped_type = TypeMapping.XSD_TYPE_MAP.get(xst.root_type.local_name) if mapped_type is None: - raise Exception (f"Could not find mapping for root type xs:{xst.root_type.local_name}") + raise Exception( + f"Could not find mapping for root type xs:{xst.root_type.local_name}" + ) mapped_type = dict(mapped_type) - + for k, v in xst.facets.items(): log.debug(f"Mapping facet {v}") if type(v) is XsdMaxLengthFacet: - mapped_type['maxLength'] = v.value + mapped_type["maxLength"] = v.value continue if type(v) is XsdMinLengthFacet: - mapped_type['minLength'] = v.value + mapped_type["minLength"] = v.value continue if type(v) is XsdPatternFacets: if len(v.regexps) > 1: - raise Exception (f"Multiple patterns given in facet {v} of {xst}") + raise Exception(f"Multiple patterns given in facet {v} of {xst}") p = v.regexps[0] - if (not p.startswith('^')) and (not p.endswith('$')): + if (not p.startswith("^")) and (not p.endswith("$")): p = f"^{p}$" - mapped_type['pattern'] = p + mapped_type["pattern"] = p continue - if type (v) is XsdMinInclusiveFacet: - mapped_type['minimum'] = v.value + if type(v) is XsdMinInclusiveFacet: + mapped_type["minimum"] = v.value continue - if type (v) is XsdMaxInclusiveFacet: - mapped_type['maximum'] = v.value + if type(v) is XsdMaxInclusiveFacet: + mapped_type["maximum"] = v.value continue - if type (v) is XsdMinExclusiveFacet: - mapped_type['exclusiveMinimum'] = v.value + if type(v) is XsdMinExclusiveFacet: + mapped_type["exclusiveMinimum"] = v.value continue - if type (v) is XsdMaxExclusiveFacet: - mapped_type['exclusiveMaximum'] = v.value + if type(v) is XsdMaxExclusiveFacet: + mapped_type["exclusiveMaximum"] = v.value continue - raise Exception (f"Unhandled facet {v}") + raise Exception(f"Unhandled facet {v}") if parent_type: - return { 'allOf' : [parent_type, mapped_type] } + return {"allOf": [parent_type, mapped_type]} return mapped_type diff --git a/utils/translate/__init__.py b/utils/translate/__init__.py index 86a3346..bf2d08d 100644 --- a/utils/translate/__init__.py +++ b/utils/translate/__init__.py @@ -17,30 +17,33 @@ mappings = [ SequenceMapping(), ] -def translate_schema (schema_path: str, ns_to_id_map: dict, schema_locations = []): + +def translate_schema(schema_path: str, ns_to_id_map: dict, schema_locations=[]): js = { - "$id" : "?", - "$schema" : "https://json-schema.org/draft/2020-12/schema", - "$defs" : {} + "$id": "?", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": {}, } logging.info(f"Translating schema {schema_path}") - xs = XMLSchema(schema_path, validation='lax', locations=schema_locations) - logging.info(f"Schema namespace: {xs.target_namespace}" ) + xs = XMLSchema(schema_path, validation="lax", locations=schema_locations) + logging.info(f"Schema namespace: {xs.target_namespace}") schema_id = ns_to_id_map[xs.target_namespace]["id"] - js['$id'] = schema_id + js["$id"] = schema_id TypeMapping.ns_to_id_map = ns_to_id_map elementList = [] for elementName, element in xs.elements.items(): logging.info(f"Processing element {elementName} : {element}") - elementList.append(TypeMapping.get_ref_for(element.type, element.namespaces[''])) + elementList.append( + TypeMapping.get_ref_for(element.type, element.namespaces[""]) + ) if len(elementList) == 1: - js['$ref'] = elementList[0]['$ref'] + js["$ref"] = elementList[0]["$ref"] elif len(elementList) > 1: - js['oneOf'] = elementList + js["oneOf"] = elementList descendent_types = {} for type_name, xsd_type in xs.types.items(): @@ -57,9 +60,7 @@ def translate_schema (schema_path: str, ns_to_id_map: dict, schema_locations = [ if j is None: raise Exception(f"Unmapped type {type_name} ({xsd_type})") js["$defs"][xsd_type.local_name] = j - logging.debug (f"Mapped {type_name} to {j}") + logging.debug(f"Mapped {type_name} to {j}") - print (descendent_types) + print(descendent_types) return js - - diff --git a/utils/translate_spec.py b/utils/translate_spec.py index f78b8ff..6ce2f84 100644 --- a/utils/translate_spec.py +++ b/utils/translate_spec.py @@ -8,68 +8,68 @@ from xmlschema import * from translate import * -logging.basicConfig(level = logging.INFO) +logging.basicConfig(level=logging.INFO) json_signature_struct = { - "properties" : { - "protected" : { "type" : "string" }, - "signature" : { "type" : "string" } - }, - "required" : ["protected", "signature" ] + "properties": {"protected": {"type": "string"}, "signature": {"type": "string"}}, + "required": ["protected", "signature"], } -def build_schema_locations (paths): + +def build_schema_locations(paths): schema_locations = [] for schemaFile in paths: try: - xs = XMLSchema(schemaFile, validation='skip') - schema_locations.append((xs.target_namespace, str(Path(schemaFile).resolve()))) - logging.debug (" [ {0} -> {1} ]".format(xs.target_namespace, schemaFile)) + xs = XMLSchema(schemaFile, validation="skip") + schema_locations.append( + (xs.target_namespace, str(Path(schemaFile).resolve())) + ) + logging.debug(" [ {0} -> {1} ]".format(xs.target_namespace, schemaFile)) except XMLSchemaParseError as ex: - logging.debug (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) + logging.debug(" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) return schema_locations + def get_json(filename): with open(filename) as f: j = json.load(f) return j -def convert_ns_to_id (ns): - if ns.startswith('http://uri.etsi.org'): + +def convert_ns_to_id(ns): + if ns.startswith("http://uri.etsi.org"): c = ns.split("/") return f"ts_1{c[3]}{'_' + c[7] if len(c) > 7 else ''}_{c[5]}_{c[6]}" else: - return ns.replace("http://","").replace("/","_") + return ns.replace("http://", "").replace("/", "_") + -def convert_xsd_to_filename (xsd): +def convert_xsd_to_filename(xsd): f = Path(xsd) - return f.name.replace('.xsd', '.schema.json') + return f.name.replace(".xsd", ".schema.json") + if __name__ == "__main__": if len(sys.argv) < 2: - logging.error ("Usage: translate_spec.py path_to_config_file") + logging.error("Usage: translate_spec.py path_to_config_file") exit(-1) config = get_json(sys.argv[1]) - logging.info("Bulding ns map...") ns_map = {} - for location, settings in config['schemas'].items(): - xs = XMLSchema(location, validation='skip') + for location, settings in config["schemas"].items(): + xs = XMLSchema(location, validation="skip") ns = xs.target_namespace id = convert_ns_to_id(ns) - ns_map[ns] = { - "id" : id, - "location" : str(Path(location).resolve()) - } | settings + ns_map[ns] = {"id": id, "location": str(Path(location).resolve())} | settings logging.debug(ns_map) - + logging.info("Building schema locations") - schema_locations = [(k, v["location"]) for k,v in ns_map.items()] + schema_locations = [(k, v["location"]) for k, v in ns_map.items()] logging.debug(schema_locations) - output_path = Path(config['output']) + output_path = Path(config["output"]) if not output_path.exists(): logging.info("Creating output directory") os.mkdir(str(output_path)) @@ -78,45 +78,56 @@ if __name__ == "__main__": json_schemas = {} for schema_tuple in schema_locations: logging.info(f" Translating {schema_tuple}") - if 'skip' in ns_map[schema_tuple[0]]: + if "skip" in ns_map[schema_tuple[0]]: logging.info(f" Skipping {schema_tuple[0]}...") continue js = translate_schema(schema_tuple[1], ns_map, schema_locations) # TODO - Special case, get rid of XML Dsig signature and insert JSON signature - if schema_tuple[0] == 'http://uri.etsi.org/03120/common/2019/10/Core': - logging.info ("Modifying signature elements") - js['$defs']['HI1Message']['patternProperties'] = { "^@" : { "type" : "string"}} - js['$defs']['HI1Message']['properties'].pop('xmldsig:Signature') - js['$defs']['HI1Message']['properties']['Signature'] = json_signature_struct - - - if 'output' in ns_map[schema_tuple[0]]: - js_path = Path(ns_map[schema_tuple[0]]['output']) / convert_xsd_to_filename(schema_tuple[1]) + if schema_tuple[0] == "http://uri.etsi.org/03120/common/2019/10/Core": + logging.info("Modifying signature elements") + js["$defs"]["HI1Message"]["patternProperties"] = {"^@": {"type": "string"}} + js["$defs"]["HI1Message"]["properties"].pop("xmldsig:Signature") + js["$defs"]["HI1Message"]["properties"]["Signature"] = json_signature_struct + + if "output" in ns_map[schema_tuple[0]]: + js_path = Path(ns_map[schema_tuple[0]]["output"]) / convert_xsd_to_filename( + schema_tuple[1] + ) else: js_path = output_path / convert_xsd_to_filename(schema_tuple[1]) # TODO - Special case - abstract HI1Object if "Core" in schema_tuple[1]: - js["$defs"]['ConcreteHI1Object'] = { - 'oneOf' : [ - {'$ref' : 'ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject'}, - {'$ref' : 'ts_103120_Task_2020_09#/$defs/LITaskObject'}, - {'$ref' : 'ts_103120_Task_2020_09#/$defs/LPTaskObject'}, - {'$ref' : 'ts_103120_Task_2020_09#/$defs/LDTaskObject'}, - {'$ref' : 'ts_103120_Document_2020_09#/$defs/DocumentObject'}, - {'$ref' : 'ts_103120_Notification_2016_02#/$defs/NotificationObject'}, - {'$ref' : 'ts_103120_Delivery_2019_10#/$defs/DeliveryObject'}, - {'$ref' : 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject'}, - {'$ref' : 'ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject'}, + js["$defs"]["ConcreteHI1Object"] = { + "oneOf": [ + { + "$ref": "ts_103120_Authorisation_2020_09#/$defs/AuthorisationObject" + }, + {"$ref": "ts_103120_Task_2020_09#/$defs/LITaskObject"}, + {"$ref": "ts_103120_Task_2020_09#/$defs/LPTaskObject"}, + {"$ref": "ts_103120_Task_2020_09#/$defs/LDTaskObject"}, + {"$ref": "ts_103120_Document_2020_09#/$defs/DocumentObject"}, + { + "$ref": "ts_103120_Notification_2016_02#/$defs/NotificationObject" + }, + {"$ref": "ts_103120_Delivery_2019_10#/$defs/DeliveryObject"}, + { + "$ref": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficPolicyObject" + }, + { + "$ref": "ts_103120_TrafficPolicy_2022_07#/$defs/TrafficRuleObject" + }, ] } json_string = json.dumps(js, indent=2) + "\n" if "Core" in schema_tuple[1]: - json_string = json_string.replace('"$ref": "#/$defs/HI1Object"', '"$ref": "#/$defs/ConcreteHI1Object"') + json_string = json_string.replace( + '"$ref": "#/$defs/HI1Object"', '"$ref": "#/$defs/ConcreteHI1Object"' + ) - with open(str(js_path), 'w', newline='\n') as f: + with open(str(js_path), "w", newline="\n") as f: f.write(json_string) - json_schemas[js['$id']] = json.loads(json_string) + json_schemas[js["$id"]] = json.loads(json_string) diff --git a/utils/verify_json.py b/utils/verify_json.py index 329c069..5f74080 100644 --- a/utils/verify_json.py +++ b/utils/verify_json.py @@ -1,4 +1,3 @@ - import argparse import sys import logging @@ -11,8 +10,19 @@ import json if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -27,28 +37,30 @@ if __name__ == "__main__": signed_json_text = args.input.read() args.input.close() - + j = json.loads(signed_json_text) - - protected_header = j['Signature']['protected'] - signature = j['Signature']['signature'] + + protected_header = j["Signature"]["protected"] + signature = j["Signature"]["signature"] # TODO some safety checks needed here # Remove the newline that appears from the console - if signed_json_text.endswith('\n'): signed_json_text = signed_json_text[:-1] - signed_json_text = signed_json_text.replace(protected_header, "").replace(signature, "") - - payload_bytes = signed_json_text.encode('utf-8') - payload_token = base64.b64encode(payload_bytes).decode('ascii') + if signed_json_text.endswith("\n"): + signed_json_text = signed_json_text[:-1] + signed_json_text = signed_json_text.replace(protected_header, "").replace( + signature, "" + ) + + payload_bytes = signed_json_text.encode("utf-8") + payload_token = base64.b64encode(payload_bytes).decode("ascii") # Un-pad the token, as per RFC7515 annex C - payload_token = payload_token.split('=')[0] - payload_token = payload_token.replace('+','-') - payload_token = payload_token.replace('/','_') + payload_token = payload_token.split("=")[0] + payload_token = payload_token.replace("+", "-") + payload_token = payload_token.replace("/", "_") token = protected_header + "." + payload_token + "." + signature - result = jws.verify(token, key="secret_key", algorithms=['HS256']) - + result = jws.verify(token, key="secret_key", algorithms=["HS256"]) + print("Signature verified") - diff --git a/utils/xml_to_json.py b/utils/xml_to_json.py index 0fc07d7..74c20f6 100644 --- a/utils/xml_to_json.py +++ b/utils/xml_to_json.py @@ -9,99 +9,99 @@ import xmltodict import argparse -def extract_prefixes (d): - return { k.split(':')[1]: d[k] for k in d.keys() if k.startswith("@xmlns:") } +def extract_prefixes(d): + return {k.split(":")[1]: d[k] for k in d.keys() if k.startswith("@xmlns:")} -def removePrefixes (o, prefixes): - if not isinstance(o, dict): return + +def removePrefixes(o, prefixes): + if not isinstance(o, dict): + return replacements = [] - for k,v in o.items(): + for k, v in o.items(): if isinstance(v, dict): removePrefixes(v, prefixes) if isinstance(v, list): for i in v: removePrefixes(i, prefixes) if ":" in k: - prefix = k.split(':')[0] + prefix = k.split(":")[0] if (prefix) in prefixes: - new_key = k.split(':')[1] - replacements.append( (k, new_key) ) + new_key = k.split(":")[1] + replacements.append((k, new_key)) for r in replacements: o[r[1]] = o.pop(r[0]) + object_namespaces = { - 'AuthorisationObject' : 'http://uri.etsi.org/03120/common/2020/09/Authorisation', - 'DeliveryObject' : 'http://uri.etsi.org/03120/common/2019/10/Delivery', - 'DocumentObject' : 'http://uri.etsi.org/03120/common/2020/09/Document', - 'NotificationObject' : 'http://uri.etsi.org/03120/common/2016/02/Notification', - 'LITaskObject' : 'http://uri.etsi.org/03120/common/2020/09/Task', - 'LPTaskObject' : 'http://uri.etsi.org/03120/common/2020/09/Task', - 'LDTaskObject' : 'http://uri.etsi.org/03120/common/2020/09/Task', - 'TrafficPolicyObject' : 'http://uri.etsi.org/03120/common/2022/07/TrafficPolicy', - 'TrafficRuleObject' : 'http://uri.etsi.org/03120/common/2022/07/TrafficPolicy' + "AuthorisationObject": "http://uri.etsi.org/03120/common/2020/09/Authorisation", + "DeliveryObject": "http://uri.etsi.org/03120/common/2019/10/Delivery", + "DocumentObject": "http://uri.etsi.org/03120/common/2020/09/Document", + "NotificationObject": "http://uri.etsi.org/03120/common/2016/02/Notification", + "LITaskObject": "http://uri.etsi.org/03120/common/2020/09/Task", + "LPTaskObject": "http://uri.etsi.org/03120/common/2020/09/Task", + "LDTaskObject": "http://uri.etsi.org/03120/common/2020/09/Task", + "TrafficPolicyObject": "http://uri.etsi.org/03120/common/2022/07/TrafficPolicy", + "TrafficRuleObject": "http://uri.etsi.org/03120/common/2022/07/TrafficPolicy", } coerce_to_list = [ - 'auth:AuthorisationApprovalDetails', - 'auth:AuthorisationFlag', - 'auth:CSPID', - 'common:ApproverContactDetails', - 'ActionRequest', - 'ActionResponse', - 'ListResponseRecord', - 'AssociatedObject', - 'doc:DocumentSignature', - 'doc:DocumentProperty', - 'notification:AssociatedObjectStatus', - 'task:ApprovalDetails', - 'task:TargetIdentifierValue', - 'task:DeliveryDestination', - 'task:TaskFlag', - 'task:AlternativePreservationReference', - 'task:ApprovalDetails', - 'task:ObservedTimes', - 'task:RequestValue', - 'task:RequestSubtype', - 'task:LDDeliveryDestination', - 'task:LDTaskFlag', - 'task:TrafficPolicyReference', - 'tp:TrafficRuleReference', - 'tp:Criteria', - 'common:DictionaryEntry', - 'dictionaries:Dictionary', - 'config:TargetFormatTypeDefinitionEntry', - 'config:SupportedLIWorkflowEndpoint', - 'config:SupportedLPWorkflowEndpoint', + "auth:AuthorisationApprovalDetails", + "auth:AuthorisationFlag", + "auth:CSPID", + "common:ApproverContactDetails", + "ActionRequest", + "ActionResponse", + "ListResponseRecord", + "AssociatedObject", + "doc:DocumentSignature", + "doc:DocumentProperty", + "notification:AssociatedObjectStatus", + "task:ApprovalDetails", + "task:TargetIdentifierValue", + "task:DeliveryDestination", + "task:TaskFlag", + "task:AlternativePreservationReference", + "task:ApprovalDetails", + "task:ObservedTimes", + "task:RequestValue", + "task:RequestSubtype", + "task:LDDeliveryDestination", + "task:LDTaskFlag", + "task:TrafficPolicyReference", + "tp:TrafficRuleReference", + "tp:Criteria", + "common:DictionaryEntry", + "dictionaries:Dictionary", + "config:TargetFormatTypeDefinitionEntry", + "config:SupportedLIWorkflowEndpoint", + "config:SupportedLPWorkflowEndpoint", ] coerce_to_int = [ - 'ActionIdentifier', - 'delivery:SequenceNumber', - 'task:Order', - 'ErrorCode', - 'Generation', - 'tp:Order' + "ActionIdentifier", + "delivery:SequenceNumber", + "task:Order", + "ErrorCode", + "Generation", + "tp:Order", ] -coerce_to_bool = [ - 'delivery:LastSequence' -] +coerce_to_bool = ["delivery:LastSequence"] -coerce_to_empty = [ - 'GETCSPCONFIG' -] +coerce_to_empty = ["GETCSPCONFIG"] coerce_null_to_empty = [ - 'SupportedLIWorkflowEndpoints', - 'SupportedLPWorkflowEndpoints', - 'config:AssociatedLDRequestTypes', - 'config:AssociatedLDRequestSubtypes', - 'config:AssociatedLPRequestTypes', - 'config:AssociatedLPRequestSubtypes', - 'config:AssociatedLIServiceTypes', + "SupportedLIWorkflowEndpoints", + "SupportedLPWorkflowEndpoints", + "config:AssociatedLDRequestTypes", + "config:AssociatedLDRequestSubtypes", + "config:AssociatedLPRequestTypes", + "config:AssociatedLPRequestSubtypes", + "config:AssociatedLIServiceTypes", ] -def postprocessor (path, key, value): + +def postprocessor(path, key, value): if key == "@xsi:type": object_name = value.split(":")[-1] if object_name in object_namespaces.keys(): @@ -118,26 +118,38 @@ def postprocessor (path, key, value): return key, {} return key, value + def recursively_fix_traffic_policy_criteria(d: dict): if isinstance(d, dict): - if ("tp:Criteria" in d + if ( + "tp:Criteria" in d and isinstance(d["tp:Criteria"], list) and len(d["tp:Criteria"]) == 1 - ): - d["tp:Criteria"] = d["tp:Criteria"][0] + ): + d["tp:Criteria"] = d["tp:Criteria"][0] else: - for k,v in d.items(): + for k, v in d.items(): recursively_fix_traffic_policy_criteria(v) elif isinstance(d, list): for d_item in d: recursively_fix_traffic_policy_criteria(d_item) - if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-v', '--verbose', action='count', help='Verbose logging (can be specified multiple times)') - parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Verbose logging (can be specified multiple times)", + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to input file (if absent, stdin is used)", + ) args = parser.parse_args() match args.verbose: @@ -155,10 +167,9 @@ if __name__ == "__main__": logging.debug(s) - d = xmltodict.parse(s, - force_list=tuple(coerce_to_list), - postprocessor=postprocessor - )['HI1Message'] + d = xmltodict.parse( + s, force_list=tuple(coerce_to_list), postprocessor=postprocessor + )["HI1Message"] # HACK # Needed because TrafficPolicy.xsd has two nested fields both called tp:Criteria -- GitLab