diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b43a20857c5fd19de5421b592931db7a423b45b..b4e3df33f5089456fc4d64b9dcd03d40ab38864c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,17 @@ -image: "python:3.7" - -before_script: - - python3 --version - - pip3 install -q asn1tools lxml xmlschema - stages: - - Check Schemas + - check -checkASN1: - stage: Check Schemas +process_asn: + image: "mcanterb/asn1test:latest" + stage: check + interruptible: true script: - - python3 testing/check_asn1.py \ No newline at end of file + - python3 testing/asn_process.py + +process_xsd: + image: "mcanterb/xsdtest:latest" + stage: check + interruptible: true + script: + - python3 testing/xsd_process.py + diff --git a/102232-1/LI-PS-PDU.asn b/102232-1/LI-PS-PDU.asn index 25f72fc6b49f91d65a74524745105796e2b7aa78..714ecf85b3130bd500dc922f76ddf0ffba14b2e3 100644 --- a/102232-1/LI-PS-PDU.asn +++ b/102232-1/LI-PS-PDU.asn @@ -1,5 +1,5 @@ LI-PS-PDU -{itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) genHeader(1) version32(32)} +{itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) genHeader(1) version33(33)} DEFINITIONS IMPLICIT TAGS ::= @@ -34,7 +34,7 @@ IMPORTS MessagingMMCC, MessagingIRI FROM EmailPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) email(2) version18(18)} + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) email(2) version19(19)} -- from ETSI TS 102 232-3 [6] IPCC, @@ -54,7 +54,7 @@ IMPORTS IPMMCC, IPMMIRI FROM IPMultimediaPDU - {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPMultimedia(5) version13(13)} + {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPMultimedia(5) version14(14)} -- from ETSI TS 102 232-6 [36] PstnIsdnCC, @@ -204,7 +204,7 @@ IMPORTS lawfulInterceptDomainId OBJECT IDENTIFIER ::= {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2)} -li-psDomainId OBJECT IDENTIFIER ::= {lawfulInterceptDomainId li-ps(5) genHeader(1) version32(32)} +li-psDomainId OBJECT IDENTIFIER ::= {lawfulInterceptDomainId li-ps(5) genHeader(1) version33(33)} -- ==================== -- Top-level definition @@ -732,4 +732,4 @@ LawfulInterceptionIdentifier ::= LIID -- module to preserve the original type name during the -- removal of imports from ETSI TS 101 671 [4]. -END --end of LI-PS-PDU \ No newline at end of file +END --end of LI-PS-PDU diff --git a/102232-2/EmailPDU.asn b/102232-2/EmailPDU.asn index afedfc6f3179f7f51edb9bdac063bc3de7dda07e..6f8457e3eb2006dddb017b7fa0b2c4727e303f54 100644 --- a/102232-2/EmailPDU.asn +++ b/102232-2/EmailPDU.asn @@ -391,7 +391,7 @@ Messaging-Party-Identity ::= CHOICE { msisdn [0] OCTET STRING (SIZE (1..9)), -- MSISDN of the target, encoded in the same format as the AddressString - -- parameters defined in MAP format 3GPP TS 09.02 [22], clause 17.7.8. + -- parameters defined in MAP format 3GPP TS 29.002 [36], clause 17.7.8. e164-format [1] OCTET STRING (SIZE (1..25)), -- E.164 address of the node in international format. Coded in the same format as -- the calling party number parameter of the ISUP (parameter part: EN 300 356 [23]) @@ -541,4 +541,4 @@ Messaging-Property-Requested-Reports ::= ENUMERATED ... } -END -- end of EmailPDU \ No newline at end of file +END -- end of EmailPDU diff --git a/102232-4/L2AccessPDU.asn b/102232-4/L2AccessPDU.asn index df562af6659bca77e854cad9faa9f5e0148ef827..7f18d897c49c6f59feef3002fd3904d1e10e1ce6 100644 --- a/102232-4/L2AccessPDU.asn +++ b/102232-4/L2AccessPDU.asn @@ -36,9 +36,9 @@ L2CC ::= SEQUENCE -- The ethernet protocol is used ..., l2ATM2684 [6] OCTET STRING, - -- The protocol RFC 2684, method "LLC Encapsulation for Bridged Protocols` [16] is used + -- The protocol RFC 2684, method "LLC Encapsulation for Bridged Protocols" [16] is used l2FR2427 [7] OCTET STRING - -- The protocol RFC 2427 “Multiprotocol Interconnect over Frame Relay” [18] is used + -- The protocol RFC 2427 "Multiprotocol Interconnect over Frame Relay" [18] is used } } @@ -194,7 +194,7 @@ L2ProtocolInformation ::= ENUMERATED l2ProtocolATM2684(7), -- The protocol RFC 2684, method "LLC Encapsulation for Bridged Protocols" [16] is used l2ProtocolFR2427(8) - -- The protocol RFC2427 “Multiprotocol Interconnect over Frame Relay” [18] is used + -- The protocol RFC2427 "Multiprotocol Interconnect over Frame Relay" [18] is used } END -- end of L2AccessPDU diff --git a/102232-5/IPMultimediaPDU.asn b/102232-5/IPMultimediaPDU.asn index d70768d6e317678923593920530fc57507fff441..c9408c388c8ff7daea692ce018fd50d67a859397 100644 --- a/102232-5/IPMultimediaPDU.asn +++ b/102232-5/IPMultimediaPDU.asn @@ -4,7 +4,7 @@ IPMultimediaPDU -{itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPMultimedia(5) version13(13)} +{itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2) li-ps(5) iPMultimedia(5) version14(14)} DEFINITIONS IMPLICIT TAGS ::= BEGIN @@ -22,8 +22,8 @@ IMPORTS -- Object Identifier Definition -- ============================ -iPMMIRIObjId RELATIVE-OID ::= {li-ps(5) iPMultimedia(5) version13(13) iRI(1)} -iPMMCCObjId RELATIVE-OID ::= {li-ps(5) iPMultimedia(5) version13(13) cC(2)} +iPMMIRIObjId RELATIVE-OID ::= {li-ps(5) iPMultimedia(5) version14(14) iRI(1)} +iPMMCCObjId RELATIVE-OID ::= {li-ps(5) iPMultimedia(5) version14(14) cC(2)} -- both definitions relative to: -- {itu-t(0) identified-organization(4) etsi(0) securityDomain(2) lawfulIntercept(2)} @@ -51,7 +51,9 @@ IPMMCC ::= SEQUENCE -- May contain c= and m= lines extracts for instance mMCCprotocol [4] MMCCprotocol OPTIONAL -- Used to identify the protocol of packets sent in MMCCContent (RTP, UDPTL, MSRP, etc.) - -- Absence means mMCCContents contains RTP/RTCP packets + -- Absence means mMCCContents contains RTP/RTCP packets or audio frame as indicated by the + -- frameType parameter. + } FrameType ::= ENUMERATED diff --git a/103120/examples/mutliple-auth-example-request.xml b/103120/examples/mutliple-auth-example-request.xml index 8fd893c24d5aec216cb2197368892483015204f0..ab1b79ea3216856cd5cde2b9ac99880b43b23b3d 100644 --- a/103120/examples/mutliple-auth-example-request.xml +++ b/103120/examples/mutliple-auth-example-request.xml @@ -12,7 +12,7 @@ d442c58c-d5e1-4fd9-90ec-9c228ad947f1 2020-09-22T08:06:17.025833Z - V1.10.1 + V1.11.1 XX v1.1.1 diff --git a/103120/examples/request1.xml b/103120/examples/request1.xml index a5e40d645c53e549c6002e105a5352b44dbbec47..2479534c73654e0576c1db35cb8a8eafbe6271d9 100644 --- a/103120/examples/request1.xml +++ b/103120/examples/request1.xml @@ -12,7 +12,7 @@ c02358b2-76cf-4ba4-a8eb-f6436ccaea2e 2015-09-01T12:00:00.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/request2.xml b/103120/examples/request2.xml index 2b1d012346c37b7d0b5689aa2fe895b1fa5c393e..582bb6e4b0dae70594ded44c55bad4a6af5a90ef 100644 --- a/103120/examples/request2.xml +++ b/103120/examples/request2.xml @@ -12,7 +12,7 @@ 45002c1e-dc4a-470a-9152-8e752638c86c 2015-09-01T12:01:00.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/request3.xml b/103120/examples/request3.xml index dce66207fc39648701cd1673074b547fb02ca5bd..7168d8401745d33330730bd9070055ebe2a41650 100644 --- a/103120/examples/request3.xml +++ b/103120/examples/request3.xml @@ -12,7 +12,7 @@ 69353ac0-9582-4c71-b162-86259c99de20 2015-09-01T12:02:00.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/request4.xml b/103120/examples/request4.xml index 73d6c07783352e8bfe3bd8327c2b261196de5e10..9093778542fe2decd848fb7b159525838d898ed9 100644 --- a/103120/examples/request4.xml +++ b/103120/examples/request4.xml @@ -12,7 +12,7 @@ c02358b2-76cf-4ba4-a8eb-f6436ccaea2e 2019-09-30T13:37:00.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/request5-Binary-Delivery.xml b/103120/examples/request5-Binary-Delivery.xml index 2a13dfbbe6b9ad652b8dd8cb82195d8757a49267..27b7f8a83614ebf1913c6c9760736a196930bfe3 100644 --- a/103120/examples/request5-Binary-Delivery.xml +++ b/103120/examples/request5-Binary-Delivery.xml @@ -12,7 +12,7 @@ 8854cfad-44ac-43b8-99ae-530b690b43da 2019-09-30T13:37:37.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/request5-XML-Delivery.xml b/103120/examples/request5-XML-Delivery.xml index c345dbbe507c78e3c2b581f5410b964232920437..57b627c5d495284298f3e6d164ea9affd718849b 100644 --- a/103120/examples/request5-XML-Delivery.xml +++ b/103120/examples/request5-XML-Delivery.xml @@ -12,7 +12,7 @@ 8854cfad-44ac-43b8-99ae-530b690b43da 2019-09-30T13:37:37.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/request6.xml b/103120/examples/request6.xml index 15a2bf1f259595a8f2c4520a146253429287bcfb..5ff9ed243e3e2ab8da1c4e1f3589cf1d28b10b76 100644 --- a/103120/examples/request6.xml +++ b/103120/examples/request6.xml @@ -12,7 +12,7 @@ c02358b2-76cf-4ba4-a8eb-f6436ccaea2f 2021-10-11T13:37:00.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/request6_signed.xml b/103120/examples/request6_signed.xml index 2bc0404ef924498e7280669c399ac4c430873787..61bbf4399da17ef7ef79681f892bf8f91c09f052 100644 --- a/103120/examples/request6_signed.xml +++ b/103120/examples/request6_signed.xml @@ -11,7 +11,7 @@ 9964584e-c1a5-4ffa-b949-d9da504c4efb 2021-06-25T12:00:00.000000Z - V1.9.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/response1.xml b/103120/examples/response1.xml index 54fad1cea1741d9c2d3fd9e8fac21816b4206551..7a989b9977170a745de48e156c9ddb2e2d3f9fd5 100644 --- a/103120/examples/response1.xml +++ b/103120/examples/response1.xml @@ -12,7 +12,7 @@ c02358b2-76cf-4ba4-a8eb-f6436ccaea2e 2015-09-01T12:00:01.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/response2.xml b/103120/examples/response2.xml index 104a76f5455b7625ca859d4e5cb919d9d6a40b57..a08f8d689d47309315b8671ffb381c5584e2281b 100644 --- a/103120/examples/response2.xml +++ b/103120/examples/response2.xml @@ -12,7 +12,7 @@ 45002c1e-dc4a-470a-9152-8e752638c86c 2015-09-01T12:01:00.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/response3.xml b/103120/examples/response3.xml index 445187d48fdb19f46e96cfb813f467c9190d9b01..184967b2f7b43940235d7eeb64ef928c2fe1eb00 100644 --- a/103120/examples/response3.xml +++ b/103120/examples/response3.xml @@ -12,7 +12,7 @@ 69353ac0-9582-4c71-b162-86259c99de20 2015-09-01T12:02:00.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/response4.xml b/103120/examples/response4.xml index 3ca45c26bb40c2ab11ec4c7dda7a2490639ad354..3c9f48204be94ed3a05329b0c24f92469294d716 100644 --- a/103120/examples/response4.xml +++ b/103120/examples/response4.xml @@ -12,7 +12,7 @@ c02358b2-76cf-4ba4-a8eb-f6436ccaea2e 2019-09-30T13:37:01.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/response5.xml b/103120/examples/response5.xml index d02dff5149c3ae24115c323f76194bfa70b93677..54572b35ffde7b3ee89b5b11d0e179bf0c8e274e 100644 --- a/103120/examples/response5.xml +++ b/103120/examples/response5.xml @@ -17,7 +17,7 @@ 8854cfad-44ac-43b8-99ae-530b690b43da 2019-09-30T13:37:37.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/examples/response6.xml b/103120/examples/response6.xml index 99b0ddc135eb3e7269dab1b19373cf53b8c5b7ef..f5ab20932f9440d9a40772f725162cb45fd197eb 100644 --- a/103120/examples/response6.xml +++ b/103120/examples/response6.xml @@ -12,7 +12,7 @@ c02358b2-76cf-4ba4-a8eb-f6436ccaea2f 2021-10-11T13:37:01.000000Z - V1.10.1 + V1.11.1 XX v1.0 diff --git a/103120/schema/ts_103120_Delivery.xsd b/103120/schema/ts_103120_Delivery.xsd index 5fb2536ca89c68c0bb21a6bb97ecf77bb95fed20..367f0650e3655f9cbd35e3103e303d32346482ec 100644 --- a/103120/schema/ts_103120_Delivery.xsd +++ b/103120/schema/ts_103120_Delivery.xsd @@ -27,7 +27,7 @@ - + @@ -39,7 +39,7 @@ - + @@ -59,7 +59,7 @@ - + @@ -73,6 +73,7 @@ + @@ -80,5 +81,5 @@ - + diff --git a/103120/schema/ts_103120_Document.xsd b/103120/schema/ts_103120_Document.xsd index af6f13979824aba2c967860b1473bc7df91fac32..499e1cde702e82a7973113efe028d53039c5248d 100644 --- a/103120/schema/ts_103120_Document.xsd +++ b/103120/schema/ts_103120_Document.xsd @@ -53,6 +53,7 @@ + diff --git a/103221-1/TS_103_221_01.xsd b/103221-1/TS_103_221_01.xsd index 490e9c4eb68f36c6bb328608b07571c4edf1ff71..fcf37429e6453e24f1b80306bfec941a67bd14cc 100644 --- a/103221-1/TS_103_221_01.xsd +++ b/103221-1/TS_103_221_01.xsd @@ -1,5 +1,5 @@ - + @@ -107,6 +107,7 @@ + @@ -299,6 +300,7 @@ + @@ -922,4 +924,21 @@ + + + + + + + + + + + + + + + + + diff --git a/103221-1/TS_103_221_01_DestinationSet.xsd b/103221-1/TS_103_221_01_DestinationSet.xsd new file mode 100644 index 0000000000000000000000000000000000000000..1bdcb98121179e30fb9899187955b904bf928f48 --- /dev/null +++ b/103221-1/TS_103_221_01_DestinationSet.xsd @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/103221-1/examples/CreateDestinationSet_example.xml b/103221-1/examples/CreateDestinationSet_example.xml new file mode 100644 index 0000000000000000000000000000000000000000..c1388b42c301ea02e20d21e36d6d156f25f736fc --- /dev/null +++ b/103221-1/examples/CreateDestinationSet_example.xml @@ -0,0 +1,88 @@ + + + + admfID + neID + 2022-02-16T18:46:21.247432Z + v1.11.1 + 7fa872e0-4bc7-4a98-a416-5f22b3de1dca + + 44b739c4-73f0-48d5-8d6a-3576ab100141 + X2andX3 + + +
+ 203.0.113.1 +
+ + 9999 + + + + + + + admfID + neID + 2022-02-16T18:46:21.247432Z + v1.11.1 + 8286e40c-0b4b-4d4c-94f0-1e7ab8489e44 + + 36adec69-e0f0-4a41-8e85-66bbe12c5a60 + X2andX3 + + +
+ 203.0.113.2 +
+ + 9999 + + + + + + + admfID + neID + 2022-02-16T18:46:21.247432Z + v1.11.1 + 7afbf8e3-7699-4008-ac1a-4475ed50c14b + + 30fbde5e-3195-4a22-b3e9-15e1d60d0528 + My First Destination Set + + + 44b739c4-73f0-48d5-8d6a-3576ab100141 + 1 + + + 36adec69-e0f0-4a41-8e85-66bbe12c5a60 + 2 + + + Redundant + + + + admfID + neID + 2022-02-16T18:46:21.247432Z + v1.11.1 + 4afef6be-5252-41cb-9275-123d308deb50 + + 1d7d7eb7-e0b9-4a5c-827a-bc3639d48124 + + + 6125550123 + + + X2andX3 + + 30fbde5e-3195-4a22-b3e9-15e1d60d0528 + + + + diff --git a/103221-1/examples/DeleteAllObjectsRequest_example.xml b/103221-1/examples/DeleteAllObjectsRequest_example.xml new file mode 100644 index 0000000000000000000000000000000000000000..86b1650e2bb80cd0b33c36b27b3aded365ed1893 --- /dev/null +++ b/103221-1/examples/DeleteAllObjectsRequest_example.xml @@ -0,0 +1,10 @@ + + + + admfID + neID + 2022-01-20T14:04:20.123456Z + v1.11.1 + 5775af53-a4a0-46ab-818a-86a406a335a1 + + diff --git a/103221-1/examples/DeleteAllObjectsResponse_example.xml b/103221-1/examples/DeleteAllObjectsResponse_example.xml new file mode 100644 index 0000000000000000000000000000000000000000..596d5189e03c23070fdd65bfae286c365addb068 --- /dev/null +++ b/103221-1/examples/DeleteAllObjectsResponse_example.xml @@ -0,0 +1,11 @@ + + + + admfID + neID + 2022-01-20T14:04:20.123456Z + v1.11.1 + 5775af53-a4a0-46ab-818a-86a406a335ac + AcknowledgedAndCompleted + + diff --git a/103221-1/examples/desktop.ini b/103221-1/examples/desktop.ini deleted file mode 100644 index 8b699c692d358fa29d7ac0976e524fe25c5da053..0000000000000000000000000000000000000000 Binary files a/103221-1/examples/desktop.ini and /dev/null differ diff --git a/103280/TS_103_280.xsd b/103280/TS_103_280.xsd index d6d91c5cca5b344d4a62035e515531407f180d38..3ff4ae13e53186ed51828193f383337448c8e3a1 100644 --- a/103280/TS_103_280.xsd +++ b/103280/TS_103_280.xsd @@ -1,5 +1,5 @@ - + @@ -246,4 +246,24 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/103707/TS_103_707.xsd b/103707/TS_103_707.xsd index 7bce63dbbddf26c259af57022ea51f89f2a12c04..f9e6da158f77ea324effb2a1b519d5e57f7b88ad 100644 --- a/103707/TS_103_707.xsd +++ b/103707/TS_103_707.xsd @@ -3,7 +3,7 @@ xmlns:etsi103280="http://uri.etsi.org/03280/common/2017/07" xmlns="http://uri.etsi.org/03707/2020/02" targetNamespace="http://uri.etsi.org/03707/2020/02" - version="1.4.1" + version="1.5.1" elementFormDefault="qualified"> diff --git a/103707/examples/example_multiple_items.xml b/103707/examples/example_multiple_items.xml new file mode 100644 index 0000000000000000000000000000000000000000..6913c2ba2d454193c7f5c95df3500df657f849b1 --- /dev/null +++ b/103707/examples/example_multiple_items.xml @@ -0,0 +1,75 @@ + + + + +
+ + + + + target@example.com + + true + + + + + someoneelse@example.com + + + + 2021-06-17T06:42:23.000000+01:00 + + + + + http://FooServiceSchema.example.com/schema/v1.1.1/ + + + + Foo + Bar + Baz + + + + + + + +
+ + + + + someoneelse@example.com + + + + + + target@example.com + + true + + + 2021-06-18T06:42:23.000000+01:00 + + + + + http://FooServiceSchema.example.com/schema/v1.1.1/ + + + + Foo + Bar + Baz + + + + + + + + diff --git a/testing/asn_compile_targets.json b/testing/asn_compile_targets.json new file mode 100644 index 0000000000000000000000000000000000000000..e186addc0214f2ecf1f25f507336f33725fafe2f --- /dev/null +++ b/testing/asn_compile_targets.json @@ -0,0 +1,33 @@ +[ + [ + "./102232-1/LI-PS-PDU.asn", + "./103280/TS_103_280.asn1", + "./testing/deps/33128/TS33128Payloads.asn", + "./testing/deps/33108/Three3gppHI1Notifications.asn", + "./testing/deps/33108/UMTSHI2Operations.asn", + "./testing/deps/33108/UMTSHI3PS.asn", + "./testing/deps/33108/EpsHI3PS.asn", + "./testing/deps/33108/ConfHI3IMS.asn", + "./testing/deps/33108/VoipHI3IMS.asn", + "./testing/deps/33108/GCSEHI3.asn", + "./testing/deps/33108/CSVoiceHI3IP.asn", + "./testing/deps/33108/UMTSCSHI2Operations.asn", + "./testing/deps/33108/EpsHI2Operations.asn", + "./testing/deps/33108/ConfHI2Operations.asn", + "./testing/deps/33108/ProSeHI2Operations.asn", + "./testing/deps/33108/GCSEHI2Operations.asn", + "./testing/deps/101671/HI1NotificationOperations,ver7.asn", + "./testing/deps/101671/HI2Operations,ver18.asn", + "./testing/deps/101909/TS101909201.asn", + "./testing/deps/101909/TS101909202.asn", + "./testing/deps/101909/PCESP.asn", + "./testing/deps/301040/06132v203_C01.asn", + "./103462/ILHIPDU.asn", + "./102232-2/EmailPDU.asn", + "./102232-3/IPAccessPDU.asn", + "./102232-4/L2AccessPDU.asn", + "./102232-5/IPMultimediaPDU.asn", + "./102232-6/PstnIsdnPDU.asn" + ], + ["./102657/RDMessage.asn"] +] \ No newline at end of file diff --git a/testing/asn_ignore.txt b/testing/asn_ignore.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d81d205c4e0a24ce60da75c4e11e6fd0bae44cb --- /dev/null +++ b/testing/asn_ignore.txt @@ -0,0 +1,2 @@ +deps +temp \ No newline at end of file diff --git a/testing/asn_ignore_lint.txt b/testing/asn_ignore_lint.txt new file mode 100644 index 0000000000000000000000000000000000000000..03e7f54e7937f1cf45bf259cef700a7624bf6912 --- /dev/null +++ b/testing/asn_ignore_lint.txt @@ -0,0 +1 @@ +dependencies \ No newline at end of file diff --git a/testing/asn_process.py b/testing/asn_process.py new file mode 100644 index 0000000000000000000000000000000000000000..aa0a0a63777202d29ee51587ffdafb760c360560 --- /dev/null +++ b/testing/asn_process.py @@ -0,0 +1,206 @@ +import logging +import json +from pathlib import Path +from subprocess import run +from re import sub + +from pycrate_asn1c.asnproc import * + +def reconstrainInteger (filename): + Path('temp.asn').write_text(Path(filename).read_text().replace("18446744073709551615", "65536")) + return 'temp.asn' + +filesWithBigInts = [ + '102232-1/LI-PS-PDU.asn', + '102232-3/IPAccessPDU.asn', + '102232-4/L2AccessPDU.asn' +] + +def syntaxCheckASN (fileList): + """ + Performs ASN syntax checking on a list of filenames (or pathlib Paths) + + :param fileList: List of filenames (str or Pathlib Path) + :returns: Dict with result, return code and message for each filename + + Calls the open-source asn1c compiler with the "syntax only" option. + As a result, asn1c must be available to run. + """ + results = {} + for file in fileList: + try: + if file.as_posix() in filesWithBigInts: + newFile = reconstrainInteger(str(file)) + p = run(['asn1c', '-E', newFile], capture_output=True) + Path(newFile).unlink() + else: + p = run(['asn1c', '-E', str(file)], capture_output=True) + if (p.returncode != 0): + errorMessage = p.stderr.decode().splitlines()[0] + if errorMessage.startswith(' Value "18446744073709551615" at line'): + results[str(file)] = { 'ok' : True} + continue + results[str(file)] = { + 'ok' : False, + 'code' : p.returncode, + 'message' : p.stderr.decode().splitlines()[0] + } + else: + results[str(file)] = { + 'ok' : True + } + except Exception as ex: + results[str(file)] = { + 'ok' : False, + 'code' : -1, + 'message' : f"{ex!r}" + } + return results + + +duplicateObjects = { + '102232-1/LI-PS-PDU.asn' : [ + 'CCPayload', + 'IRIPayload', + 'Location' + ], + 'testing/mod1.asn' : [ + 'ClashField' + ] +} +def fixDuplicateObjects(filename): + stringContent = filename.read_text() + for object in duplicateObjects[filename.as_posix()]: + stringContent = stringContent.replace(f'{object} ::=', f'Native{object} ::=') + stringContent = stringContent.replace(f'SEQUENCE OF {object}', f'SEQUENCE OF Native{object}') + #stringContent = sub(f"]\\w{object}", f"] Native{object}", stringContent) + + Path('temp.asn').write_text(stringContent) + return 'temp.asn' + + +def compileAllTargets (compileTargets): + """ + Attempts to compile a set of compile targets using the pycrate ASN1 tools + + :param compileTargets: list of compile targets, each of which is a list of filenames + :returns: A dict of outcome against the first filename of each compile target. Return code and message are included for failures. + + For each compile target (list of filenames) the first filename is assumed + to be the "primary" file. This doesn't have any relavance to the compilation, + but will be used as the identifier when reporting any compile errors. + The compilation is performed by the pycrate ASN compile functions; errors + are caught as exceptions and rendered into a list. + + Unfortunately, the pycrate compiler doesn't report line numbers. + The asn1c compiler does, but doesn't properly handle identifiers with the + same name in different modules; as this occurs multiple times in TS 33.108, + we can't use it. + """ + results = {} + for target in compileTargets: + firstTarget = target[0] + logging.debug(f"Compiling {firstTarget}") + try: + fileTexts = [] + fileNames = [] + GLOBAL.clear() + for filename in target: + pFile = Path(filename) + if pFile.as_posix() in duplicateObjects: + tmpFile = Path(fixDuplicateObjects(pFile)) + fileTexts.append(tmpFile.read_text()) + #tmpFile.unlink() + else: + fileTexts.append(pFile.read_text()) + fileNames.append(filename) + logging.debug (f" Loading {filename}") + compile_text(fileTexts, filenames = fileNames) + results[str(firstTarget)] = { + 'ok' : True, + } + except Exception as ex: + results[str(firstTarget)] = { + 'ok' : False, + 'code' : -1, + 'message' : f"{ex!r}" + } + continue + return results + + + +def processResults (results, stageName): + """ + Counts the number of errors and writes out the output per filename + + :param results: List of filenames (str or Pathlib Path) + :param stageName: Name to decorate the output with + :returns: The number of files which had errors + """ + print("") + errorCount = sum([1 for r in results.values() if not r['ok']]) + logging.info(f"{errorCount} {stageName} errors encountered") + + print(f"{'-':-<60}") + print(f"{stageName} results:") + print(f"{'-':-<60}") + for filename, result in results.items(): + print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") + if not result['ok']: + if isinstance(result['message'], list): + for thing in result['message']: + print(f" {thing['message']}") + else: + print(f" {result['message']}") + + print(f"{'-':-<60}") + print(f"{stageName} errors: {errorCount}") + print(f"{'-':-<60}") + + return errorCount + + +if __name__ == '__main__': + logging.info('Searching for ASN.1 files') + fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) + logging.info(f'{len(fileList)} ASN.1 files found') + for file in fileList: + logging.debug(f' {file}') + + ignoreList = Path('testing/asn_ignore.txt').read_text().splitlines() + ignoredFiles = [] + for ignore in ignoreList: + logging.debug(f'Ignoring pattern {ignore}') + for file in fileList: + if ignore in str(file): + ignoredFiles.append(file) + logging.debug(f" Ignoring {str(file)} as contains {ignore}") + ignoredFiles = list(set(ignoredFiles)) + logging.info(f'{len(ignoredFiles)} files ignored') + for file in ignoredFiles: + logging.debug(f' {file}') + + fileList = [file for file in fileList if file not in ignoredFiles] + logging.info(f'{len(fileList)} files to process') + for file in fileList: + logging.debug(f' {file}') + + if len(fileList) == 0: + logging.warning ("No files specified") + exit(0) + + logging.info("Parsing ASN1 files") + parseResults = syntaxCheckASN(fileList) + if processResults(parseResults, "Parsing") > 0: + exit(-1) + + logging.info ("Getting compile targets") + compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text()) + logging.info (f"{len(compileTargets)} compile targets found") + + compileResults = compileAllTargets(compileTargets) + if processResults(compileResults, "Compiling") > 0: + exit(-1) + + exit(0) diff --git a/testing/check_asn1.py b/testing/check_asn1.py deleted file mode 100644 index c87a22619f847248a1281a18356dbbfcbfaf6a13..0000000000000000000000000000000000000000 --- a/testing/check_asn1.py +++ /dev/null @@ -1,31 +0,0 @@ -import logging -from compile_asn import * - - -if __name__ == '__main__': - log = logging.getLogger() - log.setLevel(logging.INFO) - parseErrors, compileErrors, parser = validateAllASN1FilesInPath("./") - parseErrorCount = 0 - print ("ASN.1 Parser checks:") - print ("-----------------------------") - for filename, errors in parseErrors.items(): - if len(errors) > 0: - parseErrorCount += len(errors) - print (f"{filename}: {len(errors)} errors") - for error in errors: - print (" " + str(error)) - else: - print (f"{filename}: OK") - print ("-----------------------------") - print ("ASN.1 Compilation:") - print ("-----------------------------") - if len(compileErrors) > 0: - for error in compileErrors: - print (" " + str(error)) - else: - print ("Compilation OK") - print ("-----------------------------") - print (f"{parseErrorCount} parse errors, {len(compileErrors)} compile errors") - exit (parseErrorCount + len(compileErrors)) - diff --git a/testing/check_xsd.py b/testing/check_xsd.py deleted file mode 100644 index 816e3898aa2305c476a665b32c14237c82c22c3d..0000000000000000000000000000000000000000 --- a/testing/check_xsd.py +++ /dev/null @@ -1,194 +0,0 @@ -import logging - -import glob -import sys -import argparse -from pathlib import Path -from pprint import pprint -import os - -from lxml import etree -from xml.etree.ElementTree import ParseError -from xmlschema import XMLSchema, XMLSchemaParseError - -def BuildSchemaDictonary (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return [] - - logging.info("Schema locations:") - schemaLocations = [] - for schemaFile in fileList: - try: - xs = XMLSchema(schemaFile, validation='skip') - schemaLocations.append((xs.target_namespace, str(Path(schemaFile).resolve()))) - logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, schemaFile)) - except ParseError as ex: - logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) - return schemaLocations - - -def BuildSchema (coreFile, fileList = None): - schemaLocations = [] - if fileList and len(fileList) > 0: - schemaLocations = BuildSchemaDictonary(fileList) - - coreSchema = XMLSchema(str(Path(coreFile)), locations=schemaLocations) - return coreSchema - -def ValidateSingleFile (schemaFile): - try: - xs = XMLSchema(schemaFile, validation='skip') - except ParseError as ex: - logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) - return ex - return None - - -def ValidateXSDFiles (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return {} - - schemaLocations = BuildSchemaDictonary(fileList) - errors = {} - schemaDictionary = {} - - logging.info("Schema validation:") - for schemaFile in fileList: - try: - schema = XMLSchema(schemaFile, locations = schemaLocations) - logging.info(schemaFile + ": OK") - errors[schemaFile] = [] - schemaDictionary[schema.target_namespace] = schema - except XMLSchemaParseError as ex: - if (ex.schema_url) and (ex.schema_url != ex.origin_url): - logging.info(" Error {1} comes from {0}, suppressing".format(ex.schema_url, ex.message)) - errors[schemaFile] = [] - else: - logging.warning(schemaFile + ": Failed validation ({0})".format(ex)) - errors[schemaFile] = [ex.message] - return errors, schemaDictionary - - -def ValidateInstanceDocuments (coreFile, supportingSchemas, instanceDocs): - if (instanceDocs is None) or len(instanceDocs) == 0: - logging.warning ("No instance documents provided") - return [] - - schema = BuildSchema(coreFile, supportingSchemas) - errors = [] - for instanceDoc in instanceDocs: - try: - schema.validate(instanceDoc) - logging.info ("{0} passed validation".format(instanceDoc)) - except Exception as ex: - logging.error ("{0} failed validation: {1}".format(instanceDoc, ex)) - return errors - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbosity", help="verbosity level", action="count", default=0) - parser.add_argument("input", help="include a directory or file", action="append", nargs="+") - parser.add_argument("-p", "--primaryNamespace", help="Primary schema namespace for instance doc validation") - args = parser.parse_args() - - logging.getLogger().setLevel(logging.WARNING) - if (args.verbosity >= 1): - logging.getLogger().setLevel(logging.INFO) - if (args.verbosity >= 2): - logging.getLogger().setLevel(logging.DEBUG) - logging.debug("Very verbose selected") - - - logging.debug(f"Path: {args.input}") - includeFileList = [] - includeInstanceDocList = [] - for path in args.input[0]: - p = Path(path) - if not p.exists(): - logging.error(f"Include path {path} not found") - exit(1) - if p.is_dir(): - logging.debug(f"Expanding directory") - for g in glob.glob(os.path.join(str(p), "*.xsd")): - logging.info(f">Including {g}") - includeFileList.append(g) - for g in glob.glob(os.path.join(str(p), "*.xml")): - logging.info(f">Including instance doc {g}") - includeInstanceDocList.append(g) - else: - logging.info(f">Including {p.absolute()}") - if str(p.absolute()).endswith('.xml'): - includeInstanceDocList.append(str(p.absolute())) - elif str(p.absolute()).endswith('.xsd'): - includeFileList.append(str(p.absolute())) - else: - logging.warning(f'Ignoring file {p.absolute()}') - - if len(includeInstanceDocList) and (args.primaryNamespace is None): - print("Cannot validate instance documents without specifying a primary namespace (use -h for usage guidelines)") - exit(-1) - - syntaxErrors = 0 - - print ("=============================") - print ("XSD syntax checks:") - print ("-----------------------------") - for file in includeFileList: - error = ValidateSingleFile(file) - if (error): - print (f" {file} : Syntax error [{error}]") - syntaxErrors += 1 - else: - print (f" {file} : OK") - - print ("-----------------------------") - if (syntaxErrors > 0): - print (f"{syntaxErrors} syntax errors detected") - exit(syntaxErrors) - else: - print ("0 syntax errors detected") - - results, schemaDict = ValidateXSDFiles(includeFileList) - - print ("=============================") - print ("XSD build checks:") - print ("-----------------------------") - errorCount = 0 - for fileName, errors in results.items(): - if len(errors) > 0: - errorCount += len(errors) - print (f" {fileName}: {len(errors)} errors") - for error in errors: - if isinstance(error, XMLSchemaParseError): - print (error.msg) - else: - print (f" {str(error.strip())}") - else: - print (f" {fileName}: OK") - - print ("-----------------------------") - print (f"{errorCount} build errors detected") - if (errorCount > 0): - exit(errorCount) - - print ("=============================") - print ("Instance document checks") - print ("-----------------------------") - errorCount = 0 - - primarySchema = schemaDict[args.primaryNamespace] - for instanceDoc in includeInstanceDocList: - try: - results = primarySchema.validate(instanceDoc) - print (f" {instanceDoc} : OK") - except Exception as ex: - errorCount += 1 - print (f" {instanceDoc} : {str(ex)}") - print ("-----------------------------") - print (f"{errorCount} instance doc errors detected") - print ("=============================") - exit(errorCount) diff --git a/testing/compile_asn.py b/testing/compile_asn.py deleted file mode 100644 index e291e42d4077915416bbabba5be97db1cbb7ed50..0000000000000000000000000000000000000000 --- a/testing/compile_asn.py +++ /dev/null @@ -1,140 +0,0 @@ -import logging -import copy -from asn1tools import parse_files, compile_dict, ParseError, CompileError -from glob import glob -from pathlib import Path -from pprint import pprint - -def parseASN1File (asnFile): - try: - parse_files(asnFile) - except ParseError as ex: - return [ex] - return [] - - -def parseASN1Files (fileList): - if len(fileList) == 0: - logging.warning ("No files specified") - return {} - errors = {} - logging.info("Parsing files...") - for f in fileList: - ex = parseASN1File(f) - if ex: - logging.info (f" {f}: Failed - {ex!r}") - else: - logging.info (f" {f}: OK") - errors[f] = ex - return errors - -def fixDottedReference (dict, importingModule, importingType, importingMember, importedModule, importedType): - newName = importedModule + "_" + importedType - dict[importedModule]['types'][newName] = copy.deepcopy(dict[importedModule]['types'][importedType]) - dict[importingModule]['imports'][importedModule].append(newName) - member = [x for x in dict[importingModule]['types'][importingType]['members'] if x is not None and x['name'] == importingMember][0] - member['type'] = newName - -def compileASN1Files (fileList): - logging.info("Compiling files...") - errors = [] - imports = {} - - #p = re.compile(r"]\s+\S+\.\S+") - #for f in fileList: - # with open(f) as fh: - # s = fh.read() - # for match in p.findall(s): - # print (f"In {f}: {match}") - #exit() - - try: - dr = parse_files(fileList) - for modulename, module in dr.items(): - # Weird fix because the compiler doesn't like RELATIVE-OID as a type - # Not sure if the on-the-wire encoding would be affected or not - # but for most checking purposes this doesn't matter - module['types']["RELATIVE-OID"] = {'type' : 'OBJECT IDENTIFIER'} - for k,v in module['imports'].items(): - if not k in imports: - imports[k] = [] - imports[k].append({ - "in" : modulename, - "types" : v - }) - for k,v in imports.items(): - if not k in dr.keys(): - importers = [i['in'] for i in v] - errors.append(f"Unsatisfied import [{k}] for {importers}") - - fixDottedReference(dr, 'LI-PS-PDU', 'Location', 'umtsHI2Location', 'UmtsHI2Operations', 'Location') - fixDottedReference(dr, 'LI-PS-PDU', 'Location', 'epsLocation', 'EpsHI2Operations', 'Location') - fixDottedReference(dr, 'LI-PS-PDU', 'Location', 'eTSI671HI2Location', 'HI2Operations', 'Location') - - fixDottedReference(dr, 'LI-PS-PDU', 'UMTSIRI', 'iRI-Parameters', 'UmtsHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'UMTSIRI', 'iRI-CS-Parameters', 'UmtsCS-HI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'ETSI671IRI', 'iRI-Parameters', 'HI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'EPSIRI', 'iRI-EPS-Parameters', 'EpsHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'ConfIRI', 'iRI-Conf-Parameters', 'CONFHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'ProSeIRI', 'iRI-ProSe-Parameters', 'ProSeHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'GcseIRI', 'iRI-Gcse-Parameters', 'GCSEHI2Operations', 'IRI-Parameters') - - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'tTRAFFIC-1', 'TS101909201', 'TTRAFFIC') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cTTRAFFIC-1', 'TS101909201', 'CTTRAFFIC') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'tTRAFFIC-2', 'TS101909202', 'TTRAFFIC') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cTTRAFFIC-2', 'TS101909202', 'CTTRAFFIC') - #fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cCIPPacketHeader', 'CDMA2000CCModule', 'CCIPPacketHeader') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'uMTSCC-CC-PDU', 'Umts-HI3-PS', 'CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'ePSCC-CC-PDU', 'Eps-HI3-PS', 'CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'confCC-CC-PDU', 'CONF-HI3-IMS', 'Conf-CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'voipCC-CC-PDU', 'VoIP-HI3-IMS', 'Voip-CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'gcseCC-CC-PDU', 'GCSE-HI3', 'Gcse-CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cSvoice-CC-PDU', 'CSvoice-HI3-IP', 'CSvoice-CC-PDU') - - fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'tARGETACTIVITYMONITOR-1', 'TS101909201', 'TARGETACTIVITYMONITOR-1') - fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'tARGETACTIVITYMONITOR-2', 'TS101909202', 'TARGETACTIVITYMONITOR') - #fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'lAESProtocol', 'Laesp-j-std-025-b', 'LAESProtocol') - #fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'cDMA2000LAESMessage', 'CDMA2000CIIModule', 'CDMA2000LAESMessage') - - fixDottedReference(dr, 'LI-PS-PDU', 'HI4Payload', 'threeGPP-LI-Notification', 'TS33128Payloads', 'LINotificationPayload') - - fixDottedReference(dr, 'ILHIPDU', 'TimestampMapping', 'timeStampQualifier', 'LI-PS-PDU', 'TimeStampQualifier') - - fixDottedReference(dr, 'ILHIPDU', 'ILHITimestamp', 'qualifiedDateTime', 'Common-Parameters', 'QualifiedDateTime') - fixDottedReference(dr, 'ILHIPDU', 'ILHITimestamp', 'qualifiedMicrosecondDateTime', 'Common-Parameters', 'QualifiedMicrosecondDateTime') - - fixDottedReference(dr, 'ILHIPDU', 'OriginalTimestamp', 'microSecondTimeStamp', 'LI-PS-PDU', 'MicroSecondTimeStamp') - - fixDottedReference(dr, 'ILHIPDU', 'LocationMapping', 'originalLocation', 'LI-PS-PDU', 'Location') - - fixDottedReference(dr, 'ILHIPDU', 'GeocodedLocationData', 'wGS84CoordinateDecimal', 'Common-Parameters', 'WGS84CoordinateDecimal') - fixDottedReference(dr, 'ILHIPDU', 'GeocodedLocationData', 'wGS84CoordinateAngular', 'Common-Parameters', 'WGS84CoordinateAngular') - - c = compile_dict(dr) - except CompileError as ex: - logging.info (f"Compiler error: {ex}") - errors.append(ex) - return errors, None - except ParseError as ex: - logging.info (f"Parse error: {ex}") - errors.append(ex) - return errors, None - logging.info ("Compiled OK") - return errors, c - - -def validateASN1Files (fileList): - parseErrors = parseASN1Files(fileList) - errorCount = sum([len(v) for k,v in parseErrors.items()]) - if errorCount > 0: - logging.info ("Abandoning compile due to parse errors") - return parseErrors, [], None - compileErrors, parser = compileASN1Files(fileList) - return parseErrors, compileErrors, parser - - -def validateAllASN1FilesInPath (path): - p = Path(path) - fileGlob = [str(f) for f in p.rglob('*.asn')] - fileGlob += [str(f) for f in p.rglob('*.asn1')] - return validateASN1Files(fileGlob) \ No newline at end of file diff --git a/testing/deps/301040/06132v203_C01.asn b/testing/deps/301040/06132v203_C01.asn index 56305a2033eb1305d0b89a5d14d9443f3b7a361f..30c32121a5f83b5ce70fa71d29289f9fea022837 100644 --- a/testing/deps/301040/06132v203_C01.asn +++ b/testing/deps/301040/06132v203_C01.asn @@ -56,9 +56,9 @@ LISTATUSind ::= SEQUENCE TARGETACTIVITYMONITORind ::= SEQUENCE { - tLIInstanceid TLIIdType, -- header, who – - timestamp UTCTime, -- header, when – - targetLocation LocationType, -- header, where – + tLIInstanceid TLIIdType, -- header, who + timestamp UTCTime, -- header, when + targetLocation LocationType, -- header, where targetAction ActivityType, supplementaryTargetaddress AddressType OPTIONAL, cotargetaddress SEQUENCE OF AddressType OPTIONAL, @@ -178,16 +178,16 @@ TETRAAddressType ::= CHOICE { tETRAaddress TSIType, pISNaddress NumericString (SIZE (20)), - iP4address BIT STRING (SIZE (32)), -- 32 bits – - iP6address BIT STRING (SIZE (128)), -- 128 bits – + iP4address BIT STRING (SIZE (32)), -- 32 bits + iP6address BIT STRING (SIZE (128)), -- 128 bits e164address NumericString (SIZE (20)), tEI TEIType } -CellIdType ::= BIT STRING (SIZE (16)) -- 16 bits – +CellIdType ::= BIT STRING (SIZE (16)) -- 16 bits -LocationAreaType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-2 – +LocationAreaType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-2 LocationType ::= CHOICE { @@ -196,11 +196,11 @@ LocationType ::= CHOICE } -MCCType ::= BIT STRING (SIZE (10)) -- 10 bits, as defined in ETS 300 392-1 – +MCCType ::= BIT STRING (SIZE (10)) -- 10 bits, as defined in ETS 300 392-1 -MNCType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-1 – +MNCType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-1 -SSIType ::= BIT STRING (SIZE (24)) -- 24 bits, as defined in ETS 300 392-1 – +SSIType ::= BIT STRING (SIZE (24)) -- 24 bits, as defined in ETS 300 392-1 CircuitIdType ::= NumericString (SIZE (20)) @@ -255,7 +255,7 @@ TETRACGIType ::= SEQUENCE cI CellIdType OPTIONAL } -TLIIdType ::= BIT STRING (SIZE (16)) -- 16 bits – +TLIIdType ::= BIT STRING (SIZE (16)) -- 16 bits TSIType ::= SEQUENCE { @@ -264,6 +264,6 @@ TSIType ::= SEQUENCE ssi SSIType } -TEIType ::= BIT STRING (SIZE (60)) -- 60 bits, as defined in ETS 300 392-1 – +TEIType ::= BIT STRING (SIZE (60)) -- 60 bits, as defined in ETS 300 392-1 END diff --git a/testing/dockerfile_asn b/testing/dockerfile_asn new file mode 100644 index 0000000000000000000000000000000000000000..2a4198b43ab2d126a9db22fc372a3b1dbee8e4e5 --- /dev/null +++ b/testing/dockerfile_asn @@ -0,0 +1,11 @@ +# docker build -t mcanterb/asn1test +# docker push mcanterb/asn1test + +FROM python:3.9-slim-bullseye +RUN apt-get update -y && \ + apt-get install -y asn1c && \ + rm -rf /var/lib/apt/lists/* && \ + pip install --no-cache-dir asn1tools pycrate + + + diff --git a/testing/dockerfile_xsd b/testing/dockerfile_xsd new file mode 100644 index 0000000000000000000000000000000000000000..9ceeee360715afa316f1fc9dd060a6c301015cb6 --- /dev/null +++ b/testing/dockerfile_xsd @@ -0,0 +1,10 @@ +# docker build -t mcanterb/xsdtest +# docker push mcanterb/xsdtest + +FROM python:3.9-slim-bullseye +RUN apt-get update -y && \ + rm -rf /var/lib/apt/lists/* && \ + pip install --no-cache-dir lxml xmlschema + + + diff --git a/testing/merge_test.py b/testing/merge_test.py new file mode 100644 index 0000000000000000000000000000000000000000..b7a82b39c4958ea30d4ac9a96e100e3041fe73c0 --- /dev/null +++ b/testing/merge_test.py @@ -0,0 +1,68 @@ +import os +import pprint +import requests +import json +import subprocess + +crCommitBranch = os.environ.get("CI_COMMIT_REF_NAME", "NOTFOUND") +apiUrl = os.environ.get("CI_API_V4_URL", "https://forge.3gpp.org/rep/api/v4") +projectId = os.environ.get("CI_PROJECT_ID", "13") + +def gapi (query): + url = f"{apiUrl}/projects/{projectId}/{query}" + r = requests.get(url) + return json.loads(r.text) + +def do (commandline): + #print (" Attempting: " + commandline) + completedProc = subprocess.run(commandline, capture_output=True, shell=True) + #print (" STDOUT > " + ("empty" if completedProc.stdout is None else completedProc.stdout.decode('utf-8'))) + #print (" STDERR > " + ("empty" if completedProc.stderr is None else completedProc.stderr.decode('utf-8'))) + #print (f" Completed with code {completedProc.returncode}") + return (completedProc.returncode == 0, completedProc.stdout.decode('utf-8')) + +print ("Searching for corresponding MR...") + +mrs = gapi(f"merge_requests?source_branch={crCommitBranch}&state=opened") +if len(mrs) == 0: + print ("No MR found... aborting") + exit() + +if len(mrs) > 1: + print (f"{len(mrs)} MRs found, 1 expected - aborting") + for m in mrs: + pprint.pprint(m) + exit(-1) + +mr = mrs[0] + +print (f"Found MR {mr['reference']} ({mr['title']})") +print (f"Target branch is {mr['target_branch']}") +print ("Searching for open MRs targeting same branch...") + +mrs = gapi(f"merge_requests?target_branch={mr['target_branch']}&state=opened") +mrs = [m for m in mrs if m['reference'] != mr['reference']] +print (f"{len(mrs)} MRs found") + +mergeConflicts = {} + +for mr in mrs: + source_branch = mr['source_branch'] + print (source_branch) + + try: + do(f"git fetch origin {source_branch}:{source_branch}") + success, errStr = do(f"git merge --no-commit {source_branch}") + if not success: + print ("Merge NOT OK") + mergeConflicts[source_branch] = errStr + else: + print ("Merge OK") + except Exception as ex: + mergeConflicts[source_branch] = str(ex) + raise + finally: + do("git merge --abort") + +print (f"Merge conflicts with following branches: {mergeConflicts}") +exit(len(mergeConflicts.keys())) \ No newline at end of file diff --git a/testing/xsd_compile_targets.json b/testing/xsd_compile_targets.json new file mode 100644 index 0000000000000000000000000000000000000000..69ad1b50881635e0b2aa86dd159dc1110217da5e --- /dev/null +++ b/testing/xsd_compile_targets.json @@ -0,0 +1,49 @@ +[ + { + "coreSchema" : "102657/RDMessage.xsd", + "supportingSchemas" : [], + "exampleFiles" : [] + }, + { + "coreSchema" : "103280/TS_103_280.xsd", + "supportingSchemas" : [], + "exampleFiles" : [] + }, + { + "coreSchema" : "103221-1/TS_103_221_01.xsd", + "supportingSchemas" : [ + "103221-1/TS_103_221_01.xsd", + "103221-1/TS_103_221_01_HashedID.xsd", + "103221-1/TS_103_221_01_DestinationSet.xsd", + "103280/TS_103_280.xsd", + "103221-1/examples/ExampleGenericObjects.xsd" + ], + "exampleFiles" : [ + "103221-1/examples" + ] + }, + { + "coreSchema" : "103120/dictionaries/ts_103120_Dictionaries.xsd", + "supportingSchemas" : [], + "exampleFiles" : [ + "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml" + ] + }, + { + "coreSchema" : "103120/schema/ts_103120_Core.xsd", + "supportingSchemas" : [ + "103120/schema/ts_103120_Authorisation.xsd", + "103120/schema/ts_103120_Common.xsd", + "103120/schema/ts_103120_Delivery.xsd", + "103120/schema/ts_103120_Document.xsd", + "103120/schema/ts_103120_Notification.xsd", + "103120/schema/ts_103120_Task.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/FooServiceSchema.xsd" + ], + "exampleFiles" : [ + "103120/examples" + ] + } +] \ No newline at end of file diff --git a/testing/xsd_ignore.txt b/testing/xsd_ignore.txt new file mode 100644 index 0000000000000000000000000000000000000000..90bfe5d59c3b61099ed6685d7ddd6023be91efe5 --- /dev/null +++ b/testing/xsd_ignore.txt @@ -0,0 +1,3 @@ +deps +portal +temp \ No newline at end of file diff --git a/testing/xsd_process.py b/testing/xsd_process.py new file mode 100644 index 0000000000000000000000000000000000000000..a0efd07b5b43db84e9797e965c8e06d4d213e5b9 --- /dev/null +++ b/testing/xsd_process.py @@ -0,0 +1,222 @@ +import json +import logging +from pathlib import Path + +from xmlschema.etree import etree_tostring +from xmlschema import XMLSchema, XMLSchemaParseError + + +def BuildSchemaDictonary (fileList): + if len(fileList) == 0: + logging.info("No schema files provided") + return [] + + logging.info("Schema locations:") + schemaLocations = [] + for schemaFile in fileList: + try: + xs = XMLSchema(schemaFile, validation='skip') + schemaLocations.append((xs.default_namespace, str(Path(schemaFile).resolve()))) + logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, schemaFile)) + except XMLSchemaParseError as ex: + logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) + return schemaLocations + + +def BuildSchema (coreFile, fileList = None): + schemaLocations = [] + if fileList and len(fileList) > 0: + schemaLocations = BuildSchemaDictonary(fileList) + + coreSchema = XMLSchema(str(Path(coreFile)), locations=schemaLocations) + return coreSchema + + +def ValidateXSDFiles (fileList): + if len(fileList) == 0: + logging.info("No schema files provided") + return {} + + schemaLocations = BuildSchemaDictonary(fileList) + errors = {} + + logging.info("Schema validation:") + for schemaFile in fileList: + try: + schema = XMLSchema(schemaFile, locations = schemaLocations, validation="lax") + logging.info(schemaFile + ": OK") + errors[schemaFile] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in schema.all_errors] + except XMLSchemaParseError as ex: + logging.warning(schemaFile + ": Failed validation ({0})".format(ex.message)) + if (ex.schema_url) and (ex.schema_url != ex.origin_url): + logging.warning(" Error comes from {0}, suppressing".format(ex.schema_url)) + errors[schemaFile] = [] + else: + errors[schemaFile] = [ex] + return errors + + +def ValidateAllXSDFilesInPath (path): + schemaGlob = [str(f) for f in Path(path).rglob("*.xsd")] + return ValidateXSDFiles(schemaGlob) + + +def ValidateInstanceDocuments (coreFile, supportingSchemas, instanceDocs): + if (instanceDocs is None) or len(instanceDocs) == 0: + logging.warning ("No instance documents provided") + return [] + + schema = BuildSchema(coreFile, supportingSchemas) + errors = [] + for instanceDoc in instanceDocs: + try: + schema.validate(instanceDoc) + logging.info ("{0} passed validation".format(instanceDoc)) + except Exception as ex: + logging.error ("{0} failed validation: {1}".format(instanceDoc, ex)) + return errors + + +def processResults (results, stageName): + """ + Counts the number of errors and writes out the output per filename + + :param results: List of filenames (str or Pathlib Path) + :param stageName: Name to decorate the output with + :returns: The number of files which had errors + """ + print("") + errorCount = sum([1 for r in results.values() if not r['ok']]) + logging.info(f"{errorCount} {stageName} errors encountered") + + print(f"{'-':-<60}") + print(f"{stageName} results:") + print(f"{'-':-<60}") + for filename, result in results.items(): + print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") + if not result['ok']: + if isinstance(result['message'], list): + for thing in result['message']: + print(f" {thing['message']}") + else: + print(f" {result['message']}") + + print(f"{'-':-<60}") + print(f"{stageName} errors: {errorCount}") + print(f"{'-':-<60}") + + return errorCount + + +def syntaxCheckXSD (fileList): + results = {} + for file in fileList: + try: + logging.info(f"Syntax checking {str(file)}") + + schema = XMLSchema(str(file), validation="skip") + results[str(file)] = { + 'ok' : len(schema.all_errors) == 0, + 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] + } + except XMLSchemaParseError as ex: + logging.warning(str(file) + ": Failed validation ({0})".format(ex.message)) + results[str(file)] = { + 'ok' : False, + 'message' : f"{ex!r}" + } + return results + + +if __name__ == '__main__': + #logging.basicConfig(level=logging.DEBUG) + + compileTargets = json.loads(Path('testing/xsd_compile_targets.json').read_text()) + results = {} + for target in compileTargets: + coreFile = target['coreSchema'] + logging.info(f"Attempting to compile {coreFile}") + schemaLocations = [] + for supportSchema in target['supportingSchemas']: + logging.debug(f"Adding supporting schema {supportSchema}") + try: + xs = XMLSchema(supportSchema, validation='skip') + schemaLocations.append((xs.target_namespace, str(Path(supportSchema).resolve()))) + logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, supportSchema)) + except Exception as ex: + logging.warning (" [ {0} exception parsing: {1} ]".format(supportSchema, ex)) + results[coreFile] = { + 'ok' : False, + 'message' : f"{ex!r}" + } + break + try: + schema = XMLSchema(coreFile, locations = schemaLocations, validation="strict") + results[coreFile] = { + 'ok' : len(schema.all_errors) == 0, + 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] + } + target["schemaInstance"] = schema + except Exception as ex: + results[coreFile] = { + 'ok' : False, + 'message' : f"{ex!r}" + } + continue + + if (processResults(results, "Compile") > 0): + exit(-1) + + results = {} + + for target in compileTargets: + schema = target["schemaInstance"] + testResults = {} + failureCount = 0 + logging.info (f"Validating example {len(target['exampleFiles'])} entries for {target['coreSchema']}") + for example in target["exampleFiles"]: + examplePath = Path(example) + if examplePath.is_dir(): + logging.debug (f"Expanding {str(examplePath)}") + testFiles = list(examplePath.rglob("./*.xml")) + else: + testFiles = [examplePath] + logging.debug(f"Found {len(testFiles)} test files") + for test in testFiles: + logging.debug(f"Validating {str(test)} against schema") + try: + errors = list(schema.iter_errors(str(test))) + testResults[test] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in errors] + failureCount += len(errors) + except Exception as ex: + testResults[test] = [f"{ex!r}"] + failureCount += 1 + results[target['coreSchema']] = { + 'ok' : failureCount == 0, + 'testResults' : testResults, + 'failureCount' : failureCount + } + + print(f"{'-':-<75}") + print(f"Validation results:") + print(f"{'-':-<75}") + + totalErrors = 0 + for filename, result in results.items(): + if len(result['testResults']) == 0: + print (f"{filename:.<70}SKIP (0)") + continue + else: + print (f"{filename:.<70}{'..OK' if result['ok'] else 'FAIL'} ({len(result['testResults'])})") + totalErrors += result['failureCount'] + if result['failureCount'] > 0: + for testFile, testResult in result['testResults'].items(): + print(f" {str(testFile):.<65}{'..OK' if len(testResult) == 0 else 'FAIL'}") + for tr in testResult: + print(f" {tr}") + + print(f"{'-':-<75}") + print(f"Validation errors: {totalErrors}") + print(f"{'-':-<75}") + + exit(totalErrors > 0)