From e123cba06a6afddbe71fdf412ca2b47bd751eeff Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 23 Mar 2023 13:33:32 +0000 Subject: [PATCH 01/10] Initial thoughts --- 103705/examples/csp_a.schema.json | 65 ++++ 103705/examples/csp_a_results.json | 19 ++ 103705/examples/csp_b.schema.json | 98 ++++++ 103705/examples/csp_b_results.json | 28 ++ 103705/schema/280_stub.schema.json | 387 ++++++++++++++++++++++++ 103705/schema/etsi_entities.schema.json | 77 +++++ 103705/schema/response.schema.json | 73 +++++ 103705/validate.py | 93 ++++++ 8 files changed, 840 insertions(+) create mode 100644 103705/examples/csp_a.schema.json create mode 100644 103705/examples/csp_a_results.json create mode 100644 103705/examples/csp_b.schema.json create mode 100644 103705/examples/csp_b_results.json create mode 100644 103705/schema/280_stub.schema.json create mode 100644 103705/schema/etsi_entities.schema.json create mode 100644 103705/schema/response.schema.json create mode 100644 103705/validate.py diff --git a/103705/examples/csp_a.schema.json b/103705/examples/csp_a.schema.json new file mode 100644 index 0000000..d278559 --- /dev/null +++ b/103705/examples/csp_a.schema.json @@ -0,0 +1,65 @@ +{ + "$id": "http://example.com/csp_a/response", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Example CSP schema A for ETSI TS 103 705 response", + "description": "Example extended schema for CSP A. This CSP only ever has one person associated with a subscription - the account holder - and so composes a Subscription record from the ETSI-standard Subscription and PersonDetails entities", + "allOf": [ + { + "$ref": "http://etsi.org/temp/705" + }, + { + "type": "object", + "properties": { + "recordSet": { + "type": "array", + "items": { + "$ref": "#/$defs/definedRecords" + } + } + }, + "required": [ + "recordSet" + ] + } + ], + "$defs": { + "definedRecords": { + "title" : "Defined Records", + "description" : "Sets out what the valid set of Record types is for this CSP. It is expressed as a 'oneOf' list, each of which combines a type identifier and a reference to the type that is used to validate the record", + "oneOf": [ + { + "allOf": [ + { + "type": "object", + "title" : "Subscriber record", + "description" : "Uses the ETSI-standard subscriber record (the first $ref), extended to include person details (the second $ref)", + "properties": { + "type": { + "const": "Subscription" + } + } + }, + { + "description" : "Uses the basic ETSI-standard subscriber record fields as a base", + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + { + "description" : "Extends the basic ETSI standard subscriber record according to the definition in extendedSubscriberRecord", + "$ref": "#/$defs/extendedSubscriberRecord" + } + ] + } + ] + }, + "extendedSubscriberRecord" : { + "title" : "Extended Subscriber Record", + "description" : "Here CSP A can describe all the extra fields that they want to put in their subscriber record. In this case, they just add the details of the account holder using the ETSI-standard PersonDetails type", + "properties" : { + "title" : "Account Holder", + "description" : "Name and date of birth of the account holder for the description", + "accountHolder" : { "$ref" : "http://etsi.org/temp/705/entities#/$defs/PersonDetails"} + }, + "required" : ["personDetails"] + } + } +} \ No newline at end of file diff --git a/103705/examples/csp_a_results.json b/103705/examples/csp_a_results.json new file mode 100644 index 0000000..9f91108 --- /dev/null +++ b/103705/examples/csp_a_results.json @@ -0,0 +1,19 @@ +{ + "recordSetDescription" : { + "typesUsed" : { + "type1" : "uri://example.com/schema1.schema.json", + "type2" : "uri://example.com/schema2.schema.json" + } + }, + "recordSet" : [ + { + "id" : "1e997322-b813-437b-b2e2-dae732f0cf7f", + "type" : "Subscription", + "subscriptionId" : "{unique CSP subscription ID}", + "personDetails" : { + "lastName" : "Last", + "firstName" : "First" + } + } + ] +} \ No newline at end of file diff --git a/103705/examples/csp_b.schema.json b/103705/examples/csp_b.schema.json new file mode 100644 index 0000000..1e9be64 --- /dev/null +++ b/103705/examples/csp_b.schema.json @@ -0,0 +1,98 @@ +{ + "$id": "http://example.com/csp_a/response", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Example CSP schema A for ETSI TS 103 705 response", + "description": "Example extended schema for CSP B. This CSP has multiple different types of person associated with a subscription (account holder and bill payer); since these are often the same person, they want to be able to normalise this properly. They do this using a Pointer.", + "allOf": [ + { + "$ref": "http://etsi.org/temp/705" + }, + { + "type": "object", + "properties": { + "recordSet": { + "type": "array", + "items": { + "$ref": "#/$defs/definedRecords" + } + } + }, + "required": [ + "recordSet" + ] + } + ], + "$defs": { + "definedRecords": { + "title": "Defined Records", + "description": "Sets out what the valid set of Record types is for this CSP. It is expressed as a 'oneOf' list, each of which combines a type identifier and a reference to the type that is used to validate the record", + "oneOf": [ + { + "allOf": [ + { + "type": "object", + "title": "Subscriber record", + "description": "Uses the ETSI-standard subscriber record, extended to include a pointer to a Person record", + "properties": { + "type": { + "const": "Subscription" + } + } + }, + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + { + "$ref": "#/$defs/extendedSubscriberRecord" + } + ] + }, + { + "allOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "Person" + } + } + }, + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/PersonDetails" + } + ] + } + ] + }, + "extendedSubscriberRecord": { + "properties": { + "associatedPeople": { + "title" : "Associated People", + "description" : "A list of people associated with the subscription, together with their relationship", + "type" : "array", + "items" : { + "allOf": [ + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" + }, + { + "type": "object", + "properties": { + "pointerType": { + "enum": [ + "Account Holder", + "Bill Payer" + ] + } + } + } + ] + } + } + }, + "required": [ + "associatedPeople" + ] + } + } +} \ No newline at end of file diff --git a/103705/examples/csp_b_results.json b/103705/examples/csp_b_results.json new file mode 100644 index 0000000..454da56 --- /dev/null +++ b/103705/examples/csp_b_results.json @@ -0,0 +1,28 @@ +{ + "recordSetDescription" : { + "typesUsed" : { + "type1" : "uri://example.com/schema1.schema.json", + "type2" : "uri://example.com/schema2.schema.json" + } + }, + "recordSet" : [ + { + "id" : "Record1", + "type" : "Subscription", + "subscriptionId" : "{CSP B subscription identifier}", + "associatedPeople" : [{ + "destination" : "Record2", + "pointerType" : "Account Holder" + },{ + "destination" : "Record2", + "pointerType" : "Bill Payer" + }] + }, + { + "id" : "Record2", + "type" : "Person", + "firstName" : "Bill", + "lastName" : "Payer" + } + ] +} \ No newline at end of file diff --git a/103705/schema/280_stub.schema.json b/103705/schema/280_stub.schema.json new file mode 100644 index 0000000..523a1a5 --- /dev/null +++ b/103705/schema/280_stub.schema.json @@ -0,0 +1,387 @@ +{ + "$id" : "http://etsi.org/temp/280", + "$schema" : "https://json-schema.org/draft/2020-12/schema", + "title" : "ETSI TS 103 280 (temp)", + "description" : "A stubbed version of TS 103 280 definitions (need a CR to 280 for this)", + "$defs": { + "ShortString": { + "type": "string", + "maxLength": 255 + }, + "LongString": { + "type": "string", + "maxLength": 65535 + }, + "LIID": { + "type": "string", + "pattern": "^([!-~]{1,25})|([0-9a-f]{26,50})$" + }, + "UTCDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "UTCMicrosecondDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}Z$" + }, + "QualifiedDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|[+-][0-9]{2}:[0-9]{2})$" + }, + "QualifiedMicrosecondDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}(Z|[+-][0-9]{2}:[0-9]{2})$" + }, + "InternationalE164": { + "type": "string", + "pattern": "^[0-9]{1,15}$" + }, + "IMSI": { + "type": "string", + "pattern": "^[0-9]{6,15}$" + }, + "IMEI": { + "type": "string", + "pattern": "^[0-9]{14}$" + }, + "IMEICheckDigit": { + "type": "string", + "pattern": "^[0-9]{15}$" + }, + "IMEISV": { + "type": "string", + "pattern": "^[0-9]{16}$" + }, + "IPv4Address": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])$" + }, + "IPv4CIDR": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/([1-2]?[0-9]|3[0-2])$" + }, + "IPv6Address": { + "type": "string", + "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})$" + }, + "IPv6CIDR": { + "type": "string", + "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})/(([1-9][0-9]?)|(1[0-1][0-9])|(12[0-8]))$" + }, + "TCPPort": { + "type": "integer", + "exclusiveMinimum": 1, + "maximum": 65535 + }, + "UDPPort": { + "type": "integer", + "minimum": 0, + "maximum": 65535 + }, + "MACAddress": { + "type": "string", + "pattern": "^([a-f0-9]{2}:){5}[a-f0-9]{2}$" + }, + "EmailAddress": { + "allOf": [ + { + "$ref": "#/$defs/ShortString" + }, + { + "type": "string", + "pattern": "^[a-zA-Z0-9\\.!#$%&'\\*\\+\\\\/=\\?\\^_`\\{\\|\\}~\\-]+@[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" + } + ] + }, + "UUID": { + "type": "string", + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + }, + "ISOCountryCode": { + "type": "string", + "pattern": "^[A-Z]{2}$" + }, + "SIPURI": { + "type": "string", + "pattern": "^sips?:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" + }, + "TELURI": { + "type": "string", + "pattern": "^tel:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" + }, + "WGS84LatitudeDecimal": { + "type": "string", + "pattern": "^[NS][0-9]{2}\\.[0-9]{6}$" + }, + "WGS84LongitudeDecimal": { + "type": "string", + "pattern": "^[EW][0-9]{3}\\.[0-9]{6}$" + }, + "WGS84LatitudeAngular": { + "type": "string", + "pattern": "^[NS][0-9]{6}\\.[0-9]{2}$" + }, + "WGS84LongitudeAngular": { + "type": "string", + "pattern": "^[EW][0-9]{7}\\.[0-9]{2}$" + }, + "SUPIIMSI": { + "$ref": "#/$defs/IMSI" + }, + "SUPINAI": { + "$ref": "#/$defs/NAI" + }, + "SUCI": { + "type": "string", + "pattern": "^([a-fA-F0-9]{2})*$" + }, + "PEIIMEI": { + "$ref": "#/$defs/IMEI" + }, + "PEIIMEICheckDigit": { + "$ref": "#/$defs/IMEICheckDigit" + }, + "PEIIMEISV": { + "$ref": "#/$defs/IMEISV" + }, + "GPSIMSISDN": { + "type": "string", + "pattern": "^[0-9]{1,15}$" + }, + "GPSINAI": { + "$ref": "#/$defs/NAI" + }, + "NAI": { + "type": "string" + }, + "LDID": { + "type": "string", + "pattern": "^([A-Z]{2}-.+-.+)$" + }, + "InternationalizedEmailAddress": { + "allOf": [ + { + "$ref": "#/$defs/ShortString" + }, + { + "type": "string", + "pattern": "^.+@.+$" + } + ] + }, + "EUI64": { + "type": "string", + "pattern": "^([a-f0-9]{2}:){7}[a-f0-9]{2}$" + }, + "CGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{4}-[a-f0-9]{4}$" + }, + "ECGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{7}$" + }, + "NCGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{9}$" + }, + "ICCID": { + "type": "string", + "pattern": "^[0-9]{19,20}$" + }, + "IPAddress": { + "oneOf": [ + { + "type": "object", + "properties": { + "IPv4Address": { + "$ref": "#/$defs/IPv4Address" + } + }, + "required": [ + "IPv4Address" + ] + }, + { + "type": "object", + "properties": { + "IPv6Address": { + "$ref": "#/$defs/IPv6Address" + } + }, + "required": [ + "IPv6Address" + ] + } + ] + }, + "IPCIDR": { + "oneOf": [ + { + "type": "object", + "properties": { + "IPv4CIDR": { + "$ref": "#/$defs/IPv4CIDR" + } + }, + "required": [ + "IPv4CIDR" + ] + }, + { + "type": "object", + "properties": { + "IPv6CIDR": { + "$ref": "#/$defs/IPv6CIDR" + } + }, + "required": [ + "IPv6CIDR" + ] + } + ] + }, + "TCPPortRange": { + "type": "object", + "properties": { + "start": { + "$ref": "#/$defs/TCPPort" + }, + "end": { + "$ref": "#/$defs/TCPPort" + } + }, + "required": [ + "start", + "end" + ] + }, + "UDPPortRange": { + "type": "object", + "properties": { + "start": { + "$ref": "#/$defs/UDPPort" + }, + "end": { + "$ref": "#/$defs/UDPPort" + } + }, + "required": [ + "start", + "end" + ] + }, + "Port": { + "oneOf": [ + { + "type": "object", + "properties": { + "TCPPort": { + "$ref": "#/$defs/TCPPort" + } + }, + "required": [ + "TCPPort" + ] + }, + { + "type": "object", + "properties": { + "UDPPort": { + "$ref": "#/$defs/UDPPort" + } + }, + "required": [ + "UDPPort" + ] + } + ] + }, + "PortRange": { + "oneOf": [ + { + "type": "object", + "properties": { + "TCPPortRange": { + "$ref": "#/$defs/TCPPortRange" + } + }, + "required": [ + "TCPPortRange" + ] + }, + { + "type": "object", + "properties": { + "UDPPortRange": { + "$ref": "#/$defs/UDPPortRange" + } + }, + "required": [ + "UDPPortRange" + ] + } + ] + }, + "IPAddressPort": { + "type": "object", + "properties": { + "address": { + "$ref": "#/$defs/IPAddress" + }, + "port": { + "$ref": "#/$defs/Port" + } + }, + "required": [ + "address", + "port" + ] + }, + "IPAddressPortRange": { + "type": "object", + "properties": { + "address": { + "$ref": "#/$defs/IPAddress" + }, + "portRange": { + "$ref": "#/$defs/PortRange" + } + }, + "required": [ + "address", + "portRange" + ] + }, + "WGS84CoordinateDecimal": { + "type": "object", + "properties": { + "latitude": { + "$ref": "#/$defs/WGS84LatitudeDecimal" + }, + "longitude": { + "$ref": "#/$defs/WGS84LongitudeDecimal" + } + }, + "required": [ + "latitude", + "longitude" + ] + }, + "WGS84CoordinateAngular": { + "type": "object", + "properties": { + "latitude": { + "$ref": "#/$defs/WGS84LatitudeAngular" + }, + "longitude": { + "$ref": "#/$defs/WGS84LongitudeAngular" + } + }, + "required": [ + "latitude", + "longitude" + ] + } + } +} \ No newline at end of file diff --git a/103705/schema/etsi_entities.schema.json b/103705/schema/etsi_entities.schema.json new file mode 100644 index 0000000..9d76005 --- /dev/null +++ b/103705/schema/etsi_entities.schema.json @@ -0,0 +1,77 @@ +{ + "$id": "http://etsi.org/temp/705/entities", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ETSI TS 103 705", + "description": "JSON schema for ETSI-defined entities (see Annex A)", + "$defs": { + "Pointer" : { + "type" : "object", + "title" : "Pointer", + "description" : "Allows one Record in a RecordSet to point to data in another Record in the same Recordset. See clause 5.4 and Annex A.2.2", + "properties" : { + "destination" : { + "type" : "string", + "title" : "Pointer Destination", + "description" : "Identifies the Record in the RecordSet containined the referred-to data (see clause 5.3.2 and A.2.2)" + }, + "pointerType" : { + "type" : "string", + "title" : "Pointer type", + "Description" : "Indicates the nature of the relationship. Valid values should be indicated by the CSP schema" + }, + "required" : ["destination"] + } + }, + "Subscription": { + "type": "object", + "title": "Subscription", + "description": "Basic subscription record", + "properties": { + "subscriptionId": { + "title" : "Subscription Identifier", + "description" : "Unique identifier for a subscription within the CSP", + "type": "string" + }, + "startDate": { + "title" : "Subscription Start Date", + "description" : "Date that the subscription started", + "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + }, + "endDate": { + "title" : "Subscription End Date", + "description" : "Date that the subscription ended", + "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + } + }, + "required" : ["subscriptionId"] + }, + "PersonDetails": { + "type": "object", + "title": "Person Details", + "description": "Details about a person", + "properties": { + "firstName": { + "title" : "First Name", + "description" : "First name or given name", + "type": "string" + }, + "middeNames": { + "title" : "First Name", + "description" : "Middle names, given as a space-delimited string", + "type": "string" + }, + "lastName" : { + "title" : "Last Name", + "description" : "Lastname or family name", + "type" : "string" + }, + "birthDate" : { + "title" : "Date of Birth", + "description" : "Date of birth (time part can be ignored)", + "$ref" : "http://etsi.org/temp/280#/$defs/DateTime" + } + }, + "required" : ["lastName"] + } + } +} \ No newline at end of file diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json new file mode 100644 index 0000000..4bfa29f --- /dev/null +++ b/103705/schema/response.schema.json @@ -0,0 +1,73 @@ +{ + "$id": "http://etsi.org/temp/705", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ETSI TS 103 705 Response", + "description": "Overall Response structure, representing the results of a lawful disclosure query (see clause 5)", + "type:": "object", + "properties": { + "recordSetDescription": { + "$ref": "#/$defs/recordSetDescription" + }, + "recordSet": { + "$ref": "#/$defs/recordSet" + } + }, + "required": [ + "recordSetDescription", + "recordSet" + ], + "$defs": { + "recordSetDescription": { + "type": "object", + "title": "RecordSetDescription", + "description" : "Provides metadata about the records being delivered (see clause 5.2)", + "properties": { + "typesUsed": { + + } + }, + "required": [ + "typesUsed" + ] + }, + "typesUsed": { + "type": "object", + "title": "TypesUsed", + "description" : "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", + "patternProperties": { + "": { + "$ref": "#/$defs/schemaReference" + } + } + }, + "schemaReference": { + "type": "string" + }, + "recordSet": { + "type": "array", + "title": "RecordSet", + "description" : "RecordSet type (see clause 5.3.1). Contains a set of Records.", + "items": { + "$ref": "#$defs/record" + } + }, + "record": { + "type": "object", + "title": "Record", + "description" : "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", + "properties": { + "id": { + "title" : "id", + "description" : "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", + "type": "string" + }, + "type": { + "title" : "type", + "description" : "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", + "type": "string" + } + }, + "required": ["id", "type"] + } + } +} \ No newline at end of file diff --git a/103705/validate.py b/103705/validate.py new file mode 100644 index 0000000..a46be2d --- /dev/null +++ b/103705/validate.py @@ -0,0 +1,93 @@ +import sys +from jsonschema import validate, RefResolver, Draft202012Validator +import json +from pathlib import Path +import logging +import argparse + + + +# filename = sys.argv[1] + +# def load_json (path): +# with open(path) as f: +# s = json.load(f) +# return s + +# schema_store = {} + +# json_instance = load_json(filename) +# print (json_instance) + +# etsi_schema = load_json('response.schema.json') +# ext_schema = load_json('extended.schema.json') +# ext_ent_schema = load_json("extended_entities.schema.json") +# schema_store = { +# etsi_schema['$id'] : etsi_schema, +# ext_schema['$id'] : ext_schema, +# ext_ent_schema['$id'] : ext_ent_schema +# } + +# resolver = RefResolver(None, referrer=None, store=schema_store) + +# print (etsi_schema) + +# v = Draft202012Validator(ext_schema, resolver=resolver) +# v.validate(json_instance) + +# validate(json_instance, ext_schema) +# print ("OK") + +def handle_uri(u): + print(u) + +def load_json(path : str): + with open(path) as f: + return json.load(f) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") + parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('schema', help="Primary schema to validate against") + parser.add_argument('filename', help="JSON instance document to validate") + + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + instance_doc = load_json(args.filename) + main_schema = load_json(args.schema) + schema_dict = { main_schema['$id'] : main_schema } + + if args.schemadir: + schema_paths = [] + for d in args.schemadir: + schema_paths += [f for f in Path(d).rglob("*.schema.json")] + logging.info(f"Schema files loaded: {schema_paths}") + + schemas_json = [json.load(p.open()) for p in schema_paths] + schema_dict = schema_dict | { s['$id'] : s for s in schemas_json } + + logging.info(f"Schema IDs loaded: {[k for k in schema_dict.keys()]}") + + logging.debug (f"Instance doc: {instance_doc}") + logging.debug (f"Main schema: {main_schema}") + + resolver = RefResolver(None, + referrer=None, + store=schema_dict) + + v = Draft202012Validator(main_schema, resolver=resolver) + v.validate(instance_doc) + + logging.info("Done") \ No newline at end of file -- GitLab From 8617694513b30d5e385d98eeac15f60c78ce80a3 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 23 Mar 2023 15:21:50 +0000 Subject: [PATCH 02/10] Updating typesUsed --- 103705/examples/csp_a_results.json | 3 +-- 103705/examples/csp_b_results.json | 4 ++-- 103705/schema/response.schema.json | 28 ++++++++++++++++------------ 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/103705/examples/csp_a_results.json b/103705/examples/csp_a_results.json index 9f91108..3081464 100644 --- a/103705/examples/csp_a_results.json +++ b/103705/examples/csp_a_results.json @@ -1,8 +1,7 @@ { "recordSetDescription" : { "typesUsed" : { - "type1" : "uri://example.com/schema1.schema.json", - "type2" : "uri://example.com/schema2.schema.json" + "Subscription" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord" } }, "recordSet" : [ diff --git a/103705/examples/csp_b_results.json b/103705/examples/csp_b_results.json index 454da56..812ed58 100644 --- a/103705/examples/csp_b_results.json +++ b/103705/examples/csp_b_results.json @@ -1,8 +1,8 @@ { "recordSetDescription" : { "typesUsed" : { - "type1" : "uri://example.com/schema1.schema.json", - "type2" : "uri://example.com/schema2.schema.json" + "Subscriber" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord", + "Person" : "http://etsi.org/temp/705/entities/$defs/PersonDetails" } }, "recordSet" : [ diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json index 4bfa29f..9ae7fb1 100644 --- a/103705/schema/response.schema.json +++ b/103705/schema/response.schema.json @@ -20,11 +20,12 @@ "recordSetDescription": { "type": "object", "title": "RecordSetDescription", - "description" : "Provides metadata about the records being delivered (see clause 5.2)", + "description": "Provides metadata about the records being delivered (see clause 5.2)", "properties": { - "typesUsed": { - - } + "resultSetId" : { "$ref" : "#/$defs/resultSetId"}, + "version" : {"$ref" : "#/defs/version"}, + "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"}, + "typesUsed": { "$ref" : "#/$defs/typesUsed"} }, "required": [ "typesUsed" @@ -33,7 +34,7 @@ "typesUsed": { "type": "object", "title": "TypesUsed", - "description" : "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", + "description": "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", "patternProperties": { "": { "$ref": "#/$defs/schemaReference" @@ -46,7 +47,7 @@ "recordSet": { "type": "array", "title": "RecordSet", - "description" : "RecordSet type (see clause 5.3.1). Contains a set of Records.", + "description": "RecordSet type (see clause 5.3.1). Contains a set of Records.", "items": { "$ref": "#$defs/record" } @@ -54,20 +55,23 @@ "record": { "type": "object", "title": "Record", - "description" : "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", + "description": "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", "properties": { "id": { - "title" : "id", - "description" : "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", + "title": "id", + "description": "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", "type": "string" }, "type": { - "title" : "type", - "description" : "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", + "title": "type", + "description": "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", "type": "string" } }, - "required": ["id", "type"] + "required": [ + "id", + "type" + ] } } } \ No newline at end of file -- GitLab From b360801c9211c113d79d43bc6d6095ed4c66fe75 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 18 May 2023 10:48:45 +0100 Subject: [PATCH 03/10] Tweaks --- 103705/examples/csp_a.schema.json | 1 - 103705/generate_docs.py | 2 ++ 103705/metaschema/etsi_response.schema.json | 12 ++++++++++++ 103705/validate.py | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 103705/generate_docs.py create mode 100644 103705/metaschema/etsi_response.schema.json diff --git a/103705/examples/csp_a.schema.json b/103705/examples/csp_a.schema.json index d278559..b6fe543 100644 --- a/103705/examples/csp_a.schema.json +++ b/103705/examples/csp_a.schema.json @@ -1,7 +1,6 @@ { "$id": "http://example.com/csp_a/response", "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Example CSP schema A for ETSI TS 103 705 response", "description": "Example extended schema for CSP A. This CSP only ever has one person associated with a subscription - the account holder - and so composes a Subscription record from the ETSI-standard Subscription and PersonDetails entities", "allOf": [ { diff --git a/103705/generate_docs.py b/103705/generate_docs.py new file mode 100644 index 0000000..8837e1f --- /dev/null +++ b/103705/generate_docs.py @@ -0,0 +1,2 @@ +import json +from pathlib import Path diff --git a/103705/metaschema/etsi_response.schema.json b/103705/metaschema/etsi_response.schema.json new file mode 100644 index 0000000..fe23cb6 --- /dev/null +++ b/103705/metaschema/etsi_response.schema.json @@ -0,0 +1,12 @@ +{ + "$id": "http://etsi.org/temp/705_metaschema", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ETSI TS 103 705 Response Metaschema", + "description": "Metaschema for checking that a CSP's Response Schema is per-spec", + "type:": "object", + "properties" : { + "title" : { "type" : "string" }, + "description" : { "type" : "string" } + }, + "required": ["title", "description"] +} \ No newline at end of file diff --git a/103705/validate.py b/103705/validate.py index a46be2d..a290ed3 100644 --- a/103705/validate.py +++ b/103705/validate.py @@ -88,6 +88,7 @@ if __name__ == "__main__": store=schema_dict) v = Draft202012Validator(main_schema, resolver=resolver) + v.validate(instance_doc) logging.info("Done") \ No newline at end of file -- GitLab From 6e5100a2754c8f5a88e64c45c3d43ae7824bfb62 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 8 Jun 2023 09:39:37 +0100 Subject: [PATCH 04/10] Latest changes --- 103705/examples/example_csp_core_schema.json | 79 ++++++++++++++++++++ 103705/schema/response.schema.json | 34 +++------ 2 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 103705/examples/example_csp_core_schema.json diff --git a/103705/examples/example_csp_core_schema.json b/103705/examples/example_csp_core_schema.json new file mode 100644 index 0000000..e3241ce --- /dev/null +++ b/103705/examples/example_csp_core_schema.json @@ -0,0 +1,79 @@ +{ + "$id": "http://example.com/csp/1.1.1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Example CSP schema demonstrating how the requirements of TS 103 705 clause 6.5.3 can be met", + "description": "The top level of the schema is an allOf union between the ETSI Response Schema (meeting the first requirement) and a custom definition that sets out the permisslbe record types (meeting the second requirement)", + "allOf": [ + { + "$ref": "http://etsi.org/temp/705" + }, + { + "type": "object", + "properties": { + "recordSet": { + "type": "array", + "items": { + "$ref": "#/$defs/definedRecords" + } + } + }, + "required": [ + "recordSet" + ] + } + ], + "$defs": { + "definedRecords": { + "title": "Defined Records", + "description": "This oneOf alternation effectively sets what the valid Record types are for this CSP schema. The previous allOf union ensures that each Record must also be a valid ETSI record as per TS 103 705 clause 5.3", + "oneOf": [ + { + "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + { + "description" : "This means that a custom extended subcription record is a valid choice of record for this schema - the definition is given in the $defs section below", + "$ref": "#/$defs/extendedSubscriberRecord" + } + ] + } + }, + "extendedSubscriberRecord": { + "allOf" : { + { + "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + + } + "properties": { + "associatedPeople": { + "title" : "Associated People", + "description" : "A list of people associated with the subscription, together with their relationship", + "type" : "array", + "items" : { + "allOf": [ + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" + }, + { + "type": "object", + "properties": { + "pointerType": { + "enum": [ + "Account Holder", + "Bill Payer" + ] + } + } + } + ] + } + } + }, + "required": [ + "associatedPeople" + ] + } + } +} \ No newline at end of file diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json index 9ae7fb1..95bb81d 100644 --- a/103705/schema/response.schema.json +++ b/103705/schema/response.schema.json @@ -22,28 +22,24 @@ "title": "RecordSetDescription", "description": "Provides metadata about the records being delivered (see clause 5.2)", "properties": { + "schemaId" : { "$ref" : "#/$defs/schemaId" }, "resultSetId" : { "$ref" : "#/$defs/resultSetId"}, - "version" : {"$ref" : "#/defs/version"}, - "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"}, - "typesUsed": { "$ref" : "#/$defs/typesUsed"} + "queryReference" : { "type" : "#/$defs/queryReference" }, + "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"} }, "required": [ - "typesUsed" + "schemaId", "resultSetId", "queryReference", "created" ] }, - "typesUsed": { - "type": "object", - "title": "TypesUsed", - "description": "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", - "patternProperties": { - "": { - "$ref": "#/$defs/schemaReference" - } - } - }, - "schemaReference": { + "schemaId": { "type": "string" }, + "resultSetId": { + "type": "string" + }, + "queryReference": { + "type": "string" + }, "recordSet": { "type": "array", "title": "RecordSet", @@ -61,16 +57,10 @@ "title": "id", "description": "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", "type": "string" - }, - "type": { - "title": "type", - "description": "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", - "type": "string" } }, "required": [ - "id", - "type" + "id" ] } } -- GitLab From 9858681dff53fe4c17c10beee5799d32c277614e Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 8 Jun 2023 11:57:53 +0100 Subject: [PATCH 05/10] Adding per-record validator --- 103280/TS_103_280.schema.json | 389 ++++++++++++++++++ 103705/examples/csp_a.schema.json | 2 +- 103705/examples/csp_a_results.json | 8 +- 103705/examples/csp_b.schema.json | 98 ----- 103705/examples/csp_b_results.json | 28 -- 103705/examples/example_csp_core_schema.json | 79 ---- 103705/metaschema/etsi_response.schema.json | 12 - 103705/schema/280_stub.schema.json | 387 ----------------- ...ies.schema.json => etsi_types.schema.json} | 37 +- 103705/schema/response.schema.json | 24 +- 103705/validate.py | 5 +- 103705/validation/validate_705.py | 97 +++++ 12 files changed, 534 insertions(+), 632 deletions(-) create mode 100644 103280/TS_103_280.schema.json delete mode 100644 103705/examples/csp_b.schema.json delete mode 100644 103705/examples/csp_b_results.json delete mode 100644 103705/examples/example_csp_core_schema.json delete mode 100644 103705/metaschema/etsi_response.schema.json delete mode 100644 103705/schema/280_stub.schema.json rename 103705/schema/{etsi_entities.schema.json => etsi_types.schema.json} (68%) create mode 100644 103705/validation/validate_705.py diff --git a/103280/TS_103_280.schema.json b/103280/TS_103_280.schema.json new file mode 100644 index 0000000..f0effdb --- /dev/null +++ b/103280/TS_103_280.schema.json @@ -0,0 +1,389 @@ +{ + "$id": "ts_103280_2017_07", + "$defs": { + "ShortString": { + "type": "string", + "maxLength": 255 + }, + "LongString": { + "type": "string", + "maxLength": 65535 + }, + "LIID": { + "type": "string", + "pattern": "^([!-~]{1,25})|([0-9a-f]{26,50})$" + }, + "UTCDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "UTCMicrosecondDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}Z$" + }, + "QualifiedDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|[+-][0-9]{2}:[0-9]{2})$" + }, + "QualifiedMicrosecondDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}(Z|[+-][0-9]{2}:[0-9]{2})$" + }, + "InternationalE164": { + "type": "string", + "pattern": "^[0-9]{1,15}$" + }, + "IMSI": { + "type": "string", + "pattern": "^[0-9]{6,15}$" + }, + "IMEI": { + "type": "string", + "pattern": "^[0-9]{14}$" + }, + "IMEICheckDigit": { + "type": "string", + "pattern": "^[0-9]{15}$" + }, + "IMEISV": { + "type": "string", + "pattern": "^[0-9]{16}$" + }, + "IPv4Address": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])$" + }, + "IPv4CIDR": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/([1-2]?[0-9]|3[0-2])$" + }, + "IPv6Address": { + "type": "string", + "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})$" + }, + "IPv6CIDR": { + "type": "string", + "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})/(([1-9][0-9]?)|(1[0-1][0-9])|(12[0-8]))$" + }, + "TCPPort": { + "type": "integer", + "exclusiveMinimum": 1, + "maximum": 65535 + }, + "UDPPort": { + "type": "integer", + "minimum": 0, + "maximum": 65535 + }, + "MACAddress": { + "type": "string", + "pattern": "^([a-f0-9]{2}:){5}[a-f0-9]{2}$" + }, + "EmailAddress": { + "allOf": [ + { + "$ref": "#/$defs/ShortString" + }, + { + "type": "string", + "pattern": "^[a-zA-Z0-9\\.!#$%&'\\*\\+\\\\/=\\?\\^_`\\{\\|\\}~\\-]+@[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" + } + ] + }, + "UUID": { + "type": "string", + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + }, + "ISOCountryCode": { + "type": "string", + "pattern": "^[A-Z]{2}$" + }, + "SIPURI": { + "type": "string", + "pattern": "^sips?:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" + }, + "TELURI": { + "type": "string", + "pattern": "^tel:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" + }, + "WGS84LatitudeDecimal": { + "type": "string", + "pattern": "^[NS][0-9]{2}\\.[0-9]{6}$" + }, + "WGS84LongitudeDecimal": { + "type": "string", + "pattern": "^[EW][0-9]{3}\\.[0-9]{6}$" + }, + "WGS84LatitudeAngular": { + "type": "string", + "pattern": "^[NS][0-9]{6}\\.[0-9]{2}$" + }, + "WGS84LongitudeAngular": { + "type": "string", + "pattern": "^[EW][0-9]{7}\\.[0-9]{2}$" + }, + "SUPIIMSI": { + "$ref": "#/$defs/IMSI" + }, + "SUPINAI": { + "$ref": "#/$defs/NAI" + }, + "SUCI": { + "type": "string", + "pattern": "^([a-fA-F0-9]{2})*$" + }, + "PEIIMEI": { + "$ref": "#/$defs/IMEI" + }, + "PEIIMEICheckDigit": { + "$ref": "#/$defs/IMEICheckDigit" + }, + "PEIIMEISV": { + "$ref": "#/$defs/IMEISV" + }, + "GPSIMSISDN": { + "type": "string", + "pattern": "^[0-9]{1,15}$" + }, + "GPSINAI": { + "$ref": "#/$defs/NAI" + }, + "NAI": { + "type": "string" + }, + "LDID": { + "type": "string", + "pattern": "^([A-Z]{2}-.+-.+)$" + }, + "InternationalizedEmailAddress": { + "allOf": [ + { + "$ref": "#/$defs/ShortString" + }, + { + "type": "string", + "pattern": "^.+@.+$" + } + ] + }, + "EUI64": { + "type": "string", + "pattern": "^([a-f0-9]{2}:){7}[a-f0-9]{2}$" + }, + "CGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{4}-[a-f0-9]{4}$" + }, + "ECGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{7}$" + }, + "NCGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{9}$" + }, + "ICCID": { + "type": "string", + "pattern": "^[0-9]{19,20}$" + }, + "IPProtocol": { + "type": "integer", + "minimum": 0, + "maximum": 255 + }, + "IPAddress": { + "oneOf": [ + { + "type": "object", + "properties": { + "etsi280:IPv4Address": { + "$ref": "#/$defs/IPv4Address" + } + }, + "required": [ + "etsi280:IPv4Address" + ] + }, + { + "type": "object", + "properties": { + "etsi280:IPv6Address": { + "$ref": "#/$defs/IPv6Address" + } + }, + "required": [ + "etsi280:IPv6Address" + ] + } + ] + }, + "IPCIDR": { + "oneOf": [ + { + "type": "object", + "properties": { + "etsi280:IPv4CIDR": { + "$ref": "#/$defs/IPv4CIDR" + } + }, + "required": [ + "etsi280:IPv4CIDR" + ] + }, + { + "type": "object", + "properties": { + "etsi280:IPv6CIDR": { + "$ref": "#/$defs/IPv6CIDR" + } + }, + "required": [ + "etsi280:IPv6CIDR" + ] + } + ] + }, + "TCPPortRange": { + "type": "object", + "properties": { + "etsi280:start": { + "$ref": "#/$defs/TCPPort" + }, + "etsi280:end": { + "$ref": "#/$defs/TCPPort" + } + }, + "required": [ + "etsi280:start", + "etsi280:end" + ] + }, + "UDPPortRange": { + "type": "object", + "properties": { + "etsi280:start": { + "$ref": "#/$defs/UDPPort" + }, + "etsi280:end": { + "$ref": "#/$defs/UDPPort" + } + }, + "required": [ + "etsi280:start", + "etsi280:end" + ] + }, + "Port": { + "oneOf": [ + { + "type": "object", + "properties": { + "etsi280:TCPPort": { + "$ref": "#/$defs/TCPPort" + } + }, + "required": [ + "etsi280:TCPPort" + ] + }, + { + "type": "object", + "properties": { + "etsi280:UDPPort": { + "$ref": "#/$defs/UDPPort" + } + }, + "required": [ + "etsi280:UDPPort" + ] + } + ] + }, + "PortRange": { + "oneOf": [ + { + "type": "object", + "properties": { + "etsi280:TCPPortRange": { + "$ref": "#/$defs/TCPPortRange" + } + }, + "required": [ + "etsi280:TCPPortRange" + ] + }, + { + "type": "object", + "properties": { + "etsi280:UDPPortRange": { + "$ref": "#/$defs/UDPPortRange" + } + }, + "required": [ + "etsi280:UDPPortRange" + ] + } + ] + }, + "IPAddressPort": { + "type": "object", + "properties": { + "etsi280:address": { + "$ref": "#/$defs/IPAddress" + }, + "etsi280:port": { + "$ref": "#/$defs/Port" + } + }, + "required": [ + "etsi280:address", + "etsi280:port" + ] + }, + "IPAddressPortRange": { + "type": "object", + "properties": { + "etsi280:address": { + "$ref": "#/$defs/IPAddress" + }, + "etsi280:portRange": { + "$ref": "#/$defs/PortRange" + } + }, + "required": [ + "etsi280:address", + "etsi280:portRange" + ] + }, + "WGS84CoordinateDecimal": { + "type": "object", + "properties": { + "etsi280:latitude": { + "$ref": "#/$defs/WGS84LatitudeDecimal" + }, + "etsi280:longitude": { + "$ref": "#/$defs/WGS84LongitudeDecimal" + } + }, + "required": [ + "etsi280:latitude", + "etsi280:longitude" + ] + }, + "WGS84CoordinateAngular": { + "type": "object", + "properties": { + "etsi280:latitude": { + "$ref": "#/$defs/WGS84LatitudeAngular" + }, + "etsi280:longitude": { + "$ref": "#/$defs/WGS84LongitudeAngular" + } + }, + "required": [ + "etsi280:latitude", + "etsi280:longitude" + ] + } + } + } \ No newline at end of file diff --git a/103705/examples/csp_a.schema.json b/103705/examples/csp_a.schema.json index b6fe543..febb70a 100644 --- a/103705/examples/csp_a.schema.json +++ b/103705/examples/csp_a.schema.json @@ -1,5 +1,5 @@ { - "$id": "http://example.com/csp_a/response", + "$id": "example.com_csp-a-schema_1.2.3", "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Example extended schema for CSP A. This CSP only ever has one person associated with a subscription - the account holder - and so composes a Subscription record from the ETSI-standard Subscription and PersonDetails entities", "allOf": [ diff --git a/103705/examples/csp_a_results.json b/103705/examples/csp_a_results.json index 3081464..e142485 100644 --- a/103705/examples/csp_a_results.json +++ b/103705/examples/csp_a_results.json @@ -1,7 +1,11 @@ { "recordSetDescription" : { - "typesUsed" : { - "Subscription" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord" + "schemaId" : "etsi.org/ts_103_705/1.1.1", + "resultSetId" : "788e190d-fb2e-416e-9798-12a66ebe0a1a", + "queryReference" : "LDID_1", + "created" : "2023-06-08T09:31:56.000000Z", + "recordTypes" : { + "Subscription" : "example.com_csp-a-schema_1.2.3#$defs/extendedSubscriberRecord" } }, "recordSet" : [ diff --git a/103705/examples/csp_b.schema.json b/103705/examples/csp_b.schema.json deleted file mode 100644 index 1e9be64..0000000 --- a/103705/examples/csp_b.schema.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "$id": "http://example.com/csp_a/response", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Example CSP schema A for ETSI TS 103 705 response", - "description": "Example extended schema for CSP B. This CSP has multiple different types of person associated with a subscription (account holder and bill payer); since these are often the same person, they want to be able to normalise this properly. They do this using a Pointer.", - "allOf": [ - { - "$ref": "http://etsi.org/temp/705" - }, - { - "type": "object", - "properties": { - "recordSet": { - "type": "array", - "items": { - "$ref": "#/$defs/definedRecords" - } - } - }, - "required": [ - "recordSet" - ] - } - ], - "$defs": { - "definedRecords": { - "title": "Defined Records", - "description": "Sets out what the valid set of Record types is for this CSP. It is expressed as a 'oneOf' list, each of which combines a type identifier and a reference to the type that is used to validate the record", - "oneOf": [ - { - "allOf": [ - { - "type": "object", - "title": "Subscriber record", - "description": "Uses the ETSI-standard subscriber record, extended to include a pointer to a Person record", - "properties": { - "type": { - "const": "Subscription" - } - } - }, - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" - }, - { - "$ref": "#/$defs/extendedSubscriberRecord" - } - ] - }, - { - "allOf": [ - { - "type": "object", - "properties": { - "type": { - "const": "Person" - } - } - }, - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/PersonDetails" - } - ] - } - ] - }, - "extendedSubscriberRecord": { - "properties": { - "associatedPeople": { - "title" : "Associated People", - "description" : "A list of people associated with the subscription, together with their relationship", - "type" : "array", - "items" : { - "allOf": [ - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" - }, - { - "type": "object", - "properties": { - "pointerType": { - "enum": [ - "Account Holder", - "Bill Payer" - ] - } - } - } - ] - } - } - }, - "required": [ - "associatedPeople" - ] - } - } -} \ No newline at end of file diff --git a/103705/examples/csp_b_results.json b/103705/examples/csp_b_results.json deleted file mode 100644 index 812ed58..0000000 --- a/103705/examples/csp_b_results.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "recordSetDescription" : { - "typesUsed" : { - "Subscriber" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord", - "Person" : "http://etsi.org/temp/705/entities/$defs/PersonDetails" - } - }, - "recordSet" : [ - { - "id" : "Record1", - "type" : "Subscription", - "subscriptionId" : "{CSP B subscription identifier}", - "associatedPeople" : [{ - "destination" : "Record2", - "pointerType" : "Account Holder" - },{ - "destination" : "Record2", - "pointerType" : "Bill Payer" - }] - }, - { - "id" : "Record2", - "type" : "Person", - "firstName" : "Bill", - "lastName" : "Payer" - } - ] -} \ No newline at end of file diff --git a/103705/examples/example_csp_core_schema.json b/103705/examples/example_csp_core_schema.json deleted file mode 100644 index e3241ce..0000000 --- a/103705/examples/example_csp_core_schema.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "$id": "http://example.com/csp/1.1.1", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Example CSP schema demonstrating how the requirements of TS 103 705 clause 6.5.3 can be met", - "description": "The top level of the schema is an allOf union between the ETSI Response Schema (meeting the first requirement) and a custom definition that sets out the permisslbe record types (meeting the second requirement)", - "allOf": [ - { - "$ref": "http://etsi.org/temp/705" - }, - { - "type": "object", - "properties": { - "recordSet": { - "type": "array", - "items": { - "$ref": "#/$defs/definedRecords" - } - } - }, - "required": [ - "recordSet" - ] - } - ], - "$defs": { - "definedRecords": { - "title": "Defined Records", - "description": "This oneOf alternation effectively sets what the valid Record types are for this CSP schema. The previous allOf union ensures that each Record must also be a valid ETSI record as per TS 103 705 clause 5.3", - "oneOf": [ - { - "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", - "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" - }, - { - "description" : "This means that a custom extended subcription record is a valid choice of record for this schema - the definition is given in the $defs section below", - "$ref": "#/$defs/extendedSubscriberRecord" - } - ] - } - }, - "extendedSubscriberRecord": { - "allOf" : { - { - "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", - "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" - }, - - } - "properties": { - "associatedPeople": { - "title" : "Associated People", - "description" : "A list of people associated with the subscription, together with their relationship", - "type" : "array", - "items" : { - "allOf": [ - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" - }, - { - "type": "object", - "properties": { - "pointerType": { - "enum": [ - "Account Holder", - "Bill Payer" - ] - } - } - } - ] - } - } - }, - "required": [ - "associatedPeople" - ] - } - } -} \ No newline at end of file diff --git a/103705/metaschema/etsi_response.schema.json b/103705/metaschema/etsi_response.schema.json deleted file mode 100644 index fe23cb6..0000000 --- a/103705/metaschema/etsi_response.schema.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$id": "http://etsi.org/temp/705_metaschema", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "ETSI TS 103 705 Response Metaschema", - "description": "Metaschema for checking that a CSP's Response Schema is per-spec", - "type:": "object", - "properties" : { - "title" : { "type" : "string" }, - "description" : { "type" : "string" } - }, - "required": ["title", "description"] -} \ No newline at end of file diff --git a/103705/schema/280_stub.schema.json b/103705/schema/280_stub.schema.json deleted file mode 100644 index 523a1a5..0000000 --- a/103705/schema/280_stub.schema.json +++ /dev/null @@ -1,387 +0,0 @@ -{ - "$id" : "http://etsi.org/temp/280", - "$schema" : "https://json-schema.org/draft/2020-12/schema", - "title" : "ETSI TS 103 280 (temp)", - "description" : "A stubbed version of TS 103 280 definitions (need a CR to 280 for this)", - "$defs": { - "ShortString": { - "type": "string", - "maxLength": 255 - }, - "LongString": { - "type": "string", - "maxLength": 65535 - }, - "LIID": { - "type": "string", - "pattern": "^([!-~]{1,25})|([0-9a-f]{26,50})$" - }, - "UTCDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "UTCMicrosecondDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}Z$" - }, - "QualifiedDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|[+-][0-9]{2}:[0-9]{2})$" - }, - "QualifiedMicrosecondDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}(Z|[+-][0-9]{2}:[0-9]{2})$" - }, - "InternationalE164": { - "type": "string", - "pattern": "^[0-9]{1,15}$" - }, - "IMSI": { - "type": "string", - "pattern": "^[0-9]{6,15}$" - }, - "IMEI": { - "type": "string", - "pattern": "^[0-9]{14}$" - }, - "IMEICheckDigit": { - "type": "string", - "pattern": "^[0-9]{15}$" - }, - "IMEISV": { - "type": "string", - "pattern": "^[0-9]{16}$" - }, - "IPv4Address": { - "type": "string", - "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])$" - }, - "IPv4CIDR": { - "type": "string", - "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/([1-2]?[0-9]|3[0-2])$" - }, - "IPv6Address": { - "type": "string", - "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})$" - }, - "IPv6CIDR": { - "type": "string", - "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})/(([1-9][0-9]?)|(1[0-1][0-9])|(12[0-8]))$" - }, - "TCPPort": { - "type": "integer", - "exclusiveMinimum": 1, - "maximum": 65535 - }, - "UDPPort": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - }, - "MACAddress": { - "type": "string", - "pattern": "^([a-f0-9]{2}:){5}[a-f0-9]{2}$" - }, - "EmailAddress": { - "allOf": [ - { - "$ref": "#/$defs/ShortString" - }, - { - "type": "string", - "pattern": "^[a-zA-Z0-9\\.!#$%&'\\*\\+\\\\/=\\?\\^_`\\{\\|\\}~\\-]+@[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" - } - ] - }, - "UUID": { - "type": "string", - "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" - }, - "ISOCountryCode": { - "type": "string", - "pattern": "^[A-Z]{2}$" - }, - "SIPURI": { - "type": "string", - "pattern": "^sips?:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" - }, - "TELURI": { - "type": "string", - "pattern": "^tel:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" - }, - "WGS84LatitudeDecimal": { - "type": "string", - "pattern": "^[NS][0-9]{2}\\.[0-9]{6}$" - }, - "WGS84LongitudeDecimal": { - "type": "string", - "pattern": "^[EW][0-9]{3}\\.[0-9]{6}$" - }, - "WGS84LatitudeAngular": { - "type": "string", - "pattern": "^[NS][0-9]{6}\\.[0-9]{2}$" - }, - "WGS84LongitudeAngular": { - "type": "string", - "pattern": "^[EW][0-9]{7}\\.[0-9]{2}$" - }, - "SUPIIMSI": { - "$ref": "#/$defs/IMSI" - }, - "SUPINAI": { - "$ref": "#/$defs/NAI" - }, - "SUCI": { - "type": "string", - "pattern": "^([a-fA-F0-9]{2})*$" - }, - "PEIIMEI": { - "$ref": "#/$defs/IMEI" - }, - "PEIIMEICheckDigit": { - "$ref": "#/$defs/IMEICheckDigit" - }, - "PEIIMEISV": { - "$ref": "#/$defs/IMEISV" - }, - "GPSIMSISDN": { - "type": "string", - "pattern": "^[0-9]{1,15}$" - }, - "GPSINAI": { - "$ref": "#/$defs/NAI" - }, - "NAI": { - "type": "string" - }, - "LDID": { - "type": "string", - "pattern": "^([A-Z]{2}-.+-.+)$" - }, - "InternationalizedEmailAddress": { - "allOf": [ - { - "$ref": "#/$defs/ShortString" - }, - { - "type": "string", - "pattern": "^.+@.+$" - } - ] - }, - "EUI64": { - "type": "string", - "pattern": "^([a-f0-9]{2}:){7}[a-f0-9]{2}$" - }, - "CGI": { - "type": "string", - "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{4}-[a-f0-9]{4}$" - }, - "ECGI": { - "type": "string", - "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{7}$" - }, - "NCGI": { - "type": "string", - "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{9}$" - }, - "ICCID": { - "type": "string", - "pattern": "^[0-9]{19,20}$" - }, - "IPAddress": { - "oneOf": [ - { - "type": "object", - "properties": { - "IPv4Address": { - "$ref": "#/$defs/IPv4Address" - } - }, - "required": [ - "IPv4Address" - ] - }, - { - "type": "object", - "properties": { - "IPv6Address": { - "$ref": "#/$defs/IPv6Address" - } - }, - "required": [ - "IPv6Address" - ] - } - ] - }, - "IPCIDR": { - "oneOf": [ - { - "type": "object", - "properties": { - "IPv4CIDR": { - "$ref": "#/$defs/IPv4CIDR" - } - }, - "required": [ - "IPv4CIDR" - ] - }, - { - "type": "object", - "properties": { - "IPv6CIDR": { - "$ref": "#/$defs/IPv6CIDR" - } - }, - "required": [ - "IPv6CIDR" - ] - } - ] - }, - "TCPPortRange": { - "type": "object", - "properties": { - "start": { - "$ref": "#/$defs/TCPPort" - }, - "end": { - "$ref": "#/$defs/TCPPort" - } - }, - "required": [ - "start", - "end" - ] - }, - "UDPPortRange": { - "type": "object", - "properties": { - "start": { - "$ref": "#/$defs/UDPPort" - }, - "end": { - "$ref": "#/$defs/UDPPort" - } - }, - "required": [ - "start", - "end" - ] - }, - "Port": { - "oneOf": [ - { - "type": "object", - "properties": { - "TCPPort": { - "$ref": "#/$defs/TCPPort" - } - }, - "required": [ - "TCPPort" - ] - }, - { - "type": "object", - "properties": { - "UDPPort": { - "$ref": "#/$defs/UDPPort" - } - }, - "required": [ - "UDPPort" - ] - } - ] - }, - "PortRange": { - "oneOf": [ - { - "type": "object", - "properties": { - "TCPPortRange": { - "$ref": "#/$defs/TCPPortRange" - } - }, - "required": [ - "TCPPortRange" - ] - }, - { - "type": "object", - "properties": { - "UDPPortRange": { - "$ref": "#/$defs/UDPPortRange" - } - }, - "required": [ - "UDPPortRange" - ] - } - ] - }, - "IPAddressPort": { - "type": "object", - "properties": { - "address": { - "$ref": "#/$defs/IPAddress" - }, - "port": { - "$ref": "#/$defs/Port" - } - }, - "required": [ - "address", - "port" - ] - }, - "IPAddressPortRange": { - "type": "object", - "properties": { - "address": { - "$ref": "#/$defs/IPAddress" - }, - "portRange": { - "$ref": "#/$defs/PortRange" - } - }, - "required": [ - "address", - "portRange" - ] - }, - "WGS84CoordinateDecimal": { - "type": "object", - "properties": { - "latitude": { - "$ref": "#/$defs/WGS84LatitudeDecimal" - }, - "longitude": { - "$ref": "#/$defs/WGS84LongitudeDecimal" - } - }, - "required": [ - "latitude", - "longitude" - ] - }, - "WGS84CoordinateAngular": { - "type": "object", - "properties": { - "latitude": { - "$ref": "#/$defs/WGS84LatitudeAngular" - }, - "longitude": { - "$ref": "#/$defs/WGS84LongitudeAngular" - } - }, - "required": [ - "latitude", - "longitude" - ] - } - } -} \ No newline at end of file diff --git a/103705/schema/etsi_entities.schema.json b/103705/schema/etsi_types.schema.json similarity index 68% rename from 103705/schema/etsi_entities.schema.json rename to 103705/schema/etsi_types.schema.json index 9d76005..34e34a7 100644 --- a/103705/schema/etsi_entities.schema.json +++ b/103705/schema/etsi_types.schema.json @@ -1,8 +1,8 @@ { - "$id": "http://etsi.org/temp/705/entities", + "$id": "etsi.org_ts103705_1.3.1_etsi-types", "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "ETSI TS 103 705", - "description": "JSON schema for ETSI-defined entities (see Annex A)", + "title": "ETSI TS 103 705 Types schema", + "description": "JSON schema for ETSI-defined types (see clause 6.1 and Annex A)", "$defs": { "Pointer" : { "type" : "object", @@ -14,10 +14,10 @@ "title" : "Pointer Destination", "description" : "Identifies the Record in the RecordSet containined the referred-to data (see clause 5.3.2 and A.2.2)" }, - "pointerType" : { + "relationship" : { "type" : "string", - "title" : "Pointer type", - "Description" : "Indicates the nature of the relationship. Valid values should be indicated by the CSP schema" + "title" : "Relationship", + "description" : "Indicates the nature of the relationship. Valid values should be indicated by the CSP schema" }, "required" : ["destination"] } @@ -35,12 +35,12 @@ "startDate": { "title" : "Subscription Start Date", "description" : "Date that the subscription started", - "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + "$ref": "ts_103280_2017_07#/$defs/UTCDateTime" }, "endDate": { "title" : "Subscription End Date", "description" : "Date that the subscription ended", - "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + "$ref": "ts_103280_2017_07#/$defs/UTCDateTime" } }, "required" : ["subscriptionId"] @@ -50,25 +50,30 @@ "title": "Person Details", "description": "Details about a person", "properties": { - "firstName": { - "title" : "First Name", - "description" : "First name or given name", + "forename": { + "title" : "Forename", + "description" : "Forename(s)", "type": "string" }, - "middeNames": { + "middeName": { "title" : "First Name", - "description" : "Middle names, given as a space-delimited string", + "description" : "Middle name(s), given as a space-delimited string", "type": "string" }, "lastName" : { - "title" : "Last Name", - "description" : "Lastname or family name", + "title" : "Family Name", + "description" : "Family name(s)", "type" : "string" }, "birthDate" : { "title" : "Date of Birth", "description" : "Date of birth (time part can be ignored)", - "$ref" : "http://etsi.org/temp/280#/$defs/DateTime" + "$ref" : "ts_103280_2017_07#/$defs/DateTime" + }, + "gender" : { + "title" : "Gender", + "description" : "Gender of the person", + "type" : "string" } }, "required" : ["lastName"] diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json index 95bb81d..57e0e12 100644 --- a/103705/schema/response.schema.json +++ b/103705/schema/response.schema.json @@ -1,5 +1,5 @@ { - "$id": "http://etsi.org/temp/705", + "$id": "etsi.org_ts103705_1.3.1", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ETSI TS 103 705 Response", "description": "Overall Response structure, representing the results of a lawful disclosure query (see clause 5)", @@ -24,11 +24,12 @@ "properties": { "schemaId" : { "$ref" : "#/$defs/schemaId" }, "resultSetId" : { "$ref" : "#/$defs/resultSetId"}, - "queryReference" : { "type" : "#/$defs/queryReference" }, - "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"} + "queryReference" : { "$ref" : "#/$defs/queryReference" }, + "created" : {"$ref" : "ts_103280_2017_07#/$defs/QualifiedMicrosecondDateTime"}, + "recordTypes" : {"$ref" : "#/$defs/recordTypes"} }, "required": [ - "schemaId", "resultSetId", "queryReference", "created" + "schemaId", "resultSetId", "queryReference", "created", "recordTypes" ] }, "schemaId": { @@ -43,24 +44,33 @@ "recordSet": { "type": "array", "title": "RecordSet", - "description": "RecordSet type (see clause 5.3.1). Contains a set of Records.", + "description": "RecordSet (see clause 5.3). Contains a set of Records.", "items": { "$ref": "#$defs/record" } }, + "recordTypes" : { + "type" : "object", + "title" : "RecordTypes", + "description" : "Dictionary of types used by the Response document (see clause 5.2.6)" + }, "record": { "type": "object", "title": "Record", - "description": "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", + "description": "Response record (see clause 5.3)", "properties": { "id": { "title": "id", "description": "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", "type": "string" + }, + "type" : { + "title" : "type", + "description" : "Type identifier, shall be mapped to a schema reference via the recrdTypes map (see clauses 5.2.6 and 5.3.3)" } }, "required": [ - "id" + "id", "type" ] } } diff --git a/103705/validate.py b/103705/validate.py index a290ed3..fa54909 100644 --- a/103705/validate.py +++ b/103705/validate.py @@ -50,8 +50,8 @@ if __name__ == "__main__": parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") parser.add_argument('schema', help="Primary schema to validate against") - parser.add_argument('filename', help="JSON instance document to validate") args = parser.parse_args() @@ -65,7 +65,8 @@ if __name__ == "__main__": logging.debug(f"Arguments: {args}") - instance_doc = load_json(args.filename) + instance_doc = json.loads(args.input.read()) + args.input.close() main_schema = load_json(args.schema) schema_dict = { main_schema['$id'] : main_schema } diff --git a/103705/validation/validate_705.py b/103705/validation/validate_705.py new file mode 100644 index 0000000..bb6913d --- /dev/null +++ b/103705/validation/validate_705.py @@ -0,0 +1,97 @@ +import sys +from jsonschema import validate, RefResolver, Draft202012Validator +import json +from pathlib import Path +import logging +import argparse + +def handle_uri(u): + print(u) + +def load_json(path : str): + with open(path) as f: + return json.load(f) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") + parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + instance_doc = json.loads(args.input.read()) + args.input.close() + + config = { + 'schema_include_dirs' : [ + '../schema/', + '../../103280/', + ], + 'main_schema_doc' : '../schema/response.schema.json' + } + + rootPath = Path(sys.argv[0]).parent + main_schema = load_json(str(rootPath / config['main_schema_doc'])) + schema_dict = { main_schema['$id'] : main_schema } + + schema_paths = [] + for d in config['schema_include_dirs']: + schema_paths += [f for f in (rootPath / Path(d)).rglob("*.schema.json")] + logging.info(f"Core schema files loaded: {schema_paths}") + if args.schemadir: + for d in args.schemadir: + schema_paths += [f for f in Path(d).rglob("*.schema.json")] + logging.info(f"CSP schema files loaded: {schema_paths}") + else: + logging.info(f"No CSP schema files loaded") + schemas_json = [json.load(p.open()) for p in schema_paths] + schema_dict = schema_dict | { s['$id'] : s for s in schemas_json } + + logging.info(f"Schema IDs loaded: {[k for k in schema_dict.keys()]}") + + logging.debug (f"Instance doc: {instance_doc}") + logging.debug (f"Main schema: {main_schema}") + + resolver = RefResolver(None, + referrer=None, + store=schema_dict) + + logging.info("Performing ETSI validation") + v = Draft202012Validator(main_schema, resolver=resolver) + v.validate(instance_doc) + + logging.info("Building record type dictionary") + type_dict = instance_doc['recordSetDescription']['recordTypes'] + logging.debug(type_dict) + ref_dict = { k : {"$ref" : v} for k,v in type_dict.items()} + validator_dict = { k : Draft202012Validator(ref_dict[k], resolver=resolver) for k,v in ref_dict.items()} + logging.debug(ref_dict) + + logging.info("Validating records") + for r in instance_doc['recordSet']: + type_key = r['type'] + if type_key not in type_dict.keys(): + logging.error(f"Record {r['id']} has type {type_key}, not in recordType dict") + type_ref = type_dict[type_key] + type_schema_id = type_ref.split('#')[0] + logging.info(f"Using {type_schema_id} to validate {type_ref} in record {r['id']}") + if not (type_key in validator_dict.keys()): + logging.error(f'Type key {type_key} from type {type_ref} in record {r["id"]} not in validator dictionary') + print(ref_dict) + v = validator_dict[type_key] + v.validate(r) + + logging.info("Done") \ No newline at end of file -- GitLab From 5c22030a6ab61485b351956f5cc577c7689642fd Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 23 Mar 2023 13:33:32 +0000 Subject: [PATCH 06/10] Initial thoughts --- 103705/examples/csp_a.schema.json | 65 ++++ 103705/examples/csp_a_results.json | 19 ++ 103705/examples/csp_b.schema.json | 98 ++++++ 103705/examples/csp_b_results.json | 28 ++ 103705/schema/280_stub.schema.json | 387 ++++++++++++++++++++++++ 103705/schema/etsi_entities.schema.json | 77 +++++ 103705/schema/response.schema.json | 73 +++++ 103705/validate.py | 93 ++++++ 8 files changed, 840 insertions(+) create mode 100644 103705/examples/csp_a.schema.json create mode 100644 103705/examples/csp_a_results.json create mode 100644 103705/examples/csp_b.schema.json create mode 100644 103705/examples/csp_b_results.json create mode 100644 103705/schema/280_stub.schema.json create mode 100644 103705/schema/etsi_entities.schema.json create mode 100644 103705/schema/response.schema.json create mode 100644 103705/validate.py diff --git a/103705/examples/csp_a.schema.json b/103705/examples/csp_a.schema.json new file mode 100644 index 0000000..d278559 --- /dev/null +++ b/103705/examples/csp_a.schema.json @@ -0,0 +1,65 @@ +{ + "$id": "http://example.com/csp_a/response", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Example CSP schema A for ETSI TS 103 705 response", + "description": "Example extended schema for CSP A. This CSP only ever has one person associated with a subscription - the account holder - and so composes a Subscription record from the ETSI-standard Subscription and PersonDetails entities", + "allOf": [ + { + "$ref": "http://etsi.org/temp/705" + }, + { + "type": "object", + "properties": { + "recordSet": { + "type": "array", + "items": { + "$ref": "#/$defs/definedRecords" + } + } + }, + "required": [ + "recordSet" + ] + } + ], + "$defs": { + "definedRecords": { + "title" : "Defined Records", + "description" : "Sets out what the valid set of Record types is for this CSP. It is expressed as a 'oneOf' list, each of which combines a type identifier and a reference to the type that is used to validate the record", + "oneOf": [ + { + "allOf": [ + { + "type": "object", + "title" : "Subscriber record", + "description" : "Uses the ETSI-standard subscriber record (the first $ref), extended to include person details (the second $ref)", + "properties": { + "type": { + "const": "Subscription" + } + } + }, + { + "description" : "Uses the basic ETSI-standard subscriber record fields as a base", + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + { + "description" : "Extends the basic ETSI standard subscriber record according to the definition in extendedSubscriberRecord", + "$ref": "#/$defs/extendedSubscriberRecord" + } + ] + } + ] + }, + "extendedSubscriberRecord" : { + "title" : "Extended Subscriber Record", + "description" : "Here CSP A can describe all the extra fields that they want to put in their subscriber record. In this case, they just add the details of the account holder using the ETSI-standard PersonDetails type", + "properties" : { + "title" : "Account Holder", + "description" : "Name and date of birth of the account holder for the description", + "accountHolder" : { "$ref" : "http://etsi.org/temp/705/entities#/$defs/PersonDetails"} + }, + "required" : ["personDetails"] + } + } +} \ No newline at end of file diff --git a/103705/examples/csp_a_results.json b/103705/examples/csp_a_results.json new file mode 100644 index 0000000..9f91108 --- /dev/null +++ b/103705/examples/csp_a_results.json @@ -0,0 +1,19 @@ +{ + "recordSetDescription" : { + "typesUsed" : { + "type1" : "uri://example.com/schema1.schema.json", + "type2" : "uri://example.com/schema2.schema.json" + } + }, + "recordSet" : [ + { + "id" : "1e997322-b813-437b-b2e2-dae732f0cf7f", + "type" : "Subscription", + "subscriptionId" : "{unique CSP subscription ID}", + "personDetails" : { + "lastName" : "Last", + "firstName" : "First" + } + } + ] +} \ No newline at end of file diff --git a/103705/examples/csp_b.schema.json b/103705/examples/csp_b.schema.json new file mode 100644 index 0000000..1e9be64 --- /dev/null +++ b/103705/examples/csp_b.schema.json @@ -0,0 +1,98 @@ +{ + "$id": "http://example.com/csp_a/response", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Example CSP schema A for ETSI TS 103 705 response", + "description": "Example extended schema for CSP B. This CSP has multiple different types of person associated with a subscription (account holder and bill payer); since these are often the same person, they want to be able to normalise this properly. They do this using a Pointer.", + "allOf": [ + { + "$ref": "http://etsi.org/temp/705" + }, + { + "type": "object", + "properties": { + "recordSet": { + "type": "array", + "items": { + "$ref": "#/$defs/definedRecords" + } + } + }, + "required": [ + "recordSet" + ] + } + ], + "$defs": { + "definedRecords": { + "title": "Defined Records", + "description": "Sets out what the valid set of Record types is for this CSP. It is expressed as a 'oneOf' list, each of which combines a type identifier and a reference to the type that is used to validate the record", + "oneOf": [ + { + "allOf": [ + { + "type": "object", + "title": "Subscriber record", + "description": "Uses the ETSI-standard subscriber record, extended to include a pointer to a Person record", + "properties": { + "type": { + "const": "Subscription" + } + } + }, + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + { + "$ref": "#/$defs/extendedSubscriberRecord" + } + ] + }, + { + "allOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "Person" + } + } + }, + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/PersonDetails" + } + ] + } + ] + }, + "extendedSubscriberRecord": { + "properties": { + "associatedPeople": { + "title" : "Associated People", + "description" : "A list of people associated with the subscription, together with their relationship", + "type" : "array", + "items" : { + "allOf": [ + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" + }, + { + "type": "object", + "properties": { + "pointerType": { + "enum": [ + "Account Holder", + "Bill Payer" + ] + } + } + } + ] + } + } + }, + "required": [ + "associatedPeople" + ] + } + } +} \ No newline at end of file diff --git a/103705/examples/csp_b_results.json b/103705/examples/csp_b_results.json new file mode 100644 index 0000000..454da56 --- /dev/null +++ b/103705/examples/csp_b_results.json @@ -0,0 +1,28 @@ +{ + "recordSetDescription" : { + "typesUsed" : { + "type1" : "uri://example.com/schema1.schema.json", + "type2" : "uri://example.com/schema2.schema.json" + } + }, + "recordSet" : [ + { + "id" : "Record1", + "type" : "Subscription", + "subscriptionId" : "{CSP B subscription identifier}", + "associatedPeople" : [{ + "destination" : "Record2", + "pointerType" : "Account Holder" + },{ + "destination" : "Record2", + "pointerType" : "Bill Payer" + }] + }, + { + "id" : "Record2", + "type" : "Person", + "firstName" : "Bill", + "lastName" : "Payer" + } + ] +} \ No newline at end of file diff --git a/103705/schema/280_stub.schema.json b/103705/schema/280_stub.schema.json new file mode 100644 index 0000000..523a1a5 --- /dev/null +++ b/103705/schema/280_stub.schema.json @@ -0,0 +1,387 @@ +{ + "$id" : "http://etsi.org/temp/280", + "$schema" : "https://json-schema.org/draft/2020-12/schema", + "title" : "ETSI TS 103 280 (temp)", + "description" : "A stubbed version of TS 103 280 definitions (need a CR to 280 for this)", + "$defs": { + "ShortString": { + "type": "string", + "maxLength": 255 + }, + "LongString": { + "type": "string", + "maxLength": 65535 + }, + "LIID": { + "type": "string", + "pattern": "^([!-~]{1,25})|([0-9a-f]{26,50})$" + }, + "UTCDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "UTCMicrosecondDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}Z$" + }, + "QualifiedDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|[+-][0-9]{2}:[0-9]{2})$" + }, + "QualifiedMicrosecondDateTime": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}(Z|[+-][0-9]{2}:[0-9]{2})$" + }, + "InternationalE164": { + "type": "string", + "pattern": "^[0-9]{1,15}$" + }, + "IMSI": { + "type": "string", + "pattern": "^[0-9]{6,15}$" + }, + "IMEI": { + "type": "string", + "pattern": "^[0-9]{14}$" + }, + "IMEICheckDigit": { + "type": "string", + "pattern": "^[0-9]{15}$" + }, + "IMEISV": { + "type": "string", + "pattern": "^[0-9]{16}$" + }, + "IPv4Address": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])$" + }, + "IPv4CIDR": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/([1-2]?[0-9]|3[0-2])$" + }, + "IPv6Address": { + "type": "string", + "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})$" + }, + "IPv6CIDR": { + "type": "string", + "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})/(([1-9][0-9]?)|(1[0-1][0-9])|(12[0-8]))$" + }, + "TCPPort": { + "type": "integer", + "exclusiveMinimum": 1, + "maximum": 65535 + }, + "UDPPort": { + "type": "integer", + "minimum": 0, + "maximum": 65535 + }, + "MACAddress": { + "type": "string", + "pattern": "^([a-f0-9]{2}:){5}[a-f0-9]{2}$" + }, + "EmailAddress": { + "allOf": [ + { + "$ref": "#/$defs/ShortString" + }, + { + "type": "string", + "pattern": "^[a-zA-Z0-9\\.!#$%&'\\*\\+\\\\/=\\?\\^_`\\{\\|\\}~\\-]+@[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" + } + ] + }, + "UUID": { + "type": "string", + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + }, + "ISOCountryCode": { + "type": "string", + "pattern": "^[A-Z]{2}$" + }, + "SIPURI": { + "type": "string", + "pattern": "^sips?:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" + }, + "TELURI": { + "type": "string", + "pattern": "^tel:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" + }, + "WGS84LatitudeDecimal": { + "type": "string", + "pattern": "^[NS][0-9]{2}\\.[0-9]{6}$" + }, + "WGS84LongitudeDecimal": { + "type": "string", + "pattern": "^[EW][0-9]{3}\\.[0-9]{6}$" + }, + "WGS84LatitudeAngular": { + "type": "string", + "pattern": "^[NS][0-9]{6}\\.[0-9]{2}$" + }, + "WGS84LongitudeAngular": { + "type": "string", + "pattern": "^[EW][0-9]{7}\\.[0-9]{2}$" + }, + "SUPIIMSI": { + "$ref": "#/$defs/IMSI" + }, + "SUPINAI": { + "$ref": "#/$defs/NAI" + }, + "SUCI": { + "type": "string", + "pattern": "^([a-fA-F0-9]{2})*$" + }, + "PEIIMEI": { + "$ref": "#/$defs/IMEI" + }, + "PEIIMEICheckDigit": { + "$ref": "#/$defs/IMEICheckDigit" + }, + "PEIIMEISV": { + "$ref": "#/$defs/IMEISV" + }, + "GPSIMSISDN": { + "type": "string", + "pattern": "^[0-9]{1,15}$" + }, + "GPSINAI": { + "$ref": "#/$defs/NAI" + }, + "NAI": { + "type": "string" + }, + "LDID": { + "type": "string", + "pattern": "^([A-Z]{2}-.+-.+)$" + }, + "InternationalizedEmailAddress": { + "allOf": [ + { + "$ref": "#/$defs/ShortString" + }, + { + "type": "string", + "pattern": "^.+@.+$" + } + ] + }, + "EUI64": { + "type": "string", + "pattern": "^([a-f0-9]{2}:){7}[a-f0-9]{2}$" + }, + "CGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{4}-[a-f0-9]{4}$" + }, + "ECGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{7}$" + }, + "NCGI": { + "type": "string", + "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{9}$" + }, + "ICCID": { + "type": "string", + "pattern": "^[0-9]{19,20}$" + }, + "IPAddress": { + "oneOf": [ + { + "type": "object", + "properties": { + "IPv4Address": { + "$ref": "#/$defs/IPv4Address" + } + }, + "required": [ + "IPv4Address" + ] + }, + { + "type": "object", + "properties": { + "IPv6Address": { + "$ref": "#/$defs/IPv6Address" + } + }, + "required": [ + "IPv6Address" + ] + } + ] + }, + "IPCIDR": { + "oneOf": [ + { + "type": "object", + "properties": { + "IPv4CIDR": { + "$ref": "#/$defs/IPv4CIDR" + } + }, + "required": [ + "IPv4CIDR" + ] + }, + { + "type": "object", + "properties": { + "IPv6CIDR": { + "$ref": "#/$defs/IPv6CIDR" + } + }, + "required": [ + "IPv6CIDR" + ] + } + ] + }, + "TCPPortRange": { + "type": "object", + "properties": { + "start": { + "$ref": "#/$defs/TCPPort" + }, + "end": { + "$ref": "#/$defs/TCPPort" + } + }, + "required": [ + "start", + "end" + ] + }, + "UDPPortRange": { + "type": "object", + "properties": { + "start": { + "$ref": "#/$defs/UDPPort" + }, + "end": { + "$ref": "#/$defs/UDPPort" + } + }, + "required": [ + "start", + "end" + ] + }, + "Port": { + "oneOf": [ + { + "type": "object", + "properties": { + "TCPPort": { + "$ref": "#/$defs/TCPPort" + } + }, + "required": [ + "TCPPort" + ] + }, + { + "type": "object", + "properties": { + "UDPPort": { + "$ref": "#/$defs/UDPPort" + } + }, + "required": [ + "UDPPort" + ] + } + ] + }, + "PortRange": { + "oneOf": [ + { + "type": "object", + "properties": { + "TCPPortRange": { + "$ref": "#/$defs/TCPPortRange" + } + }, + "required": [ + "TCPPortRange" + ] + }, + { + "type": "object", + "properties": { + "UDPPortRange": { + "$ref": "#/$defs/UDPPortRange" + } + }, + "required": [ + "UDPPortRange" + ] + } + ] + }, + "IPAddressPort": { + "type": "object", + "properties": { + "address": { + "$ref": "#/$defs/IPAddress" + }, + "port": { + "$ref": "#/$defs/Port" + } + }, + "required": [ + "address", + "port" + ] + }, + "IPAddressPortRange": { + "type": "object", + "properties": { + "address": { + "$ref": "#/$defs/IPAddress" + }, + "portRange": { + "$ref": "#/$defs/PortRange" + } + }, + "required": [ + "address", + "portRange" + ] + }, + "WGS84CoordinateDecimal": { + "type": "object", + "properties": { + "latitude": { + "$ref": "#/$defs/WGS84LatitudeDecimal" + }, + "longitude": { + "$ref": "#/$defs/WGS84LongitudeDecimal" + } + }, + "required": [ + "latitude", + "longitude" + ] + }, + "WGS84CoordinateAngular": { + "type": "object", + "properties": { + "latitude": { + "$ref": "#/$defs/WGS84LatitudeAngular" + }, + "longitude": { + "$ref": "#/$defs/WGS84LongitudeAngular" + } + }, + "required": [ + "latitude", + "longitude" + ] + } + } +} \ No newline at end of file diff --git a/103705/schema/etsi_entities.schema.json b/103705/schema/etsi_entities.schema.json new file mode 100644 index 0000000..9d76005 --- /dev/null +++ b/103705/schema/etsi_entities.schema.json @@ -0,0 +1,77 @@ +{ + "$id": "http://etsi.org/temp/705/entities", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ETSI TS 103 705", + "description": "JSON schema for ETSI-defined entities (see Annex A)", + "$defs": { + "Pointer" : { + "type" : "object", + "title" : "Pointer", + "description" : "Allows one Record in a RecordSet to point to data in another Record in the same Recordset. See clause 5.4 and Annex A.2.2", + "properties" : { + "destination" : { + "type" : "string", + "title" : "Pointer Destination", + "description" : "Identifies the Record in the RecordSet containined the referred-to data (see clause 5.3.2 and A.2.2)" + }, + "pointerType" : { + "type" : "string", + "title" : "Pointer type", + "Description" : "Indicates the nature of the relationship. Valid values should be indicated by the CSP schema" + }, + "required" : ["destination"] + } + }, + "Subscription": { + "type": "object", + "title": "Subscription", + "description": "Basic subscription record", + "properties": { + "subscriptionId": { + "title" : "Subscription Identifier", + "description" : "Unique identifier for a subscription within the CSP", + "type": "string" + }, + "startDate": { + "title" : "Subscription Start Date", + "description" : "Date that the subscription started", + "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + }, + "endDate": { + "title" : "Subscription End Date", + "description" : "Date that the subscription ended", + "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + } + }, + "required" : ["subscriptionId"] + }, + "PersonDetails": { + "type": "object", + "title": "Person Details", + "description": "Details about a person", + "properties": { + "firstName": { + "title" : "First Name", + "description" : "First name or given name", + "type": "string" + }, + "middeNames": { + "title" : "First Name", + "description" : "Middle names, given as a space-delimited string", + "type": "string" + }, + "lastName" : { + "title" : "Last Name", + "description" : "Lastname or family name", + "type" : "string" + }, + "birthDate" : { + "title" : "Date of Birth", + "description" : "Date of birth (time part can be ignored)", + "$ref" : "http://etsi.org/temp/280#/$defs/DateTime" + } + }, + "required" : ["lastName"] + } + } +} \ No newline at end of file diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json new file mode 100644 index 0000000..4bfa29f --- /dev/null +++ b/103705/schema/response.schema.json @@ -0,0 +1,73 @@ +{ + "$id": "http://etsi.org/temp/705", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ETSI TS 103 705 Response", + "description": "Overall Response structure, representing the results of a lawful disclosure query (see clause 5)", + "type:": "object", + "properties": { + "recordSetDescription": { + "$ref": "#/$defs/recordSetDescription" + }, + "recordSet": { + "$ref": "#/$defs/recordSet" + } + }, + "required": [ + "recordSetDescription", + "recordSet" + ], + "$defs": { + "recordSetDescription": { + "type": "object", + "title": "RecordSetDescription", + "description" : "Provides metadata about the records being delivered (see clause 5.2)", + "properties": { + "typesUsed": { + + } + }, + "required": [ + "typesUsed" + ] + }, + "typesUsed": { + "type": "object", + "title": "TypesUsed", + "description" : "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", + "patternProperties": { + "": { + "$ref": "#/$defs/schemaReference" + } + } + }, + "schemaReference": { + "type": "string" + }, + "recordSet": { + "type": "array", + "title": "RecordSet", + "description" : "RecordSet type (see clause 5.3.1). Contains a set of Records.", + "items": { + "$ref": "#$defs/record" + } + }, + "record": { + "type": "object", + "title": "Record", + "description" : "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", + "properties": { + "id": { + "title" : "id", + "description" : "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", + "type": "string" + }, + "type": { + "title" : "type", + "description" : "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", + "type": "string" + } + }, + "required": ["id", "type"] + } + } +} \ No newline at end of file diff --git a/103705/validate.py b/103705/validate.py new file mode 100644 index 0000000..a46be2d --- /dev/null +++ b/103705/validate.py @@ -0,0 +1,93 @@ +import sys +from jsonschema import validate, RefResolver, Draft202012Validator +import json +from pathlib import Path +import logging +import argparse + + + +# filename = sys.argv[1] + +# def load_json (path): +# with open(path) as f: +# s = json.load(f) +# return s + +# schema_store = {} + +# json_instance = load_json(filename) +# print (json_instance) + +# etsi_schema = load_json('response.schema.json') +# ext_schema = load_json('extended.schema.json') +# ext_ent_schema = load_json("extended_entities.schema.json") +# schema_store = { +# etsi_schema['$id'] : etsi_schema, +# ext_schema['$id'] : ext_schema, +# ext_ent_schema['$id'] : ext_ent_schema +# } + +# resolver = RefResolver(None, referrer=None, store=schema_store) + +# print (etsi_schema) + +# v = Draft202012Validator(ext_schema, resolver=resolver) +# v.validate(json_instance) + +# validate(json_instance, ext_schema) +# print ("OK") + +def handle_uri(u): + print(u) + +def load_json(path : str): + with open(path) as f: + return json.load(f) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") + parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('schema', help="Primary schema to validate against") + parser.add_argument('filename', help="JSON instance document to validate") + + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + instance_doc = load_json(args.filename) + main_schema = load_json(args.schema) + schema_dict = { main_schema['$id'] : main_schema } + + if args.schemadir: + schema_paths = [] + for d in args.schemadir: + schema_paths += [f for f in Path(d).rglob("*.schema.json")] + logging.info(f"Schema files loaded: {schema_paths}") + + schemas_json = [json.load(p.open()) for p in schema_paths] + schema_dict = schema_dict | { s['$id'] : s for s in schemas_json } + + logging.info(f"Schema IDs loaded: {[k for k in schema_dict.keys()]}") + + logging.debug (f"Instance doc: {instance_doc}") + logging.debug (f"Main schema: {main_schema}") + + resolver = RefResolver(None, + referrer=None, + store=schema_dict) + + v = Draft202012Validator(main_schema, resolver=resolver) + v.validate(instance_doc) + + logging.info("Done") \ No newline at end of file -- GitLab From e30b5430a46746a74a533c21660eb89980bba1c9 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 23 Mar 2023 15:21:50 +0000 Subject: [PATCH 07/10] Updating typesUsed --- 103705/examples/csp_a_results.json | 3 +-- 103705/examples/csp_b_results.json | 4 ++-- 103705/schema/response.schema.json | 28 ++++++++++++++++------------ 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/103705/examples/csp_a_results.json b/103705/examples/csp_a_results.json index 9f91108..3081464 100644 --- a/103705/examples/csp_a_results.json +++ b/103705/examples/csp_a_results.json @@ -1,8 +1,7 @@ { "recordSetDescription" : { "typesUsed" : { - "type1" : "uri://example.com/schema1.schema.json", - "type2" : "uri://example.com/schema2.schema.json" + "Subscription" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord" } }, "recordSet" : [ diff --git a/103705/examples/csp_b_results.json b/103705/examples/csp_b_results.json index 454da56..812ed58 100644 --- a/103705/examples/csp_b_results.json +++ b/103705/examples/csp_b_results.json @@ -1,8 +1,8 @@ { "recordSetDescription" : { "typesUsed" : { - "type1" : "uri://example.com/schema1.schema.json", - "type2" : "uri://example.com/schema2.schema.json" + "Subscriber" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord", + "Person" : "http://etsi.org/temp/705/entities/$defs/PersonDetails" } }, "recordSet" : [ diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json index 4bfa29f..9ae7fb1 100644 --- a/103705/schema/response.schema.json +++ b/103705/schema/response.schema.json @@ -20,11 +20,12 @@ "recordSetDescription": { "type": "object", "title": "RecordSetDescription", - "description" : "Provides metadata about the records being delivered (see clause 5.2)", + "description": "Provides metadata about the records being delivered (see clause 5.2)", "properties": { - "typesUsed": { - - } + "resultSetId" : { "$ref" : "#/$defs/resultSetId"}, + "version" : {"$ref" : "#/defs/version"}, + "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"}, + "typesUsed": { "$ref" : "#/$defs/typesUsed"} }, "required": [ "typesUsed" @@ -33,7 +34,7 @@ "typesUsed": { "type": "object", "title": "TypesUsed", - "description" : "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", + "description": "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", "patternProperties": { "": { "$ref": "#/$defs/schemaReference" @@ -46,7 +47,7 @@ "recordSet": { "type": "array", "title": "RecordSet", - "description" : "RecordSet type (see clause 5.3.1). Contains a set of Records.", + "description": "RecordSet type (see clause 5.3.1). Contains a set of Records.", "items": { "$ref": "#$defs/record" } @@ -54,20 +55,23 @@ "record": { "type": "object", "title": "Record", - "description" : "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", + "description": "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", "properties": { "id": { - "title" : "id", - "description" : "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", + "title": "id", + "description": "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", "type": "string" }, "type": { - "title" : "type", - "description" : "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", + "title": "type", + "description": "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", "type": "string" } }, - "required": ["id", "type"] + "required": [ + "id", + "type" + ] } } } \ No newline at end of file -- GitLab From a8a82c1c420b441490fc1fd64785afb7623dc9ad Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 18 May 2023 10:48:45 +0100 Subject: [PATCH 08/10] Tweaks --- 103705/examples/csp_a.schema.json | 1 - 103705/generate_docs.py | 2 ++ 103705/metaschema/etsi_response.schema.json | 12 ++++++++++++ 103705/validate.py | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 103705/generate_docs.py create mode 100644 103705/metaschema/etsi_response.schema.json diff --git a/103705/examples/csp_a.schema.json b/103705/examples/csp_a.schema.json index d278559..b6fe543 100644 --- a/103705/examples/csp_a.schema.json +++ b/103705/examples/csp_a.schema.json @@ -1,7 +1,6 @@ { "$id": "http://example.com/csp_a/response", "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Example CSP schema A for ETSI TS 103 705 response", "description": "Example extended schema for CSP A. This CSP only ever has one person associated with a subscription - the account holder - and so composes a Subscription record from the ETSI-standard Subscription and PersonDetails entities", "allOf": [ { diff --git a/103705/generate_docs.py b/103705/generate_docs.py new file mode 100644 index 0000000..8837e1f --- /dev/null +++ b/103705/generate_docs.py @@ -0,0 +1,2 @@ +import json +from pathlib import Path diff --git a/103705/metaschema/etsi_response.schema.json b/103705/metaschema/etsi_response.schema.json new file mode 100644 index 0000000..fe23cb6 --- /dev/null +++ b/103705/metaschema/etsi_response.schema.json @@ -0,0 +1,12 @@ +{ + "$id": "http://etsi.org/temp/705_metaschema", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ETSI TS 103 705 Response Metaschema", + "description": "Metaschema for checking that a CSP's Response Schema is per-spec", + "type:": "object", + "properties" : { + "title" : { "type" : "string" }, + "description" : { "type" : "string" } + }, + "required": ["title", "description"] +} \ No newline at end of file diff --git a/103705/validate.py b/103705/validate.py index a46be2d..a290ed3 100644 --- a/103705/validate.py +++ b/103705/validate.py @@ -88,6 +88,7 @@ if __name__ == "__main__": store=schema_dict) v = Draft202012Validator(main_schema, resolver=resolver) + v.validate(instance_doc) logging.info("Done") \ No newline at end of file -- GitLab From 688704694435324d462369ec17da28ff6106a814 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 8 Jun 2023 09:39:37 +0100 Subject: [PATCH 09/10] Latest changes --- 103705/examples/example_csp_core_schema.json | 79 ++++++++++++++++++++ 103705/schema/response.schema.json | 34 +++------ 2 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 103705/examples/example_csp_core_schema.json diff --git a/103705/examples/example_csp_core_schema.json b/103705/examples/example_csp_core_schema.json new file mode 100644 index 0000000..e3241ce --- /dev/null +++ b/103705/examples/example_csp_core_schema.json @@ -0,0 +1,79 @@ +{ + "$id": "http://example.com/csp/1.1.1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Example CSP schema demonstrating how the requirements of TS 103 705 clause 6.5.3 can be met", + "description": "The top level of the schema is an allOf union between the ETSI Response Schema (meeting the first requirement) and a custom definition that sets out the permisslbe record types (meeting the second requirement)", + "allOf": [ + { + "$ref": "http://etsi.org/temp/705" + }, + { + "type": "object", + "properties": { + "recordSet": { + "type": "array", + "items": { + "$ref": "#/$defs/definedRecords" + } + } + }, + "required": [ + "recordSet" + ] + } + ], + "$defs": { + "definedRecords": { + "title": "Defined Records", + "description": "This oneOf alternation effectively sets what the valid Record types are for this CSP schema. The previous allOf union ensures that each Record must also be a valid ETSI record as per TS 103 705 clause 5.3", + "oneOf": [ + { + "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + { + "description" : "This means that a custom extended subcription record is a valid choice of record for this schema - the definition is given in the $defs section below", + "$ref": "#/$defs/extendedSubscriberRecord" + } + ] + } + }, + "extendedSubscriberRecord": { + "allOf" : { + { + "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", + "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" + }, + + } + "properties": { + "associatedPeople": { + "title" : "Associated People", + "description" : "A list of people associated with the subscription, together with their relationship", + "type" : "array", + "items" : { + "allOf": [ + { + "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" + }, + { + "type": "object", + "properties": { + "pointerType": { + "enum": [ + "Account Holder", + "Bill Payer" + ] + } + } + } + ] + } + } + }, + "required": [ + "associatedPeople" + ] + } + } +} \ No newline at end of file diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json index 9ae7fb1..95bb81d 100644 --- a/103705/schema/response.schema.json +++ b/103705/schema/response.schema.json @@ -22,28 +22,24 @@ "title": "RecordSetDescription", "description": "Provides metadata about the records being delivered (see clause 5.2)", "properties": { + "schemaId" : { "$ref" : "#/$defs/schemaId" }, "resultSetId" : { "$ref" : "#/$defs/resultSetId"}, - "version" : {"$ref" : "#/defs/version"}, - "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"}, - "typesUsed": { "$ref" : "#/$defs/typesUsed"} + "queryReference" : { "type" : "#/$defs/queryReference" }, + "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"} }, "required": [ - "typesUsed" + "schemaId", "resultSetId", "queryReference", "created" ] }, - "typesUsed": { - "type": "object", - "title": "TypesUsed", - "description": "Mapping between type identifiers (see clause 5.3.3) and JSON schema identifiers. Described in clause 5.2.2.", - "patternProperties": { - "": { - "$ref": "#/$defs/schemaReference" - } - } - }, - "schemaReference": { + "schemaId": { "type": "string" }, + "resultSetId": { + "type": "string" + }, + "queryReference": { + "type": "string" + }, "recordSet": { "type": "array", "title": "RecordSet", @@ -61,16 +57,10 @@ "title": "id", "description": "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", "type": "string" - }, - "type": { - "title": "type", - "description": "Identifier the type of the record (see clause 5.3.3). The value must match one given in the typesUsed field, which in turn must refer to a schema available to the receiver", - "type": "string" } }, "required": [ - "id", - "type" + "id" ] } } -- GitLab From 69b66272b44b0cc9a8069450b6864b3e887f6e30 Mon Sep 17 00:00:00 2001 From: mark Date: Thu, 8 Jun 2023 11:57:53 +0100 Subject: [PATCH 10/10] Adding per-record validator --- 103705/examples/csp_a.schema.json | 2 +- 103705/examples/csp_a_results.json | 8 +- 103705/examples/csp_b.schema.json | 98 ----- 103705/examples/csp_b_results.json | 28 -- 103705/examples/example_csp_core_schema.json | 79 ---- 103705/metaschema/etsi_response.schema.json | 12 - 103705/schema/280_stub.schema.json | 387 ------------------ ...ies.schema.json => etsi_types.schema.json} | 37 +- 103705/schema/response.schema.json | 24 +- 103705/validate.py | 5 +- 103705/validation/validate_705.py | 97 +++++ 11 files changed, 145 insertions(+), 632 deletions(-) delete mode 100644 103705/examples/csp_b.schema.json delete mode 100644 103705/examples/csp_b_results.json delete mode 100644 103705/examples/example_csp_core_schema.json delete mode 100644 103705/metaschema/etsi_response.schema.json delete mode 100644 103705/schema/280_stub.schema.json rename 103705/schema/{etsi_entities.schema.json => etsi_types.schema.json} (68%) create mode 100644 103705/validation/validate_705.py diff --git a/103705/examples/csp_a.schema.json b/103705/examples/csp_a.schema.json index b6fe543..febb70a 100644 --- a/103705/examples/csp_a.schema.json +++ b/103705/examples/csp_a.schema.json @@ -1,5 +1,5 @@ { - "$id": "http://example.com/csp_a/response", + "$id": "example.com_csp-a-schema_1.2.3", "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Example extended schema for CSP A. This CSP only ever has one person associated with a subscription - the account holder - and so composes a Subscription record from the ETSI-standard Subscription and PersonDetails entities", "allOf": [ diff --git a/103705/examples/csp_a_results.json b/103705/examples/csp_a_results.json index 3081464..e142485 100644 --- a/103705/examples/csp_a_results.json +++ b/103705/examples/csp_a_results.json @@ -1,7 +1,11 @@ { "recordSetDescription" : { - "typesUsed" : { - "Subscription" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord" + "schemaId" : "etsi.org/ts_103_705/1.1.1", + "resultSetId" : "788e190d-fb2e-416e-9798-12a66ebe0a1a", + "queryReference" : "LDID_1", + "created" : "2023-06-08T09:31:56.000000Z", + "recordTypes" : { + "Subscription" : "example.com_csp-a-schema_1.2.3#$defs/extendedSubscriberRecord" } }, "recordSet" : [ diff --git a/103705/examples/csp_b.schema.json b/103705/examples/csp_b.schema.json deleted file mode 100644 index 1e9be64..0000000 --- a/103705/examples/csp_b.schema.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "$id": "http://example.com/csp_a/response", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Example CSP schema A for ETSI TS 103 705 response", - "description": "Example extended schema for CSP B. This CSP has multiple different types of person associated with a subscription (account holder and bill payer); since these are often the same person, they want to be able to normalise this properly. They do this using a Pointer.", - "allOf": [ - { - "$ref": "http://etsi.org/temp/705" - }, - { - "type": "object", - "properties": { - "recordSet": { - "type": "array", - "items": { - "$ref": "#/$defs/definedRecords" - } - } - }, - "required": [ - "recordSet" - ] - } - ], - "$defs": { - "definedRecords": { - "title": "Defined Records", - "description": "Sets out what the valid set of Record types is for this CSP. It is expressed as a 'oneOf' list, each of which combines a type identifier and a reference to the type that is used to validate the record", - "oneOf": [ - { - "allOf": [ - { - "type": "object", - "title": "Subscriber record", - "description": "Uses the ETSI-standard subscriber record, extended to include a pointer to a Person record", - "properties": { - "type": { - "const": "Subscription" - } - } - }, - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" - }, - { - "$ref": "#/$defs/extendedSubscriberRecord" - } - ] - }, - { - "allOf": [ - { - "type": "object", - "properties": { - "type": { - "const": "Person" - } - } - }, - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/PersonDetails" - } - ] - } - ] - }, - "extendedSubscriberRecord": { - "properties": { - "associatedPeople": { - "title" : "Associated People", - "description" : "A list of people associated with the subscription, together with their relationship", - "type" : "array", - "items" : { - "allOf": [ - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" - }, - { - "type": "object", - "properties": { - "pointerType": { - "enum": [ - "Account Holder", - "Bill Payer" - ] - } - } - } - ] - } - } - }, - "required": [ - "associatedPeople" - ] - } - } -} \ No newline at end of file diff --git a/103705/examples/csp_b_results.json b/103705/examples/csp_b_results.json deleted file mode 100644 index 812ed58..0000000 --- a/103705/examples/csp_b_results.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "recordSetDescription" : { - "typesUsed" : { - "Subscriber" : "http://example.com/csp_a/response#/$defs/extendedSubscriberRecord", - "Person" : "http://etsi.org/temp/705/entities/$defs/PersonDetails" - } - }, - "recordSet" : [ - { - "id" : "Record1", - "type" : "Subscription", - "subscriptionId" : "{CSP B subscription identifier}", - "associatedPeople" : [{ - "destination" : "Record2", - "pointerType" : "Account Holder" - },{ - "destination" : "Record2", - "pointerType" : "Bill Payer" - }] - }, - { - "id" : "Record2", - "type" : "Person", - "firstName" : "Bill", - "lastName" : "Payer" - } - ] -} \ No newline at end of file diff --git a/103705/examples/example_csp_core_schema.json b/103705/examples/example_csp_core_schema.json deleted file mode 100644 index e3241ce..0000000 --- a/103705/examples/example_csp_core_schema.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "$id": "http://example.com/csp/1.1.1", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Example CSP schema demonstrating how the requirements of TS 103 705 clause 6.5.3 can be met", - "description": "The top level of the schema is an allOf union between the ETSI Response Schema (meeting the first requirement) and a custom definition that sets out the permisslbe record types (meeting the second requirement)", - "allOf": [ - { - "$ref": "http://etsi.org/temp/705" - }, - { - "type": "object", - "properties": { - "recordSet": { - "type": "array", - "items": { - "$ref": "#/$defs/definedRecords" - } - } - }, - "required": [ - "recordSet" - ] - } - ], - "$defs": { - "definedRecords": { - "title": "Defined Records", - "description": "This oneOf alternation effectively sets what the valid Record types are for this CSP schema. The previous allOf union ensures that each Record must also be a valid ETSI record as per TS 103 705 clause 5.3", - "oneOf": [ - { - "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", - "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" - }, - { - "description" : "This means that a custom extended subcription record is a valid choice of record for this schema - the definition is given in the $defs section below", - "$ref": "#/$defs/extendedSubscriberRecord" - } - ] - } - }, - "extendedSubscriberRecord": { - "allOf" : { - { - "description" : "This means that a standard ETSI subcription record is a valid choice of record for this schema", - "$ref": "http://etsi.org/temp/705/entities#/$defs/Subscription" - }, - - } - "properties": { - "associatedPeople": { - "title" : "Associated People", - "description" : "A list of people associated with the subscription, together with their relationship", - "type" : "array", - "items" : { - "allOf": [ - { - "$ref": "http://etsi.org/temp/705/entities#/$defs/Pointer" - }, - { - "type": "object", - "properties": { - "pointerType": { - "enum": [ - "Account Holder", - "Bill Payer" - ] - } - } - } - ] - } - } - }, - "required": [ - "associatedPeople" - ] - } - } -} \ No newline at end of file diff --git a/103705/metaschema/etsi_response.schema.json b/103705/metaschema/etsi_response.schema.json deleted file mode 100644 index fe23cb6..0000000 --- a/103705/metaschema/etsi_response.schema.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$id": "http://etsi.org/temp/705_metaschema", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "ETSI TS 103 705 Response Metaschema", - "description": "Metaschema for checking that a CSP's Response Schema is per-spec", - "type:": "object", - "properties" : { - "title" : { "type" : "string" }, - "description" : { "type" : "string" } - }, - "required": ["title", "description"] -} \ No newline at end of file diff --git a/103705/schema/280_stub.schema.json b/103705/schema/280_stub.schema.json deleted file mode 100644 index 523a1a5..0000000 --- a/103705/schema/280_stub.schema.json +++ /dev/null @@ -1,387 +0,0 @@ -{ - "$id" : "http://etsi.org/temp/280", - "$schema" : "https://json-schema.org/draft/2020-12/schema", - "title" : "ETSI TS 103 280 (temp)", - "description" : "A stubbed version of TS 103 280 definitions (need a CR to 280 for this)", - "$defs": { - "ShortString": { - "type": "string", - "maxLength": 255 - }, - "LongString": { - "type": "string", - "maxLength": 65535 - }, - "LIID": { - "type": "string", - "pattern": "^([!-~]{1,25})|([0-9a-f]{26,50})$" - }, - "UTCDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "UTCMicrosecondDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}Z$" - }, - "QualifiedDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|[+-][0-9]{2}:[0-9]{2})$" - }, - "QualifiedMicrosecondDateTime": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{6}(Z|[+-][0-9]{2}:[0-9]{2})$" - }, - "InternationalE164": { - "type": "string", - "pattern": "^[0-9]{1,15}$" - }, - "IMSI": { - "type": "string", - "pattern": "^[0-9]{6,15}$" - }, - "IMEI": { - "type": "string", - "pattern": "^[0-9]{14}$" - }, - "IMEICheckDigit": { - "type": "string", - "pattern": "^[0-9]{15}$" - }, - "IMEISV": { - "type": "string", - "pattern": "^[0-9]{16}$" - }, - "IPv4Address": { - "type": "string", - "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])$" - }, - "IPv4CIDR": { - "type": "string", - "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/([1-2]?[0-9]|3[0-2])$" - }, - "IPv6Address": { - "type": "string", - "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})$" - }, - "IPv6CIDR": { - "type": "string", - "pattern": "^([0-9a-f]{4}:){7}([0-9a-f]{4})/(([1-9][0-9]?)|(1[0-1][0-9])|(12[0-8]))$" - }, - "TCPPort": { - "type": "integer", - "exclusiveMinimum": 1, - "maximum": 65535 - }, - "UDPPort": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - }, - "MACAddress": { - "type": "string", - "pattern": "^([a-f0-9]{2}:){5}[a-f0-9]{2}$" - }, - "EmailAddress": { - "allOf": [ - { - "$ref": "#/$defs/ShortString" - }, - { - "type": "string", - "pattern": "^[a-zA-Z0-9\\.!#$%&'\\*\\+\\\\/=\\?\\^_`\\{\\|\\}~\\-]+@[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" - } - ] - }, - "UUID": { - "type": "string", - "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" - }, - "ISOCountryCode": { - "type": "string", - "pattern": "^[A-Z]{2}$" - }, - "SIPURI": { - "type": "string", - "pattern": "^sips?:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" - }, - "TELURI": { - "type": "string", - "pattern": "^tel:[a-zA-Z0-9!#$&-;=?-\\[\\]_~%]+$" - }, - "WGS84LatitudeDecimal": { - "type": "string", - "pattern": "^[NS][0-9]{2}\\.[0-9]{6}$" - }, - "WGS84LongitudeDecimal": { - "type": "string", - "pattern": "^[EW][0-9]{3}\\.[0-9]{6}$" - }, - "WGS84LatitudeAngular": { - "type": "string", - "pattern": "^[NS][0-9]{6}\\.[0-9]{2}$" - }, - "WGS84LongitudeAngular": { - "type": "string", - "pattern": "^[EW][0-9]{7}\\.[0-9]{2}$" - }, - "SUPIIMSI": { - "$ref": "#/$defs/IMSI" - }, - "SUPINAI": { - "$ref": "#/$defs/NAI" - }, - "SUCI": { - "type": "string", - "pattern": "^([a-fA-F0-9]{2})*$" - }, - "PEIIMEI": { - "$ref": "#/$defs/IMEI" - }, - "PEIIMEICheckDigit": { - "$ref": "#/$defs/IMEICheckDigit" - }, - "PEIIMEISV": { - "$ref": "#/$defs/IMEISV" - }, - "GPSIMSISDN": { - "type": "string", - "pattern": "^[0-9]{1,15}$" - }, - "GPSINAI": { - "$ref": "#/$defs/NAI" - }, - "NAI": { - "type": "string" - }, - "LDID": { - "type": "string", - "pattern": "^([A-Z]{2}-.+-.+)$" - }, - "InternationalizedEmailAddress": { - "allOf": [ - { - "$ref": "#/$defs/ShortString" - }, - { - "type": "string", - "pattern": "^.+@.+$" - } - ] - }, - "EUI64": { - "type": "string", - "pattern": "^([a-f0-9]{2}:){7}[a-f0-9]{2}$" - }, - "CGI": { - "type": "string", - "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{4}-[a-f0-9]{4}$" - }, - "ECGI": { - "type": "string", - "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{7}$" - }, - "NCGI": { - "type": "string", - "pattern": "^[0-9]{3}-[0-9]{2,3}-[a-f0-9]{9}$" - }, - "ICCID": { - "type": "string", - "pattern": "^[0-9]{19,20}$" - }, - "IPAddress": { - "oneOf": [ - { - "type": "object", - "properties": { - "IPv4Address": { - "$ref": "#/$defs/IPv4Address" - } - }, - "required": [ - "IPv4Address" - ] - }, - { - "type": "object", - "properties": { - "IPv6Address": { - "$ref": "#/$defs/IPv6Address" - } - }, - "required": [ - "IPv6Address" - ] - } - ] - }, - "IPCIDR": { - "oneOf": [ - { - "type": "object", - "properties": { - "IPv4CIDR": { - "$ref": "#/$defs/IPv4CIDR" - } - }, - "required": [ - "IPv4CIDR" - ] - }, - { - "type": "object", - "properties": { - "IPv6CIDR": { - "$ref": "#/$defs/IPv6CIDR" - } - }, - "required": [ - "IPv6CIDR" - ] - } - ] - }, - "TCPPortRange": { - "type": "object", - "properties": { - "start": { - "$ref": "#/$defs/TCPPort" - }, - "end": { - "$ref": "#/$defs/TCPPort" - } - }, - "required": [ - "start", - "end" - ] - }, - "UDPPortRange": { - "type": "object", - "properties": { - "start": { - "$ref": "#/$defs/UDPPort" - }, - "end": { - "$ref": "#/$defs/UDPPort" - } - }, - "required": [ - "start", - "end" - ] - }, - "Port": { - "oneOf": [ - { - "type": "object", - "properties": { - "TCPPort": { - "$ref": "#/$defs/TCPPort" - } - }, - "required": [ - "TCPPort" - ] - }, - { - "type": "object", - "properties": { - "UDPPort": { - "$ref": "#/$defs/UDPPort" - } - }, - "required": [ - "UDPPort" - ] - } - ] - }, - "PortRange": { - "oneOf": [ - { - "type": "object", - "properties": { - "TCPPortRange": { - "$ref": "#/$defs/TCPPortRange" - } - }, - "required": [ - "TCPPortRange" - ] - }, - { - "type": "object", - "properties": { - "UDPPortRange": { - "$ref": "#/$defs/UDPPortRange" - } - }, - "required": [ - "UDPPortRange" - ] - } - ] - }, - "IPAddressPort": { - "type": "object", - "properties": { - "address": { - "$ref": "#/$defs/IPAddress" - }, - "port": { - "$ref": "#/$defs/Port" - } - }, - "required": [ - "address", - "port" - ] - }, - "IPAddressPortRange": { - "type": "object", - "properties": { - "address": { - "$ref": "#/$defs/IPAddress" - }, - "portRange": { - "$ref": "#/$defs/PortRange" - } - }, - "required": [ - "address", - "portRange" - ] - }, - "WGS84CoordinateDecimal": { - "type": "object", - "properties": { - "latitude": { - "$ref": "#/$defs/WGS84LatitudeDecimal" - }, - "longitude": { - "$ref": "#/$defs/WGS84LongitudeDecimal" - } - }, - "required": [ - "latitude", - "longitude" - ] - }, - "WGS84CoordinateAngular": { - "type": "object", - "properties": { - "latitude": { - "$ref": "#/$defs/WGS84LatitudeAngular" - }, - "longitude": { - "$ref": "#/$defs/WGS84LongitudeAngular" - } - }, - "required": [ - "latitude", - "longitude" - ] - } - } -} \ No newline at end of file diff --git a/103705/schema/etsi_entities.schema.json b/103705/schema/etsi_types.schema.json similarity index 68% rename from 103705/schema/etsi_entities.schema.json rename to 103705/schema/etsi_types.schema.json index 9d76005..34e34a7 100644 --- a/103705/schema/etsi_entities.schema.json +++ b/103705/schema/etsi_types.schema.json @@ -1,8 +1,8 @@ { - "$id": "http://etsi.org/temp/705/entities", + "$id": "etsi.org_ts103705_1.3.1_etsi-types", "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "ETSI TS 103 705", - "description": "JSON schema for ETSI-defined entities (see Annex A)", + "title": "ETSI TS 103 705 Types schema", + "description": "JSON schema for ETSI-defined types (see clause 6.1 and Annex A)", "$defs": { "Pointer" : { "type" : "object", @@ -14,10 +14,10 @@ "title" : "Pointer Destination", "description" : "Identifies the Record in the RecordSet containined the referred-to data (see clause 5.3.2 and A.2.2)" }, - "pointerType" : { + "relationship" : { "type" : "string", - "title" : "Pointer type", - "Description" : "Indicates the nature of the relationship. Valid values should be indicated by the CSP schema" + "title" : "Relationship", + "description" : "Indicates the nature of the relationship. Valid values should be indicated by the CSP schema" }, "required" : ["destination"] } @@ -35,12 +35,12 @@ "startDate": { "title" : "Subscription Start Date", "description" : "Date that the subscription started", - "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + "$ref": "ts_103280_2017_07#/$defs/UTCDateTime" }, "endDate": { "title" : "Subscription End Date", "description" : "Date that the subscription ended", - "$ref": "http://etsi.org/temp/280#/$defs/UTCDateTime" + "$ref": "ts_103280_2017_07#/$defs/UTCDateTime" } }, "required" : ["subscriptionId"] @@ -50,25 +50,30 @@ "title": "Person Details", "description": "Details about a person", "properties": { - "firstName": { - "title" : "First Name", - "description" : "First name or given name", + "forename": { + "title" : "Forename", + "description" : "Forename(s)", "type": "string" }, - "middeNames": { + "middeName": { "title" : "First Name", - "description" : "Middle names, given as a space-delimited string", + "description" : "Middle name(s), given as a space-delimited string", "type": "string" }, "lastName" : { - "title" : "Last Name", - "description" : "Lastname or family name", + "title" : "Family Name", + "description" : "Family name(s)", "type" : "string" }, "birthDate" : { "title" : "Date of Birth", "description" : "Date of birth (time part can be ignored)", - "$ref" : "http://etsi.org/temp/280#/$defs/DateTime" + "$ref" : "ts_103280_2017_07#/$defs/DateTime" + }, + "gender" : { + "title" : "Gender", + "description" : "Gender of the person", + "type" : "string" } }, "required" : ["lastName"] diff --git a/103705/schema/response.schema.json b/103705/schema/response.schema.json index 95bb81d..57e0e12 100644 --- a/103705/schema/response.schema.json +++ b/103705/schema/response.schema.json @@ -1,5 +1,5 @@ { - "$id": "http://etsi.org/temp/705", + "$id": "etsi.org_ts103705_1.3.1", "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ETSI TS 103 705 Response", "description": "Overall Response structure, representing the results of a lawful disclosure query (see clause 5)", @@ -24,11 +24,12 @@ "properties": { "schemaId" : { "$ref" : "#/$defs/schemaId" }, "resultSetId" : { "$ref" : "#/$defs/resultSetId"}, - "queryReference" : { "type" : "#/$defs/queryReference" }, - "created" : {"$ref" : "http://etsi.org/temp/280#/$defs/QualifiedMicrosecondDateTime"} + "queryReference" : { "$ref" : "#/$defs/queryReference" }, + "created" : {"$ref" : "ts_103280_2017_07#/$defs/QualifiedMicrosecondDateTime"}, + "recordTypes" : {"$ref" : "#/$defs/recordTypes"} }, "required": [ - "schemaId", "resultSetId", "queryReference", "created" + "schemaId", "resultSetId", "queryReference", "created", "recordTypes" ] }, "schemaId": { @@ -43,24 +44,33 @@ "recordSet": { "type": "array", "title": "RecordSet", - "description": "RecordSet type (see clause 5.3.1). Contains a set of Records.", + "description": "RecordSet (see clause 5.3). Contains a set of Records.", "items": { "$ref": "#$defs/record" } }, + "recordTypes" : { + "type" : "object", + "title" : "RecordTypes", + "description" : "Dictionary of types used by the Response document (see clause 5.2.6)" + }, "record": { "type": "object", "title": "Record", - "description": "Record type (see clause 5.3.1). It is expected that each CSP schema will provide a concrete set of extended Record types that may be used here, each of which shall have a schema referenced using the typesUsed field in the recordDescription", + "description": "Response record (see clause 5.3)", "properties": { "id": { "title": "id", "description": "Unique identifier for the Record within the RecordSet (see clause 5.3.2).", "type": "string" + }, + "type" : { + "title" : "type", + "description" : "Type identifier, shall be mapped to a schema reference via the recrdTypes map (see clauses 5.2.6 and 5.3.3)" } }, "required": [ - "id" + "id", "type" ] } } diff --git a/103705/validate.py b/103705/validate.py index a290ed3..fa54909 100644 --- a/103705/validate.py +++ b/103705/validate.py @@ -50,8 +50,8 @@ if __name__ == "__main__": parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") parser.add_argument('schema', help="Primary schema to validate against") - parser.add_argument('filename', help="JSON instance document to validate") args = parser.parse_args() @@ -65,7 +65,8 @@ if __name__ == "__main__": logging.debug(f"Arguments: {args}") - instance_doc = load_json(args.filename) + instance_doc = json.loads(args.input.read()) + args.input.close() main_schema = load_json(args.schema) schema_dict = { main_schema['$id'] : main_schema } diff --git a/103705/validation/validate_705.py b/103705/validation/validate_705.py new file mode 100644 index 0000000..bb6913d --- /dev/null +++ b/103705/validation/validate_705.py @@ -0,0 +1,97 @@ +import sys +from jsonschema import validate, RefResolver, Draft202012Validator +import json +from pathlib import Path +import logging +import argparse + +def handle_uri(u): + print(u) + +def load_json(path : str): + with open(path) as f: + return json.load(f) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-s','--schemadir', action="append", help="Directory containing supporting schema files to use for validation") + parser.add_argument('-v', '--verbose', action="count", help="Verbose logging (can be specified multiple times)") + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help="Path to input file (if absent, stdin is used)") + + args = parser.parse_args() + + match args.verbose: + case v if v and v >= 2: + logging.basicConfig(level=logging.DEBUG) + case 1: + logging.basicConfig(level=logging.INFO) + case _: + logging.basicConfig(level=logging.WARNING) + + logging.debug(f"Arguments: {args}") + + instance_doc = json.loads(args.input.read()) + args.input.close() + + config = { + 'schema_include_dirs' : [ + '../schema/', + '../../103280/', + ], + 'main_schema_doc' : '../schema/response.schema.json' + } + + rootPath = Path(sys.argv[0]).parent + main_schema = load_json(str(rootPath / config['main_schema_doc'])) + schema_dict = { main_schema['$id'] : main_schema } + + schema_paths = [] + for d in config['schema_include_dirs']: + schema_paths += [f for f in (rootPath / Path(d)).rglob("*.schema.json")] + logging.info(f"Core schema files loaded: {schema_paths}") + if args.schemadir: + for d in args.schemadir: + schema_paths += [f for f in Path(d).rglob("*.schema.json")] + logging.info(f"CSP schema files loaded: {schema_paths}") + else: + logging.info(f"No CSP schema files loaded") + schemas_json = [json.load(p.open()) for p in schema_paths] + schema_dict = schema_dict | { s['$id'] : s for s in schemas_json } + + logging.info(f"Schema IDs loaded: {[k for k in schema_dict.keys()]}") + + logging.debug (f"Instance doc: {instance_doc}") + logging.debug (f"Main schema: {main_schema}") + + resolver = RefResolver(None, + referrer=None, + store=schema_dict) + + logging.info("Performing ETSI validation") + v = Draft202012Validator(main_schema, resolver=resolver) + v.validate(instance_doc) + + logging.info("Building record type dictionary") + type_dict = instance_doc['recordSetDescription']['recordTypes'] + logging.debug(type_dict) + ref_dict = { k : {"$ref" : v} for k,v in type_dict.items()} + validator_dict = { k : Draft202012Validator(ref_dict[k], resolver=resolver) for k,v in ref_dict.items()} + logging.debug(ref_dict) + + logging.info("Validating records") + for r in instance_doc['recordSet']: + type_key = r['type'] + if type_key not in type_dict.keys(): + logging.error(f"Record {r['id']} has type {type_key}, not in recordType dict") + type_ref = type_dict[type_key] + type_schema_id = type_ref.split('#')[0] + logging.info(f"Using {type_schema_id} to validate {type_ref} in record {r['id']}") + if not (type_key in validator_dict.keys()): + logging.error(f'Type key {type_key} from type {type_ref} in record {r["id"]} not in validator dictionary') + print(ref_dict) + v = validator_dict[type_key] + v.validate(r) + + logging.info("Done") \ No newline at end of file -- GitLab