diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b43a20857c5fd19de5421b592931db7a423b45b..b4e3df33f5089456fc4d64b9dcd03d40ab38864c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,17 @@ -image: "python:3.7" - -before_script: - - python3 --version - - pip3 install -q asn1tools lxml xmlschema - stages: - - Check Schemas + - check -checkASN1: - stage: Check Schemas +process_asn: + image: "mcanterb/asn1test:latest" + stage: check + interruptible: true script: - - python3 testing/check_asn1.py \ No newline at end of file + - python3 testing/asn_process.py + +process_xsd: + image: "mcanterb/xsdtest:latest" + stage: check + interruptible: true + script: + - python3 testing/xsd_process.py + diff --git a/102232-4/L2AccessPDU.asn b/102232-4/L2AccessPDU.asn index df562af6659bca77e854cad9faa9f5e0148ef827..7f18d897c49c6f59feef3002fd3904d1e10e1ce6 100644 --- a/102232-4/L2AccessPDU.asn +++ b/102232-4/L2AccessPDU.asn @@ -36,9 +36,9 @@ L2CC ::= SEQUENCE -- The ethernet protocol is used ..., l2ATM2684 [6] OCTET STRING, - -- The protocol RFC 2684, method "LLC Encapsulation for Bridged Protocols` [16] is used + -- The protocol RFC 2684, method "LLC Encapsulation for Bridged Protocols" [16] is used l2FR2427 [7] OCTET STRING - -- The protocol RFC 2427 “Multiprotocol Interconnect over Frame Relay” [18] is used + -- The protocol RFC 2427 "Multiprotocol Interconnect over Frame Relay" [18] is used } } @@ -194,7 +194,7 @@ L2ProtocolInformation ::= ENUMERATED l2ProtocolATM2684(7), -- The protocol RFC 2684, method "LLC Encapsulation for Bridged Protocols" [16] is used l2ProtocolFR2427(8) - -- The protocol RFC2427 “Multiprotocol Interconnect over Frame Relay” [18] is used + -- The protocol RFC2427 "Multiprotocol Interconnect over Frame Relay" [18] is used } END -- end of L2AccessPDU diff --git a/103221-1/examples/desktop.ini b/103221-1/examples/desktop.ini deleted file mode 100644 index 8b699c692d358fa29d7ac0976e524fe25c5da053..0000000000000000000000000000000000000000 Binary files a/103221-1/examples/desktop.ini and /dev/null differ diff --git a/testing/asn_compile_targets.json b/testing/asn_compile_targets.json new file mode 100644 index 0000000000000000000000000000000000000000..e186addc0214f2ecf1f25f507336f33725fafe2f --- /dev/null +++ b/testing/asn_compile_targets.json @@ -0,0 +1,33 @@ +[ + [ + "./102232-1/LI-PS-PDU.asn", + "./103280/TS_103_280.asn1", + "./testing/deps/33128/TS33128Payloads.asn", + "./testing/deps/33108/Three3gppHI1Notifications.asn", + "./testing/deps/33108/UMTSHI2Operations.asn", + "./testing/deps/33108/UMTSHI3PS.asn", + "./testing/deps/33108/EpsHI3PS.asn", + "./testing/deps/33108/ConfHI3IMS.asn", + "./testing/deps/33108/VoipHI3IMS.asn", + "./testing/deps/33108/GCSEHI3.asn", + "./testing/deps/33108/CSVoiceHI3IP.asn", + "./testing/deps/33108/UMTSCSHI2Operations.asn", + "./testing/deps/33108/EpsHI2Operations.asn", + "./testing/deps/33108/ConfHI2Operations.asn", + "./testing/deps/33108/ProSeHI2Operations.asn", + "./testing/deps/33108/GCSEHI2Operations.asn", + "./testing/deps/101671/HI1NotificationOperations,ver7.asn", + "./testing/deps/101671/HI2Operations,ver18.asn", + "./testing/deps/101909/TS101909201.asn", + "./testing/deps/101909/TS101909202.asn", + "./testing/deps/101909/PCESP.asn", + "./testing/deps/301040/06132v203_C01.asn", + "./103462/ILHIPDU.asn", + "./102232-2/EmailPDU.asn", + "./102232-3/IPAccessPDU.asn", + "./102232-4/L2AccessPDU.asn", + "./102232-5/IPMultimediaPDU.asn", + "./102232-6/PstnIsdnPDU.asn" + ], + ["./102657/RDMessage.asn"] +] \ No newline at end of file diff --git a/testing/asn_ignore.txt b/testing/asn_ignore.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d81d205c4e0a24ce60da75c4e11e6fd0bae44cb --- /dev/null +++ b/testing/asn_ignore.txt @@ -0,0 +1,2 @@ +deps +temp \ No newline at end of file diff --git a/testing/asn_ignore_lint.txt b/testing/asn_ignore_lint.txt new file mode 100644 index 0000000000000000000000000000000000000000..03e7f54e7937f1cf45bf259cef700a7624bf6912 --- /dev/null +++ b/testing/asn_ignore_lint.txt @@ -0,0 +1 @@ +dependencies \ No newline at end of file diff --git a/testing/asn_process.py b/testing/asn_process.py new file mode 100644 index 0000000000000000000000000000000000000000..aa0a0a63777202d29ee51587ffdafb760c360560 --- /dev/null +++ b/testing/asn_process.py @@ -0,0 +1,206 @@ +import logging +import json +from pathlib import Path +from subprocess import run +from re import sub + +from pycrate_asn1c.asnproc import * + +def reconstrainInteger (filename): + Path('temp.asn').write_text(Path(filename).read_text().replace("18446744073709551615", "65536")) + return 'temp.asn' + +filesWithBigInts = [ + '102232-1/LI-PS-PDU.asn', + '102232-3/IPAccessPDU.asn', + '102232-4/L2AccessPDU.asn' +] + +def syntaxCheckASN (fileList): + """ + Performs ASN syntax checking on a list of filenames (or pathlib Paths) + + :param fileList: List of filenames (str or Pathlib Path) + :returns: Dict with result, return code and message for each filename + + Calls the open-source asn1c compiler with the "syntax only" option. + As a result, asn1c must be available to run. + """ + results = {} + for file in fileList: + try: + if file.as_posix() in filesWithBigInts: + newFile = reconstrainInteger(str(file)) + p = run(['asn1c', '-E', newFile], capture_output=True) + Path(newFile).unlink() + else: + p = run(['asn1c', '-E', str(file)], capture_output=True) + if (p.returncode != 0): + errorMessage = p.stderr.decode().splitlines()[0] + if errorMessage.startswith(' Value "18446744073709551615" at line'): + results[str(file)] = { 'ok' : True} + continue + results[str(file)] = { + 'ok' : False, + 'code' : p.returncode, + 'message' : p.stderr.decode().splitlines()[0] + } + else: + results[str(file)] = { + 'ok' : True + } + except Exception as ex: + results[str(file)] = { + 'ok' : False, + 'code' : -1, + 'message' : f"{ex!r}" + } + return results + + +duplicateObjects = { + '102232-1/LI-PS-PDU.asn' : [ + 'CCPayload', + 'IRIPayload', + 'Location' + ], + 'testing/mod1.asn' : [ + 'ClashField' + ] +} +def fixDuplicateObjects(filename): + stringContent = filename.read_text() + for object in duplicateObjects[filename.as_posix()]: + stringContent = stringContent.replace(f'{object} ::=', f'Native{object} ::=') + stringContent = stringContent.replace(f'SEQUENCE OF {object}', f'SEQUENCE OF Native{object}') + #stringContent = sub(f"]\\w{object}", f"] Native{object}", stringContent) + + Path('temp.asn').write_text(stringContent) + return 'temp.asn' + + +def compileAllTargets (compileTargets): + """ + Attempts to compile a set of compile targets using the pycrate ASN1 tools + + :param compileTargets: list of compile targets, each of which is a list of filenames + :returns: A dict of outcome against the first filename of each compile target. Return code and message are included for failures. + + For each compile target (list of filenames) the first filename is assumed + to be the "primary" file. This doesn't have any relavance to the compilation, + but will be used as the identifier when reporting any compile errors. + The compilation is performed by the pycrate ASN compile functions; errors + are caught as exceptions and rendered into a list. + + Unfortunately, the pycrate compiler doesn't report line numbers. + The asn1c compiler does, but doesn't properly handle identifiers with the + same name in different modules; as this occurs multiple times in TS 33.108, + we can't use it. + """ + results = {} + for target in compileTargets: + firstTarget = target[0] + logging.debug(f"Compiling {firstTarget}") + try: + fileTexts = [] + fileNames = [] + GLOBAL.clear() + for filename in target: + pFile = Path(filename) + if pFile.as_posix() in duplicateObjects: + tmpFile = Path(fixDuplicateObjects(pFile)) + fileTexts.append(tmpFile.read_text()) + #tmpFile.unlink() + else: + fileTexts.append(pFile.read_text()) + fileNames.append(filename) + logging.debug (f" Loading {filename}") + compile_text(fileTexts, filenames = fileNames) + results[str(firstTarget)] = { + 'ok' : True, + } + except Exception as ex: + results[str(firstTarget)] = { + 'ok' : False, + 'code' : -1, + 'message' : f"{ex!r}" + } + continue + return results + + + +def processResults (results, stageName): + """ + Counts the number of errors and writes out the output per filename + + :param results: List of filenames (str or Pathlib Path) + :param stageName: Name to decorate the output with + :returns: The number of files which had errors + """ + print("") + errorCount = sum([1 for r in results.values() if not r['ok']]) + logging.info(f"{errorCount} {stageName} errors encountered") + + print(f"{'-':-<60}") + print(f"{stageName} results:") + print(f"{'-':-<60}") + for filename, result in results.items(): + print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") + if not result['ok']: + if isinstance(result['message'], list): + for thing in result['message']: + print(f" {thing['message']}") + else: + print(f" {result['message']}") + + print(f"{'-':-<60}") + print(f"{stageName} errors: {errorCount}") + print(f"{'-':-<60}") + + return errorCount + + +if __name__ == '__main__': + logging.info('Searching for ASN.1 files') + fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) + logging.info(f'{len(fileList)} ASN.1 files found') + for file in fileList: + logging.debug(f' {file}') + + ignoreList = Path('testing/asn_ignore.txt').read_text().splitlines() + ignoredFiles = [] + for ignore in ignoreList: + logging.debug(f'Ignoring pattern {ignore}') + for file in fileList: + if ignore in str(file): + ignoredFiles.append(file) + logging.debug(f" Ignoring {str(file)} as contains {ignore}") + ignoredFiles = list(set(ignoredFiles)) + logging.info(f'{len(ignoredFiles)} files ignored') + for file in ignoredFiles: + logging.debug(f' {file}') + + fileList = [file for file in fileList if file not in ignoredFiles] + logging.info(f'{len(fileList)} files to process') + for file in fileList: + logging.debug(f' {file}') + + if len(fileList) == 0: + logging.warning ("No files specified") + exit(0) + + logging.info("Parsing ASN1 files") + parseResults = syntaxCheckASN(fileList) + if processResults(parseResults, "Parsing") > 0: + exit(-1) + + logging.info ("Getting compile targets") + compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text()) + logging.info (f"{len(compileTargets)} compile targets found") + + compileResults = compileAllTargets(compileTargets) + if processResults(compileResults, "Compiling") > 0: + exit(-1) + + exit(0) diff --git a/testing/check_asn1.py b/testing/check_asn1.py deleted file mode 100644 index c87a22619f847248a1281a18356dbbfcbfaf6a13..0000000000000000000000000000000000000000 --- a/testing/check_asn1.py +++ /dev/null @@ -1,31 +0,0 @@ -import logging -from compile_asn import * - - -if __name__ == '__main__': - log = logging.getLogger() - log.setLevel(logging.INFO) - parseErrors, compileErrors, parser = validateAllASN1FilesInPath("./") - parseErrorCount = 0 - print ("ASN.1 Parser checks:") - print ("-----------------------------") - for filename, errors in parseErrors.items(): - if len(errors) > 0: - parseErrorCount += len(errors) - print (f"{filename}: {len(errors)} errors") - for error in errors: - print (" " + str(error)) - else: - print (f"{filename}: OK") - print ("-----------------------------") - print ("ASN.1 Compilation:") - print ("-----------------------------") - if len(compileErrors) > 0: - for error in compileErrors: - print (" " + str(error)) - else: - print ("Compilation OK") - print ("-----------------------------") - print (f"{parseErrorCount} parse errors, {len(compileErrors)} compile errors") - exit (parseErrorCount + len(compileErrors)) - diff --git a/testing/check_xsd.py b/testing/check_xsd.py deleted file mode 100644 index 816e3898aa2305c476a665b32c14237c82c22c3d..0000000000000000000000000000000000000000 --- a/testing/check_xsd.py +++ /dev/null @@ -1,194 +0,0 @@ -import logging - -import glob -import sys -import argparse -from pathlib import Path -from pprint import pprint -import os - -from lxml import etree -from xml.etree.ElementTree import ParseError -from xmlschema import XMLSchema, XMLSchemaParseError - -def BuildSchemaDictonary (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return [] - - logging.info("Schema locations:") - schemaLocations = [] - for schemaFile in fileList: - try: - xs = XMLSchema(schemaFile, validation='skip') - schemaLocations.append((xs.target_namespace, str(Path(schemaFile).resolve()))) - logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, schemaFile)) - except ParseError as ex: - logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) - return schemaLocations - - -def BuildSchema (coreFile, fileList = None): - schemaLocations = [] - if fileList and len(fileList) > 0: - schemaLocations = BuildSchemaDictonary(fileList) - - coreSchema = XMLSchema(str(Path(coreFile)), locations=schemaLocations) - return coreSchema - -def ValidateSingleFile (schemaFile): - try: - xs = XMLSchema(schemaFile, validation='skip') - except ParseError as ex: - logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) - return ex - return None - - -def ValidateXSDFiles (fileList): - if len(fileList) == 0: - logging.info("No schema files provided") - return {} - - schemaLocations = BuildSchemaDictonary(fileList) - errors = {} - schemaDictionary = {} - - logging.info("Schema validation:") - for schemaFile in fileList: - try: - schema = XMLSchema(schemaFile, locations = schemaLocations) - logging.info(schemaFile + ": OK") - errors[schemaFile] = [] - schemaDictionary[schema.target_namespace] = schema - except XMLSchemaParseError as ex: - if (ex.schema_url) and (ex.schema_url != ex.origin_url): - logging.info(" Error {1} comes from {0}, suppressing".format(ex.schema_url, ex.message)) - errors[schemaFile] = [] - else: - logging.warning(schemaFile + ": Failed validation ({0})".format(ex)) - errors[schemaFile] = [ex.message] - return errors, schemaDictionary - - -def ValidateInstanceDocuments (coreFile, supportingSchemas, instanceDocs): - if (instanceDocs is None) or len(instanceDocs) == 0: - logging.warning ("No instance documents provided") - return [] - - schema = BuildSchema(coreFile, supportingSchemas) - errors = [] - for instanceDoc in instanceDocs: - try: - schema.validate(instanceDoc) - logging.info ("{0} passed validation".format(instanceDoc)) - except Exception as ex: - logging.error ("{0} failed validation: {1}".format(instanceDoc, ex)) - return errors - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbosity", help="verbosity level", action="count", default=0) - parser.add_argument("input", help="include a directory or file", action="append", nargs="+") - parser.add_argument("-p", "--primaryNamespace", help="Primary schema namespace for instance doc validation") - args = parser.parse_args() - - logging.getLogger().setLevel(logging.WARNING) - if (args.verbosity >= 1): - logging.getLogger().setLevel(logging.INFO) - if (args.verbosity >= 2): - logging.getLogger().setLevel(logging.DEBUG) - logging.debug("Very verbose selected") - - - logging.debug(f"Path: {args.input}") - includeFileList = [] - includeInstanceDocList = [] - for path in args.input[0]: - p = Path(path) - if not p.exists(): - logging.error(f"Include path {path} not found") - exit(1) - if p.is_dir(): - logging.debug(f"Expanding directory") - for g in glob.glob(os.path.join(str(p), "*.xsd")): - logging.info(f">Including {g}") - includeFileList.append(g) - for g in glob.glob(os.path.join(str(p), "*.xml")): - logging.info(f">Including instance doc {g}") - includeInstanceDocList.append(g) - else: - logging.info(f">Including {p.absolute()}") - if str(p.absolute()).endswith('.xml'): - includeInstanceDocList.append(str(p.absolute())) - elif str(p.absolute()).endswith('.xsd'): - includeFileList.append(str(p.absolute())) - else: - logging.warning(f'Ignoring file {p.absolute()}') - - if len(includeInstanceDocList) and (args.primaryNamespace is None): - print("Cannot validate instance documents without specifying a primary namespace (use -h for usage guidelines)") - exit(-1) - - syntaxErrors = 0 - - print ("=============================") - print ("XSD syntax checks:") - print ("-----------------------------") - for file in includeFileList: - error = ValidateSingleFile(file) - if (error): - print (f" {file} : Syntax error [{error}]") - syntaxErrors += 1 - else: - print (f" {file} : OK") - - print ("-----------------------------") - if (syntaxErrors > 0): - print (f"{syntaxErrors} syntax errors detected") - exit(syntaxErrors) - else: - print ("0 syntax errors detected") - - results, schemaDict = ValidateXSDFiles(includeFileList) - - print ("=============================") - print ("XSD build checks:") - print ("-----------------------------") - errorCount = 0 - for fileName, errors in results.items(): - if len(errors) > 0: - errorCount += len(errors) - print (f" {fileName}: {len(errors)} errors") - for error in errors: - if isinstance(error, XMLSchemaParseError): - print (error.msg) - else: - print (f" {str(error.strip())}") - else: - print (f" {fileName}: OK") - - print ("-----------------------------") - print (f"{errorCount} build errors detected") - if (errorCount > 0): - exit(errorCount) - - print ("=============================") - print ("Instance document checks") - print ("-----------------------------") - errorCount = 0 - - primarySchema = schemaDict[args.primaryNamespace] - for instanceDoc in includeInstanceDocList: - try: - results = primarySchema.validate(instanceDoc) - print (f" {instanceDoc} : OK") - except Exception as ex: - errorCount += 1 - print (f" {instanceDoc} : {str(ex)}") - print ("-----------------------------") - print (f"{errorCount} instance doc errors detected") - print ("=============================") - exit(errorCount) diff --git a/testing/compile_asn.py b/testing/compile_asn.py deleted file mode 100644 index e291e42d4077915416bbabba5be97db1cbb7ed50..0000000000000000000000000000000000000000 --- a/testing/compile_asn.py +++ /dev/null @@ -1,140 +0,0 @@ -import logging -import copy -from asn1tools import parse_files, compile_dict, ParseError, CompileError -from glob import glob -from pathlib import Path -from pprint import pprint - -def parseASN1File (asnFile): - try: - parse_files(asnFile) - except ParseError as ex: - return [ex] - return [] - - -def parseASN1Files (fileList): - if len(fileList) == 0: - logging.warning ("No files specified") - return {} - errors = {} - logging.info("Parsing files...") - for f in fileList: - ex = parseASN1File(f) - if ex: - logging.info (f" {f}: Failed - {ex!r}") - else: - logging.info (f" {f}: OK") - errors[f] = ex - return errors - -def fixDottedReference (dict, importingModule, importingType, importingMember, importedModule, importedType): - newName = importedModule + "_" + importedType - dict[importedModule]['types'][newName] = copy.deepcopy(dict[importedModule]['types'][importedType]) - dict[importingModule]['imports'][importedModule].append(newName) - member = [x for x in dict[importingModule]['types'][importingType]['members'] if x is not None and x['name'] == importingMember][0] - member['type'] = newName - -def compileASN1Files (fileList): - logging.info("Compiling files...") - errors = [] - imports = {} - - #p = re.compile(r"]\s+\S+\.\S+") - #for f in fileList: - # with open(f) as fh: - # s = fh.read() - # for match in p.findall(s): - # print (f"In {f}: {match}") - #exit() - - try: - dr = parse_files(fileList) - for modulename, module in dr.items(): - # Weird fix because the compiler doesn't like RELATIVE-OID as a type - # Not sure if the on-the-wire encoding would be affected or not - # but for most checking purposes this doesn't matter - module['types']["RELATIVE-OID"] = {'type' : 'OBJECT IDENTIFIER'} - for k,v in module['imports'].items(): - if not k in imports: - imports[k] = [] - imports[k].append({ - "in" : modulename, - "types" : v - }) - for k,v in imports.items(): - if not k in dr.keys(): - importers = [i['in'] for i in v] - errors.append(f"Unsatisfied import [{k}] for {importers}") - - fixDottedReference(dr, 'LI-PS-PDU', 'Location', 'umtsHI2Location', 'UmtsHI2Operations', 'Location') - fixDottedReference(dr, 'LI-PS-PDU', 'Location', 'epsLocation', 'EpsHI2Operations', 'Location') - fixDottedReference(dr, 'LI-PS-PDU', 'Location', 'eTSI671HI2Location', 'HI2Operations', 'Location') - - fixDottedReference(dr, 'LI-PS-PDU', 'UMTSIRI', 'iRI-Parameters', 'UmtsHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'UMTSIRI', 'iRI-CS-Parameters', 'UmtsCS-HI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'ETSI671IRI', 'iRI-Parameters', 'HI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'EPSIRI', 'iRI-EPS-Parameters', 'EpsHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'ConfIRI', 'iRI-Conf-Parameters', 'CONFHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'ProSeIRI', 'iRI-ProSe-Parameters', 'ProSeHI2Operations', 'IRI-Parameters') - fixDottedReference(dr, 'LI-PS-PDU', 'GcseIRI', 'iRI-Gcse-Parameters', 'GCSEHI2Operations', 'IRI-Parameters') - - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'tTRAFFIC-1', 'TS101909201', 'TTRAFFIC') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cTTRAFFIC-1', 'TS101909201', 'CTTRAFFIC') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'tTRAFFIC-2', 'TS101909202', 'TTRAFFIC') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cTTRAFFIC-2', 'TS101909202', 'CTTRAFFIC') - #fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cCIPPacketHeader', 'CDMA2000CCModule', 'CCIPPacketHeader') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'uMTSCC-CC-PDU', 'Umts-HI3-PS', 'CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'ePSCC-CC-PDU', 'Eps-HI3-PS', 'CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'confCC-CC-PDU', 'CONF-HI3-IMS', 'Conf-CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'voipCC-CC-PDU', 'VoIP-HI3-IMS', 'Voip-CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'gcseCC-CC-PDU', 'GCSE-HI3', 'Gcse-CC-PDU') - fixDottedReference(dr, 'LI-PS-PDU', 'CCContents', 'cSvoice-CC-PDU', 'CSvoice-HI3-IP', 'CSvoice-CC-PDU') - - fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'tARGETACTIVITYMONITOR-1', 'TS101909201', 'TARGETACTIVITYMONITOR-1') - fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'tARGETACTIVITYMONITOR-2', 'TS101909202', 'TARGETACTIVITYMONITOR') - #fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'lAESProtocol', 'Laesp-j-std-025-b', 'LAESProtocol') - #fixDottedReference(dr, 'LI-PS-PDU', 'IRIContents', 'cDMA2000LAESMessage', 'CDMA2000CIIModule', 'CDMA2000LAESMessage') - - fixDottedReference(dr, 'LI-PS-PDU', 'HI4Payload', 'threeGPP-LI-Notification', 'TS33128Payloads', 'LINotificationPayload') - - fixDottedReference(dr, 'ILHIPDU', 'TimestampMapping', 'timeStampQualifier', 'LI-PS-PDU', 'TimeStampQualifier') - - fixDottedReference(dr, 'ILHIPDU', 'ILHITimestamp', 'qualifiedDateTime', 'Common-Parameters', 'QualifiedDateTime') - fixDottedReference(dr, 'ILHIPDU', 'ILHITimestamp', 'qualifiedMicrosecondDateTime', 'Common-Parameters', 'QualifiedMicrosecondDateTime') - - fixDottedReference(dr, 'ILHIPDU', 'OriginalTimestamp', 'microSecondTimeStamp', 'LI-PS-PDU', 'MicroSecondTimeStamp') - - fixDottedReference(dr, 'ILHIPDU', 'LocationMapping', 'originalLocation', 'LI-PS-PDU', 'Location') - - fixDottedReference(dr, 'ILHIPDU', 'GeocodedLocationData', 'wGS84CoordinateDecimal', 'Common-Parameters', 'WGS84CoordinateDecimal') - fixDottedReference(dr, 'ILHIPDU', 'GeocodedLocationData', 'wGS84CoordinateAngular', 'Common-Parameters', 'WGS84CoordinateAngular') - - c = compile_dict(dr) - except CompileError as ex: - logging.info (f"Compiler error: {ex}") - errors.append(ex) - return errors, None - except ParseError as ex: - logging.info (f"Parse error: {ex}") - errors.append(ex) - return errors, None - logging.info ("Compiled OK") - return errors, c - - -def validateASN1Files (fileList): - parseErrors = parseASN1Files(fileList) - errorCount = sum([len(v) for k,v in parseErrors.items()]) - if errorCount > 0: - logging.info ("Abandoning compile due to parse errors") - return parseErrors, [], None - compileErrors, parser = compileASN1Files(fileList) - return parseErrors, compileErrors, parser - - -def validateAllASN1FilesInPath (path): - p = Path(path) - fileGlob = [str(f) for f in p.rglob('*.asn')] - fileGlob += [str(f) for f in p.rglob('*.asn1')] - return validateASN1Files(fileGlob) \ No newline at end of file diff --git a/testing/deps/301040/06132v203_C01.asn b/testing/deps/301040/06132v203_C01.asn index 56305a2033eb1305d0b89a5d14d9443f3b7a361f..30c32121a5f83b5ce70fa71d29289f9fea022837 100644 --- a/testing/deps/301040/06132v203_C01.asn +++ b/testing/deps/301040/06132v203_C01.asn @@ -56,9 +56,9 @@ LISTATUSind ::= SEQUENCE TARGETACTIVITYMONITORind ::= SEQUENCE { - tLIInstanceid TLIIdType, -- header, who – - timestamp UTCTime, -- header, when – - targetLocation LocationType, -- header, where – + tLIInstanceid TLIIdType, -- header, who + timestamp UTCTime, -- header, when + targetLocation LocationType, -- header, where targetAction ActivityType, supplementaryTargetaddress AddressType OPTIONAL, cotargetaddress SEQUENCE OF AddressType OPTIONAL, @@ -178,16 +178,16 @@ TETRAAddressType ::= CHOICE { tETRAaddress TSIType, pISNaddress NumericString (SIZE (20)), - iP4address BIT STRING (SIZE (32)), -- 32 bits – - iP6address BIT STRING (SIZE (128)), -- 128 bits – + iP4address BIT STRING (SIZE (32)), -- 32 bits + iP6address BIT STRING (SIZE (128)), -- 128 bits e164address NumericString (SIZE (20)), tEI TEIType } -CellIdType ::= BIT STRING (SIZE (16)) -- 16 bits – +CellIdType ::= BIT STRING (SIZE (16)) -- 16 bits -LocationAreaType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-2 – +LocationAreaType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-2 LocationType ::= CHOICE { @@ -196,11 +196,11 @@ LocationType ::= CHOICE } -MCCType ::= BIT STRING (SIZE (10)) -- 10 bits, as defined in ETS 300 392-1 – +MCCType ::= BIT STRING (SIZE (10)) -- 10 bits, as defined in ETS 300 392-1 -MNCType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-1 – +MNCType ::= BIT STRING (SIZE (14)) -- 14 bits, as defined in ETS 300 392-1 -SSIType ::= BIT STRING (SIZE (24)) -- 24 bits, as defined in ETS 300 392-1 – +SSIType ::= BIT STRING (SIZE (24)) -- 24 bits, as defined in ETS 300 392-1 CircuitIdType ::= NumericString (SIZE (20)) @@ -255,7 +255,7 @@ TETRACGIType ::= SEQUENCE cI CellIdType OPTIONAL } -TLIIdType ::= BIT STRING (SIZE (16)) -- 16 bits – +TLIIdType ::= BIT STRING (SIZE (16)) -- 16 bits TSIType ::= SEQUENCE { @@ -264,6 +264,6 @@ TSIType ::= SEQUENCE ssi SSIType } -TEIType ::= BIT STRING (SIZE (60)) -- 60 bits, as defined in ETS 300 392-1 – +TEIType ::= BIT STRING (SIZE (60)) -- 60 bits, as defined in ETS 300 392-1 END diff --git a/testing/dockerfile_asn b/testing/dockerfile_asn new file mode 100644 index 0000000000000000000000000000000000000000..2a4198b43ab2d126a9db22fc372a3b1dbee8e4e5 --- /dev/null +++ b/testing/dockerfile_asn @@ -0,0 +1,11 @@ +# docker build -t mcanterb/asn1test +# docker push mcanterb/asn1test + +FROM python:3.9-slim-bullseye +RUN apt-get update -y && \ + apt-get install -y asn1c && \ + rm -rf /var/lib/apt/lists/* && \ + pip install --no-cache-dir asn1tools pycrate + + + diff --git a/testing/dockerfile_xsd b/testing/dockerfile_xsd new file mode 100644 index 0000000000000000000000000000000000000000..9ceeee360715afa316f1fc9dd060a6c301015cb6 --- /dev/null +++ b/testing/dockerfile_xsd @@ -0,0 +1,10 @@ +# docker build -t mcanterb/xsdtest +# docker push mcanterb/xsdtest + +FROM python:3.9-slim-bullseye +RUN apt-get update -y && \ + rm -rf /var/lib/apt/lists/* && \ + pip install --no-cache-dir lxml xmlschema + + + diff --git a/testing/merge_test.py b/testing/merge_test.py new file mode 100644 index 0000000000000000000000000000000000000000..b7a82b39c4958ea30d4ac9a96e100e3041fe73c0 --- /dev/null +++ b/testing/merge_test.py @@ -0,0 +1,68 @@ +import os +import pprint +import requests +import json +import subprocess + +crCommitBranch = os.environ.get("CI_COMMIT_REF_NAME", "NOTFOUND") +apiUrl = os.environ.get("CI_API_V4_URL", "https://forge.3gpp.org/rep/api/v4") +projectId = os.environ.get("CI_PROJECT_ID", "13") + +def gapi (query): + url = f"{apiUrl}/projects/{projectId}/{query}" + r = requests.get(url) + return json.loads(r.text) + +def do (commandline): + #print (" Attempting: " + commandline) + completedProc = subprocess.run(commandline, capture_output=True, shell=True) + #print (" STDOUT > " + ("empty" if completedProc.stdout is None else completedProc.stdout.decode('utf-8'))) + #print (" STDERR > " + ("empty" if completedProc.stderr is None else completedProc.stderr.decode('utf-8'))) + #print (f" Completed with code {completedProc.returncode}") + return (completedProc.returncode == 0, completedProc.stdout.decode('utf-8')) + +print ("Searching for corresponding MR...") + +mrs = gapi(f"merge_requests?source_branch={crCommitBranch}&state=opened") +if len(mrs) == 0: + print ("No MR found... aborting") + exit() + +if len(mrs) > 1: + print (f"{len(mrs)} MRs found, 1 expected - aborting") + for m in mrs: + pprint.pprint(m) + exit(-1) + +mr = mrs[0] + +print (f"Found MR {mr['reference']} ({mr['title']})") +print (f"Target branch is {mr['target_branch']}") +print ("Searching for open MRs targeting same branch...") + +mrs = gapi(f"merge_requests?target_branch={mr['target_branch']}&state=opened") +mrs = [m for m in mrs if m['reference'] != mr['reference']] +print (f"{len(mrs)} MRs found") + +mergeConflicts = {} + +for mr in mrs: + source_branch = mr['source_branch'] + print (source_branch) + + try: + do(f"git fetch origin {source_branch}:{source_branch}") + success, errStr = do(f"git merge --no-commit {source_branch}") + if not success: + print ("Merge NOT OK") + mergeConflicts[source_branch] = errStr + else: + print ("Merge OK") + except Exception as ex: + mergeConflicts[source_branch] = str(ex) + raise + finally: + do("git merge --abort") + +print (f"Merge conflicts with following branches: {mergeConflicts}") +exit(len(mergeConflicts.keys())) \ No newline at end of file diff --git a/testing/xsd_compile_targets.json b/testing/xsd_compile_targets.json new file mode 100644 index 0000000000000000000000000000000000000000..6fe2c6869eebbea44d265e6d9010c8729468682e --- /dev/null +++ b/testing/xsd_compile_targets.json @@ -0,0 +1,48 @@ +[ + { + "coreSchema" : "102657/RDMessage.xsd", + "supportingSchemas" : [], + "exampleFiles" : [] + }, + { + "coreSchema" : "103280/TS_103_280.xsd", + "supportingSchemas" : [], + "exampleFiles" : [] + }, + { + "coreSchema" : "103221-1/TS_103_221_01.xsd", + "supportingSchemas" : [ + "103221-1/TS_103_221_01.xsd", + "103221-1/TS_103_221_01_HashedID.xsd", + "103280/TS_103_280.xsd", + "103221-1/examples/ExampleGenericObjects.xsd" + ], + "exampleFiles" : [ + "103221-1/examples" + ] + }, + { + "coreSchema" : "103120/dictionaries/ts_103120_Dictionaries.xsd", + "supportingSchemas" : [], + "exampleFiles" : [ + "103120/dictionaries/ts_103120_ETSIDictionaryDefinitions.xml" + ] + }, + { + "coreSchema" : "103120/schema/ts_103120_Core.xsd", + "supportingSchemas" : [ + "103120/schema/ts_103120_Authorisation.xsd", + "103120/schema/ts_103120_Common.xsd", + "103120/schema/ts_103120_Delivery.xsd", + "103120/schema/ts_103120_Document.xsd", + "103120/schema/ts_103120_Notification.xsd", + "103120/schema/ts_103120_Task.xsd", + "103280/TS_103_280.xsd", + "testing/deps/xmldsig/xmldsig-core-schema.xsd", + "103120/examples/FooServiceSchema.xsd" + ], + "exampleFiles" : [ + "103120/examples" + ] + } +] \ No newline at end of file diff --git a/testing/xsd_ignore.txt b/testing/xsd_ignore.txt new file mode 100644 index 0000000000000000000000000000000000000000..90bfe5d59c3b61099ed6685d7ddd6023be91efe5 --- /dev/null +++ b/testing/xsd_ignore.txt @@ -0,0 +1,3 @@ +deps +portal +temp \ No newline at end of file diff --git a/testing/xsd_process.py b/testing/xsd_process.py new file mode 100644 index 0000000000000000000000000000000000000000..a0efd07b5b43db84e9797e965c8e06d4d213e5b9 --- /dev/null +++ b/testing/xsd_process.py @@ -0,0 +1,222 @@ +import json +import logging +from pathlib import Path + +from xmlschema.etree import etree_tostring +from xmlschema import XMLSchema, XMLSchemaParseError + + +def BuildSchemaDictonary (fileList): + if len(fileList) == 0: + logging.info("No schema files provided") + return [] + + logging.info("Schema locations:") + schemaLocations = [] + for schemaFile in fileList: + try: + xs = XMLSchema(schemaFile, validation='skip') + schemaLocations.append((xs.default_namespace, str(Path(schemaFile).resolve()))) + logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, schemaFile)) + except XMLSchemaParseError as ex: + logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) + return schemaLocations + + +def BuildSchema (coreFile, fileList = None): + schemaLocations = [] + if fileList and len(fileList) > 0: + schemaLocations = BuildSchemaDictonary(fileList) + + coreSchema = XMLSchema(str(Path(coreFile)), locations=schemaLocations) + return coreSchema + + +def ValidateXSDFiles (fileList): + if len(fileList) == 0: + logging.info("No schema files provided") + return {} + + schemaLocations = BuildSchemaDictonary(fileList) + errors = {} + + logging.info("Schema validation:") + for schemaFile in fileList: + try: + schema = XMLSchema(schemaFile, locations = schemaLocations, validation="lax") + logging.info(schemaFile + ": OK") + errors[schemaFile] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in schema.all_errors] + except XMLSchemaParseError as ex: + logging.warning(schemaFile + ": Failed validation ({0})".format(ex.message)) + if (ex.schema_url) and (ex.schema_url != ex.origin_url): + logging.warning(" Error comes from {0}, suppressing".format(ex.schema_url)) + errors[schemaFile] = [] + else: + errors[schemaFile] = [ex] + return errors + + +def ValidateAllXSDFilesInPath (path): + schemaGlob = [str(f) for f in Path(path).rglob("*.xsd")] + return ValidateXSDFiles(schemaGlob) + + +def ValidateInstanceDocuments (coreFile, supportingSchemas, instanceDocs): + if (instanceDocs is None) or len(instanceDocs) == 0: + logging.warning ("No instance documents provided") + return [] + + schema = BuildSchema(coreFile, supportingSchemas) + errors = [] + for instanceDoc in instanceDocs: + try: + schema.validate(instanceDoc) + logging.info ("{0} passed validation".format(instanceDoc)) + except Exception as ex: + logging.error ("{0} failed validation: {1}".format(instanceDoc, ex)) + return errors + + +def processResults (results, stageName): + """ + Counts the number of errors and writes out the output per filename + + :param results: List of filenames (str or Pathlib Path) + :param stageName: Name to decorate the output with + :returns: The number of files which had errors + """ + print("") + errorCount = sum([1 for r in results.values() if not r['ok']]) + logging.info(f"{errorCount} {stageName} errors encountered") + + print(f"{'-':-<60}") + print(f"{stageName} results:") + print(f"{'-':-<60}") + for filename, result in results.items(): + print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") + if not result['ok']: + if isinstance(result['message'], list): + for thing in result['message']: + print(f" {thing['message']}") + else: + print(f" {result['message']}") + + print(f"{'-':-<60}") + print(f"{stageName} errors: {errorCount}") + print(f"{'-':-<60}") + + return errorCount + + +def syntaxCheckXSD (fileList): + results = {} + for file in fileList: + try: + logging.info(f"Syntax checking {str(file)}") + + schema = XMLSchema(str(file), validation="skip") + results[str(file)] = { + 'ok' : len(schema.all_errors) == 0, + 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] + } + except XMLSchemaParseError as ex: + logging.warning(str(file) + ": Failed validation ({0})".format(ex.message)) + results[str(file)] = { + 'ok' : False, + 'message' : f"{ex!r}" + } + return results + + +if __name__ == '__main__': + #logging.basicConfig(level=logging.DEBUG) + + compileTargets = json.loads(Path('testing/xsd_compile_targets.json').read_text()) + results = {} + for target in compileTargets: + coreFile = target['coreSchema'] + logging.info(f"Attempting to compile {coreFile}") + schemaLocations = [] + for supportSchema in target['supportingSchemas']: + logging.debug(f"Adding supporting schema {supportSchema}") + try: + xs = XMLSchema(supportSchema, validation='skip') + schemaLocations.append((xs.target_namespace, str(Path(supportSchema).resolve()))) + logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, supportSchema)) + except Exception as ex: + logging.warning (" [ {0} exception parsing: {1} ]".format(supportSchema, ex)) + results[coreFile] = { + 'ok' : False, + 'message' : f"{ex!r}" + } + break + try: + schema = XMLSchema(coreFile, locations = schemaLocations, validation="strict") + results[coreFile] = { + 'ok' : len(schema.all_errors) == 0, + 'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}"} for e in schema.all_errors] + } + target["schemaInstance"] = schema + except Exception as ex: + results[coreFile] = { + 'ok' : False, + 'message' : f"{ex!r}" + } + continue + + if (processResults(results, "Compile") > 0): + exit(-1) + + results = {} + + for target in compileTargets: + schema = target["schemaInstance"] + testResults = {} + failureCount = 0 + logging.info (f"Validating example {len(target['exampleFiles'])} entries for {target['coreSchema']}") + for example in target["exampleFiles"]: + examplePath = Path(example) + if examplePath.is_dir(): + logging.debug (f"Expanding {str(examplePath)}") + testFiles = list(examplePath.rglob("./*.xml")) + else: + testFiles = [examplePath] + logging.debug(f"Found {len(testFiles)} test files") + for test in testFiles: + logging.debug(f"Validating {str(test)} against schema") + try: + errors = list(schema.iter_errors(str(test))) + testResults[test] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in errors] + failureCount += len(errors) + except Exception as ex: + testResults[test] = [f"{ex!r}"] + failureCount += 1 + results[target['coreSchema']] = { + 'ok' : failureCount == 0, + 'testResults' : testResults, + 'failureCount' : failureCount + } + + print(f"{'-':-<75}") + print(f"Validation results:") + print(f"{'-':-<75}") + + totalErrors = 0 + for filename, result in results.items(): + if len(result['testResults']) == 0: + print (f"{filename:.<70}SKIP (0)") + continue + else: + print (f"{filename:.<70}{'..OK' if result['ok'] else 'FAIL'} ({len(result['testResults'])})") + totalErrors += result['failureCount'] + if result['failureCount'] > 0: + for testFile, testResult in result['testResults'].items(): + print(f" {str(testFile):.<65}{'..OK' if len(testResult) == 0 else 'FAIL'}") + for tr in testResult: + print(f" {tr}") + + print(f"{'-':-<75}") + print(f"Validation errors: {totalErrors}") + print(f"{'-':-<75}") + + exit(totalErrors > 0)