Restructing XSD compilation

103221-1/examples/GetTaskDetailsResponse_example.xml

@@ -16,7 +16,7 @@
         </ns1:targetIdentifiers>
         <ns1:deliveryType>X2andX3</ns1:deliveryType>
         <ns1:listOfDIDs>
-          <ns1:dId>1eb4406a-8b28-41fd-8f0f-e1a0a3eb5ff9</ns1:dId>
+          <ns1:dId>1eb4406a-8b28-41fd-8f0f-e1a0a3eb5ff9foooo</ns1:dId>
         </ns1:listOfDIDs>
       </ns1:taskDetails>
       <ns1:taskStatus>

testing/xsd_compile_targets.json

 [
     {
-        "coreSchema" : "RDMessage.xsd",
+        "coreSchema" : "102657/RDMessage.xsd",
         "supportingSchemas" : [],
         "exampleFiles" : []
+    },
+    {
+        "coreSchema" : "103280/TS_103_280.xsd",
+        "supportingSchemas" : [],
+        "exampleFiles" : []
+    },
+    {
+        "coreSchema" : "103221-1/TS_103_221_01.xsd",
+        "supportingSchemas" : [
+            "103221-1/TS_103_221_01.xsd",
+            "103221-1/TS_103_221_01_HashedID.xsd",
+            "103280/TS_103_280.xsd",
+            "103221-1/examples/ExampleGenericObjects.xsd"
+        ],
+        "exampleFiles" : [
+            "103221-1/examples"
+        ]
     }
 ]
\ No newline at end of file

testing/xsd_process.py

+import json
 import logging
 from pathlib import Path
 
@@ -107,46 +108,110 @@ def processResults (results, stageName):
     return errorCount
 
 
-if __name__ == '__main__':
-    logging.basicConfig(level=logging.DEBUG)
-
-    logging.info('Searching for XSD files')
-    fileList = list(Path(".").rglob("*.xsd")) + list(Path(".").rglob("*.xsd"))
-    logging.info(f'{len(fileList)} XSD files found')
+def syntaxCheckXSD (fileList):
+    results = {}
     for file in fileList:
-        logging.debug(f'  {file}')
+        try:
+            logging.info(f"Syntax checking {str(file)}")
 
-    ignoreList = Path('testing/xsd_ignore.txt').read_text().splitlines()
-    ignoredFiles = []
-    for ignore in ignoreList:
-        logging.debug(f'Ignoring pattern {ignore}')
-        for file in fileList:
-            if ignore in str(file):
-                ignoredFiles.append(file)
-                logging.debug(f" Ignoring {str(file)} as contains {ignore}")
-    ignoredFiles = list(set(ignoredFiles))
-    logging.info(f'{len(ignoredFiles)} files ignored')
-    for file in ignoredFiles:
-        logging.debug(f'  {file}')
-
-    fileList = [file for file in fileList if file not in ignoredFiles]
-    logging.info(f'{len(fileList)} files to process')
-    for file in fileList:
-        logging.debug(f'  {file}')
+            schema = XMLSchema(str(file), validation="skip")
+            results[str(file)] = {
+                'ok' : len(schema.all_errors) == 0,
+                'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, '  ', 20)} - {e.message}"} for e in schema.all_errors]
+            }
+        except XMLSchemaParseError as ex:
+            logging.warning(str(file) + ": Failed validation ({0})".format(ex.message))
+            results[str(file)] = {
+                'ok' : False,
+                'message' : f"{ex!r}"
+            }
+    return results
 
-    if len(fileList) == 0:
-        logging.warning ("No files specified")
-        exit(0)
 
-    logging.info("Parsing ASN1 files")
-    parseResults = syntaxCheckXSD(fileList)
-    if processResults(parseResults, "Parsing") > 0:
-        exit(-1)
+if __name__ == '__main__':
+    #logging.basicConfig(level=logging.DEBUG)
 
-    logging.info ("Getting compile targets")
-    compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text())
-    logging.info (f"{len(compileTargets)} compile targets found")
+    compileTargets = json.loads(Path('testing/xsd_compile_targets.json').read_text())
+    results = {}
+    for target in compileTargets:
+        coreFile = target['coreSchema']
+        logging.info(f"Attempting to compile {coreFile}")
+        schemaLocations = []
+        for supportSchema in target['supportingSchemas']:
+            logging.debug(f"Adding supporting schema {supportSchema}")
+            try:
+                xs = XMLSchema(supportSchema, validation='skip')
+                schemaLocations.append((xs.default_namespace, str(Path(supportSchema).resolve())))
+                logging.info(" [ {0}  ->  {1} ]".format(xs.default_namespace, supportSchema))
+            except Exception as ex:
+                logging.warning (" [ {0} exception parsing:  {1} ]".format(supportSchema, ex))
+                results[coreFile] = {
+                    'ok' : False,
+                    'message' : f"{ex!r}"
+                }
+                break
+        try:
+            schema = XMLSchema(coreFile, locations = schemaLocations, validation="strict")
+            results[coreFile] = {
+                'ok' : len(schema.all_errors) == 0,
+                'message' : None if len(schema.all_errors) == 0 else [{'message' : f"{etree_tostring(e.elem, e.namespaces, '  ', 20)} - {e.message}"} for e in schema.all_errors]
+            }
+            target["schemaInstance"] = schema
+        except Exception as ex:
+            results[coreFile] = {
+                'ok' : False,
+                'message' : f"{ex!r}"
+            }
+            continue
     
-    compileResults = compileAllTargets(compileTargets)
-    if processResults(compileResults, "Compiling") > 0:
+    if (processResults(results, "Compile") > 0):
         exit(-1)
+    
+    results = {}
+
+    for target in compileTargets:
+        schema = target["schemaInstance"]
+        testResults = {}
+        failureCount = 0
+        logging.info (f"Validating example {len(target['exampleFiles'])} entries for {target['coreSchema']}")
+        for example in target["exampleFiles"]:
+            examplePath = Path(example)
+            if examplePath.is_dir:
+                logging.debug (f"Expanding {str(examplePath)}")
+                testFiles = list(examplePath.rglob("./*.xml"))
+            else:
+                testFiles = [examplePath]
+            logging.debug(f"Found {len(testFiles)} test files")
+            for test in testFiles:
+                logging.debug(f"Validating {str(test)} against schema")
+                try:
+                    errors = list(schema.iter_errors(str(test)))
+                    testResults[test] = [f"{etree_tostring(e.elem, e.namespaces, '  ', 20)} - {e.message}" for e in errors]
+                    failureCount += len(errors)
+                except Exception as ex:
+                    testResults[test] = [f"{ex!r}"]                        
+                    failureCount += 1
+        results[target['coreSchema']] = {
+            'ok' : failureCount == 0,
+            'testResults' : testResults,
+            'failureCount' : failureCount
+        }
+    
+    print(f"{'-':-<75}")
+    print(f"Validation results:")
+    print(f"{'-':-<75}")
+
+    totalErrors = 0
+    for filename, result in results.items():
+        print (f"{filename:.<70}{'..OK' if result['ok'] else 'FAIL'}")
+        totalErrors += result['failureCount']
+        for testFile, testResult in result['testResults'].items():
+            print(f"  {str(testFile):.<65}{'..OK' if len(testResult) == 0 else 'FAIL'}")
+            for tr in testResult:
+                print(f"    {tr}")
+
+    print(f"{'-':-<75}")
+    print(f"Validation errors: {totalErrors}")
+    print(f"{'-':-<75}")
+
+    exit(totalErrors > 0)