Commit 9c10f5d7 authored by Mark Canterbury's avatar Mark Canterbury
Browse files

Linting JSON

parent a051d5c4

.gitlab-ci.yml

+13 −4
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ workflow: 


stages:

  - preflight

  - check

  - compile

  - lint

  - build
@@ -20,14 +20,14 @@ preflight: 
  script:

    - forgelib-preflight https://$CI_SERVER_HOST $CI_PROJECT_ID $CI_MERGE_REQUEST_IID



process_asn:

compile_asn:

  image: "forge.etsi.org:5050/li/schemas-definitions/asn1test:latest"

  stage: check

  interruptible: true

  script:

    - python3 testing/asn_process.py



process_xsd:

compile_xsd:

  image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest"

  stage: check

  interruptible: true
@@ -57,7 +57,7 @@ process_xsd: 
      echo "┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅┅"

      echo "✅ XSD validation OK for $file"



process_json:

compile_json:

  image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest"

  stage: check

  interruptible: true
@@ -91,9 +91,18 @@ lint_xml: 
  image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest"

  stage: lint

  interruptible: true

  allow_failure: true

  script:

  - python testing/xml/lint_xml.py



lint_xml:

  image: "forge.etsi.org:5050/li/schemas-definitions/forgeschema:latest"

  stage: lint

  interruptible: true

  allow_failure: true

  script:

  - python testing/xml/lint_json.py



generate_artefacts:

  image: "forge.etsi.org:5050/li/schemas-definitions/forgelib"

  stage: build

testing/json/lint_json.py

0 → 100644
+78 −0
Original line number Diff line number Diff line 
from pathlib import Path

from difflib import *

import subprocess



import colorama

from colorama import Fore, Style



colorama.init()



ignore_paths = [Path(x) for x in [

    'testing/deps'

]]



def print_colorized_diff_line(line: str):

    if line.startswith("-"):

        print(f"{Fore.RED}{line}{Style.RESET_ALL}")

    elif line.startswith("+"):

        print(f"{Fore.GREEN}{line}{Style.RESET_ALL}")

    else:

        print(line)



def lint(file : Path):

    completed = subprocess.run(["jq", ".", str(file)], capture_output=True, text=True, encoding="utf8")



    if completed.returncode != 0:

        print (f"  {str(f)}: FAIL")

        print (f"      jq error code {completed.returncode}")

        lines = completed.stderr.splitlines()

        for line in lines:

            print (f"      {line}")

        return len(lines)



    linted_xml = completed.stdout

    orig_xml = file.read_text(encoding="utf8")

    diff = list(unified_diff(orig_xml.splitlines(), linted_xml.splitlines()))

    if len(diff) == 0:

        print (f"✅ {str(f)}")

        return 0

    else:

        print (f"❌ {str(f)}: {len(diff)} linting errors")

        # for d in diff:

        #     print("".join(d))

        for d in diff:

            print_colorized_diff_line(d)

        return len(diff)

    





if __name__ == "__main__":

    root = Path("./")

    files = list(root.rglob("*.json"))



    files_with_errors = 0

    errors = 0



    print ("───────────────────────────────────────────────────────────────────")

    print (f"Linting JSON files...")



    for f in files:

        if len(list(set(f.parents) & set(ignore_paths))) > 0:

            print (f"(Ignoring {f})")

            continue

        new_errors = lint (f)

        errors += new_errors

        files_with_errors += 1 if new_errors > 0 else 0



    print ("───────────────────────────────────────────────────────────────────")

    print (f"Files: {len(files)} ({files_with_errors} with errors)  Total errors: {errors}")

    if (files_with_errors == 0):

        print ("✅ OK")

    else:

        print("❌ Fail")

    print ("───────────────────────────────────────────────────────────────────")



    if (files_with_errors > 0):

        exit(-1)

    else:

        exit(0)
 
 No newline at end of file

testing/xml/lint_xml.py

+20 −3
Original line number Diff line number Diff line
@@ -2,10 +2,23 @@ from pathlib import Path 
from difflib import *

import subprocess



import colorama

from colorama import Fore, Style



colorama.init()



ignore_paths = [Path(x) for x in [

    'testing/deps'

]]



def print_colorized_diff_line(line: str):

    if line.startswith("-"):

        print(f"{Fore.RED}{line}{Style.RESET_ALL}")

    elif line.startswith("+"):

        print(f"{Fore.GREEN}{line}{Style.RESET_ALL}")

    else:

        print(line)



def lint(file : Path):

    completed = subprocess.run(["xmllint", str(file)], capture_output=True, text=True, encoding="utf8")
@@ -38,7 +51,7 @@ if __name__ == "__main__": 
    files_with_errors = 0

    errors = 0



    print ("-------------------------------------------------")

    print ("")

    print (f"Linting XML / XSD files...")



    for f in files:
@@ -49,9 +62,13 @@ if __name__ == "__main__": 
        errors += new_errors

        files_with_errors += 1 if new_errors > 0 else 0



    print ("-------------------------------------------------")

    print ("───────────────────────────────────────────────────────────────────")

    print (f"Files: {len(files)} ({files_with_errors} with errors)  Total errors: {errors}")

    print ("-------------------------------------------------")

    if (files_with_errors == 0):

        print ("✅ OK")

    else:

        print("❌ Fail")

    print ("───────────────────────────────────────────────────────────────────")



    if (files_with_errors > 0):

        exit(-1)