Generate automatically openapi files for uploaded document:

# CI/CD:

#

# BUILD triggered by:

#  - any (branch commit)

#

# VALIDATION triggered by:

#  - any (branch commit)

#  

# GENERATION triggered by:

#  - any (branch commit) for Generate config

#  - merge request for CR contribution

#    

# OPENAPI

#

#  - any (branch commit) for Generate config

#  - merge request for CR contribution

#

workflow:

  rules:

    - if: $CI_PIPELINE_SOURCE == "merge_request_event"

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS

      when: never

    - if: $CI_COMMIT_BRANCH

variables:

  LOGFILE_OPENAPI: "openapi-validation.log"

  GIT_CLEAN_FLAGS: -ffdxq

  #GIT_SSL_NO_VERIFY: "true"

  GIT_STRATEGY: clone

stages:

  - validation

  - generation

  - openapi

Validate OpenAPI:

  stage: validation

  before_script:

    - |

     curl --header "PRIVATE-TOKEN: $PIPELINE_PROJECT_TOKEN" "${CI_API_V4_URL}/projects/694/repository/files/common-openapi%2Fcli-validate-openapis-mec%2Esh/raw?ref=main" >> cli-validate-openapis-mec.sh

    - chmod +x cli-validate-openapis-mec.sh

  script:

    - echo 'Validate OpenAPI'

    - ./cli-validate-openapis-mec.sh .

  artifacts:

    name: "$CI_JOB_NAME"

    when: always

    paths:

      - ${LOGFILE_OPENAPI}

CR contribution:

  stage: generation

  when: on_success

  only:

    - merge_requests

  before_script:

    - |

     curl --header "PRIVATE-TOKEN: $PIPELINE_PROJECT_TOKEN" "${CI_API_V4_URL}/projects/694/repository/files/common-openapi%2Fgenerate_changemarks%2Esh/raw?ref=main" >> generate_changemarks.sh

    - chmod +x generate_changemarks.sh

    # - |

     # curl "${CI_API_V4_URL}/projects/57/repository/files/all%2Fall_delimiter_start%2Edocx/raw?ref=master" >> all_delimiter_start.docx

    # - |

     # curl "${CI_API_V4_URL}/projects/57/repository/files/all%2Fall_delimiter_end%2Edocx/raw?ref=master" >> all_delimiter_end.docx

  script:

    - echo 'CR contribution'

    - mkdir docs

    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID}

    #- ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} all_delimiter_start.docx all_delimiter_end.docx

  artifacts:

    paths:

      - docs/

    expose_as: 'Word document contribution'

Generation of OpenAPI:

  stage: openapi

  before_script:

    - |

     curl --header "PRIVATE-TOKEN: $PIPELINE_PROJECT_TOKEN" "${CI_API_V4_URL}/projects/694/repository/files/common-openapi%2FgenerateOpenapi%2Esh/raw?ref=main" >> generateOpenapi.sh

    - chmod +x generateOpenapi.sh

    - |

     curl --header "PRIVATE-TOKEN: $PIPELINE_PROJECT_TOKEN" "${CI_API_V4_URL}/projects/694/repository/files/common-openapi%2Fproject-config%2Eyaml/raw?ref=main" >> project-config.yaml

    - git clone "https://oauth2:$AUTOMATED_OPENAPI_GENERATION@$CI_SERVER_HOST/rep/$CI_PROJECT_PATH.git" ${CI_PROJECT_NAME}

    - git config user.name ${AUTOMATED_OPENAPI_GENERATION_NAME}

    - git config user.email ${AUTOMATED_OPENAPI_GENERATION_NAME}@noreply.forge.etsi.org

  script: 

    - echo 'Generate OpenAPI'

    - ./generateOpenapi.sh ${CI_PROJECT_NAME} project-config.yaml $CI_COMMIT_BRANCH ${AUTOMATED_OPENAPI_GENERATION_NAME}

  after_script:

    - version=$(cat version.txt)  # Read version from file

    - cd ${CI_PROJECT_NAME}

    - |

     git push -u origin "openapi-update-$version"

    # Create merge request using GitLab API

    - |

     curl --request POST \

       --header "PRIVATE-TOKEN: $AUTOMATED_OPENAPI_GENERATION" \

       --header "Content-Type: application/json" \

       --data "{

         \"id\": \"$CI_PROJECT_ID\",

         \"source_branch\": \"openapi-update-$version\",

         \"target_branch\": \"$CI_COMMIT_BRANCH\",

         \"title\": \"Update OpenAPI files to version $version\",

         \"description\": \"Automatically generated OpenAPI files from specification version $version\",

         \"remove_source_branch\": true

       }" \

       "$CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/merge_requests"

  rules:

    - if: $CI_PIPELINE_SOURCE == "merge_request_event"

      when: never

    - if: $CI_COMMIT_BRANCH

      changes:

        - doc/*.docx

```mermaid

graph TB

  classDef gitlab fill:#f90,stroke:#333,stroke-width:3px;

  classDef doc2oas fill:#00f,stroke:#333,stroke-width:3px;

  uploadDraftProcess["Generation of OpenAPI process"]

  uploadNewDraft["Upload Draft Word document to /doc folder of corresponding repository by the Rapporteur"]

  reviewMR{"Autogenerated Merge-Request review?"}

  applyMR["Apply the Merge-Request by the Rapporteur"]

  tag["Tag repository by the Rapporteur"]

  Fix{"Is a correction needed?"}

  Feedback["Feedback to Working Group"]

  support["Contact support (STF/CTI) to implement fixes"]

  implementEditorials["Implementation of editorials by the Rapporteur"]

  class tag,reviewMR,uploadNewDraft,applyMR gitlab;

  class upload,syntax_validation,fixConfig doc2oas;

  uploadDraftProcess-->uploadNewDraft

  uploadNewDraft --> reviewMR

  reviewMR --Rejected-->Fix

  reviewMR --Approved-->applyMR

  Fix --doc2oas and/or config file-->support

  support-->uploadNewDraft

  Fix --Editorial spec fixes-->implementEditorials

  implementEditorials-->uploadNewDraft

  Fix --Major spec fixes-->Feedback

  applyMR --> tag 

  tag --> stop

```

#!/bin/bash

# Copyright ETSI 2019-2021

# See: https://forge.etsi.org/etsi-forge-copyright-statement.txt

LOGFILE=openapi-validation.log

DOCKER_IMAGE=etsiforge/swagger-cli:4.0.3

DOCKER_IMAGE_LINT=wework/speccy:0.8

DOCKER_IMAGE_REDOC=redocly/openapi-cli:v1.0.0-beta.93

echo -e "\n------ Switching to $1 folder --------"

cd $1

echo "------ Checking for previous logs ------"

if [ -f "$LOGFILE" ] ; then

    echo "Found previous log. Removing it"

    rm "$LOGFILE"

fi

if [ -x "$(command -v docker)" ] ; then

    RUN_IN_DOCKER=0

    echo "Using dockerized validator ($DOCKER_IMAGE_REDOC)"

    docker pull "$DOCKER_IMAGE_REDOC"

	#docker pull -q "$DOCKER_IMAGE" --> Pull quietly only from Ubuntu 20.04

else

    RUN_IN_DOCKER=1

    if [ ! -x "$(command -v swagger-cli)" ] ; then

        echo "Validator swagger-cli not found. Quitting."

        exit 1

    else

        echo "Using local validator ($(which swagger-cli))"

    fi

fi

function validate {

    echo -e "----  Validating $i:  "    

    if [ $RUN_IN_DOCKER == 0 ] ; then

	   bname=$(basename "$1")

       #docker run -v "$(pwd)":"/specs" $DOCKER_IMAGE swagger-cli validate "/specs/$bname"

	   #docker run -v "$(pwd)":"/specs" $DOCKER_IMAGE_LINT lint "/specs/$bname"

       docker run -v "$(pwd)":"/specs" $DOCKER_IMAGE_REDOC lint "/specs/$bname"

    else

        swagger-cli validate "$1"

    fi

}

echo -e "\n------ Validating all YAML files (may takes several minutes) ------"

# If there are no YAML file, simply exit

ls | grep -q yaml

found_yaml=$?

if [ ! $found_yaml ] ; then 

    echo "-- No YAML files."

    exit 0

fi

fres=0

LOG=""

ERR=""

#specfiles command used if we want to exclude definitions file from MEC-10.2 for instance

#specfiles=$(ls -I*definitions.*| egrep "^[^.]*.(json|yaml)")

# for i in $specfiles ; do

for i in ./*.json ./*.yaml ;  do

    MSG=$(validate "$i" 2>&1)

    res=$?

    if [ ! $res == 0 ] ; then 

        ERR="$ERR$MSG\n"

    fi 

    LOG="$LOG$MSG\n"

    fres=$(($fres||$res))

done

echo -e "\n-- Final validator returns $fres." | tee -a $LOGFILE

if [ $fres == 0 ] ; then

   echo "No errors found, all files validate the OpenAPI definition. Below the complete log." | tee -a $LOGFILE

else

   echo "Some errors found, see below the details." | tee -a $LOGFILE

fi

if [ ! "$ERR" == "" ] ; then 

    echo -e "\n---- Errors detected ----" | tee -a $LOGFILE

    echo -e $ERR  | tee -a $LOGFILE

fi

echo -e "\n---- Complete log of validation ----" | tee -a $LOGFILE

echo -e $LOG | tee -a $LOGFILE

echo -e "\n------ Content of the folder ------" | tee -a $LOGFILE

ls | tee -a $LOGFILE

# Exit code needed for jenkins to know the verdict of the build

exit $fres

```mermaid

graph TB

  classDef doc2oas fill:#0ff,stroke:#333,stroke-width:3px;

  start("Start")

  draft["Draft specification stage"]

  upload["Generate the openapi from tools.etsi.org/doc2oas-ie"]

  syntax_validation["Automatic syntax validation"]

  isValidated{"Is syntax valid?"}

  isCompleted{"Is generated openapi completed?"}

  manualReview["Manual review for completeness"]

  fix["Feedback to Working Group"]

  fixConfig["Fix config file and/or spec CRs"]

  isStableDraft{"Draft is stable?"}

  edithelpReview{"editHelp review completed?"}

  uploadDraftProcess["Go to Automatic OpenAPI generation process"]

  class upload,syntax_validation,fixConfig doc2oas;

  start-->draft

  draft-->upload

  upload-->syntax_validation

  syntax_validation --> isValidated

  isValidated--No-->fix

  isValidated--Yes-->manualReview

  fix --> draft

  manualReview --> isCompleted

  isCompleted --No--> fixConfig

  fixConfig --> upload

  isCompleted --Yes--> isStableDraft

  isStableDraft --Yes--> edithelpReview

  isStableDraft --No--> draft

  edithelpReview --No--> isStableDraft

  edithelpReview --Yes--> uploadDraftProcess

```

#!/bin/bash

if [ "$#" -ne 4 ]; then

    echo "Usage: $0 <project-path> <project-config.yaml> <commit-branch> <automated-openapi-generation-name>"

    exit 1

fi

DOC2OAS_DOCKER_IMAGE=doc2oas-ie-sample:master

# Get version from spec file name in root doc directory

cd doc

# Get the pushed .docx file from the Git commit

spec=$(basename $(git diff --name-only HEAD^ HEAD | grep '\.docx$'))

if [ -z "$spec" ]; then 

    echo "-- No spec file found in the latest commit"

    exit 0

fi

version=$(echo "$spec" | grep -e v[0-9]* -o)

if [ -z "$version" ]; then

    echo "No version found in spec filename"

    exit 1

fi

# Write version to a file for CI script to use

echo "$version" > ../version.txt

# Create new branch for the changes in the cloned repo

branch_name="openapi-update-$version"

echo "\n------ Creating new branch $branch_name --------"

cd ../$1  # Go to cloned repo

#Remove uploaded docx file from default branch

git config user.name $4

git config user.email "$4@noreply.forge.etsi.org"

git checkout $3

git rm doc/$spec

git commit -m "Removing uploaded docx file"

git push --push-option="ci.skip" -u origin $3

# Checkout to the existing branch or create it if it does not exist (it does not contain the uploaded docx document)

git checkout "$branch_name" || git checkout -b "$branch_name"

cd ../doc  # Back to root doc directory for processing

echo "\n------ Checking for docker image --------"

#docker pull "$DOCKER_IMAGE"

echo "------ Getting spec and config files--------"

# Get project prefix (e.g., gs030) from the project path

project_prefix=$(echo $1 | grep -o 'gs[0-9]\{3\}')

echo "Project prefix: $project_prefix"

echo "Project path: $1"

echo "Config file: $2"

echo "Branch name: $3"

# Get all config files for this project

# Extract the section between project prefix and the next project or end of file

project_section=$(sed -n "/^[[:space:]]*$project_prefix:/,/^[[:space:]]*gs[0-9]\{3\}:/p" "../$2" | sed '$d')

# Extract config_file values

configs=$(echo "$project_section" | grep "config_file:" | sed 's/.*config_file:[[:space:]]*"\(.*\)".*/\1/')

if [ -z "$configs" ]; then

    echo "No config files found for project $project_prefix"

    exit 1

fi

echo "Found configs: $configs"

# Process each config file

echo "$configs" | while read -r config; do

    echo "Processing config file: $config"

    echo "\n------ Preparing spec for $config --------"

    echo -e "import sys" | tee "command.py"

    echo -e "\nsys.path.append(\"/app\")" | tee -a "command.py"

    echo -e "\nimport main" | tee -a "command.py"

    echo -e "\nif __name__ == '__main__':" | tee -a "command.py"

    echo -e "\n    sys.exit(main.maindoc2oas(\"/app/tmp/$spec\", \"/app/tmp/$config\"))" | tee -a "command.py"

    docker run --rm -v $(pwd):/app/tmp -u $(id -u):$(id -g) $DOC2OAS_DOCKER_IMAGE tmp/command.py

    echo "------ Move generated openapi to the correct folder --------"

    openapi=$(ls | grep gs_*.yaml)

    if [ -z "$openapi" ] ; then 

        echo "-- No openapi file generated for $config"

        continue

    fi

    # Find corresponding openapi name for this config

    # Get the entire config block containing our config file

    config_block=$(echo "$project_section" | awk -v cfg="$config" '

        /[[:space:]]*-/ { in_block=0 }  # Start of new block

        in_block { print }              # Print lines while in block

        $0 ~ "config_file:.*\""cfg"\"" { in_block=1; print }  # Found our config, start block

    ')

    echo "Config block found:"

    echo "$config_block"

    # Then extract the openapi_name from it

    new_name=$(echo "$config_block" | grep "openapi_name" | sed 's/.*openapi_name:[[:space:]]*"\(.*\)".*/\1/')

    echo "Found OpenAPI file name: $new_name"

    if [ ! -z "$new_name" ]; then

        echo "Renaming $openapi to $new_name"

        mv "$openapi" "$new_name"

        openapi="$new_name"

    fi

    # Move openapi to the cloned repo

    mv "$openapi" ../$1/$openapi

    # Add to git in cloned repo

    cd ../$1

    git add $openapi

    cd ../doc

done

echo "\n------ Prepare to push -------"

cd ../$1  # Go to cloned repo for git operations

git commit -m "Autogenerated openapi version $version"

# Git push and merge request creation are handled by the CI file

exit 0

 No newline at end of file