Pipelines for specifications CR generation

# CI/CD:

#

# GENERATION triggered by:

#  - merge request

#

workflow:

  rules:

    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Creation of Merge Request (generation of CR)

    - if: $CI_PIPELINE_SOURCE == "trigger" # Upgrade of the index.html page

    - if: $CI_COMMIT_TAG # Creation of tag (publish spec)

    - if: $CI_PIPELINE_SOURCE == "web" # Removal of a specific entry

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS

      when: never # Pipeline to be run manually as defined in the contribution procedure as it is not known when the CR is ready

variables:

stages:

  - checking

  - generation

  - publication

  - web

Checking conflicts:

  stage: checking

  rules:

    - if: $CI_MERGE_REQUEST_TITLE !~ /v.*_baseline$/ && $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_MILESTONE

      when: always

      variables:

        MERGE_REQUEST_MILESTONE: $CI_MERGE_REQUEST_MILESTONE

    - if: $CI_MERGE_REQUEST_TITLE !~ /v.*_baseline$/ && $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_MILESTONE == null

      when: always

      variables:

        MERGE_REQUEST_MILESTONE: "all"

  allow_failure: true

  image: python:3.9.18-slim-bullseye

  tags:

    - docker

  before_script:

    # Installation of required software

    - apt-get update -qq && apt-get -qq install -y git curl jq > /dev/null

    - |

     curl "https://git.onem2m.org/api/v4/projects/42/repository/files/checking_conflicts%2Esh/raw?ref=master" >> checking_conflicts.sh

    - chmod +x checking_conflicts.sh

    - git clone "https://oauth2:${SPECIFICATIONS_PIPELINE_TOKEN}@$CI_SERVER_HOST/rep/$CI_PROJECT_PATH.git" ${CI_PROJECT_NAME}

  script:

    - echo 'Checking conflicts'

    - ./checking_conflicts.sh ${CI_API_V4_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${MERGE_REQUEST_MILESTONE} ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} ${CI_MERGE_REQUEST_IID} ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} ${CI_PROJECT_NAME} "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $SPECIFICATIONS_PIPELINE_TOKEN

  artifacts:

    when: on_failure

    paths:

      - conflicting_merge_requests.txt

    expose_as: 'Checking conflicts log'

Word CR:

  stage: generation

  rules:

    - if: $CI_MERGE_REQUEST_TITLE !~ /v.*_baseline$/ && $CI_PIPELINE_SOURCE == "merge_request_event"

      when: on_success

  dependencies: []

  when: on_success

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/specifications%2FgenerateCR%2Fgenerate_changemarks%2Esh/raw?ref=main" >> generate_changemarks.sh

    - chmod +x generate_changemarks.sh

    - |

     curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/specifications%2Fmec_delimiter_start%2Edocx/raw?ref=main" >> mec_delimiter_start.docx

    #- |

    # curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/specifications%2Fmec_delimiter_end%2Edocx/raw?ref=specifications" >> mec_delimiter_end.docx

    - |

     curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/specifications%2Fmec_coversheet_template%2Edocx/raw?ref=main" >> mec_coversheet_template.docx

    #- |

    # curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/specifications%2FSpec-template%2Edocx/raw?ref=specifications" >> mec_spec_template.docx

  script:

    - echo 'CR contribution'

    - mkdir docs

    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} mec_delimiter_start.docx mec_coversheet_template.docx $SPECIFICATIONS_PIPELINE_TOKEN

  artifacts:

    paths:

      - docs/

    expose_as: 'Word CR'

Baseline contribution:

  stage: generation

  rules:

    - if: $CI_MERGE_REQUEST_TITLE =~ /v.*_baseline$/ && $CI_PIPELINE_SOURCE == "merge_request_event"

      when: on_success

  dependencies: []

  when: on_success

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=main" >> publish_spec.sh

    - chmod +x publish_spec.sh

    - |

     curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/specifications%2FSpec-template%2Edocx/raw?ref=main" >> mec_spec_template.docx

    - |

     export SPEC_NAME=$(ls | grep T*.md | cut -d'.' -f1)

     export BASELINE_VERSION=$(echo ${CI_MERGE_REQUEST_TITLE} | grep -o v.*_baseline | cut -d'_' -f1-3 )

  script:

    - echo 'Generating baseline'

    - ./publish_spec.sh ${CI_PROJECT_URL} ${BASELINE_VERSION} mec_spec_template.docx $SPEC_NAME "onlyDocx"

  artifacts:

    name: "Baseline"

    paths:

      - baseline/

    expose_as: 'Baseline'

# Do not protect branch until creation and update of merge request can be differenciated for triggering pipeline, otherwise branch is protected_branches

# with a single update of a merge request (which is not desired). This should be run only for a creation of merge request

# Protect branch:

#   stage: generation

#   when: on_success

#   needs: ["Word CR"]

#   only:

#     - merge_requests

#   script:

#     - |

#      curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"

#

Publish spec:

  stage: publication

  only:

    - tags

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/specifications%2Fpublish_spec%2Esh/raw?ref=main" >> publish_spec.sh

    - chmod +x publish_spec.sh

    - |

     curl "${CI_API_V4_URL}/projects/$PIPELINE_SCRIPTS_PROJECT_ID/repository/files/specifications%2FSpec-template%2Edocx/raw?ref=main" >> mec_spec_template.docx

    - |

     export SPEC_NAME=$(ls | grep -E "(GS|GR).*\.md" | cut -d'.' -f1)

  script:

    - echo 'Publishing spec'

    - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} mec_spec_template.docx $SPEC_NAME

  artifacts:

    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"

    paths:

      - "baseline/*_${CI_COMMIT_TAG}.docx"

      - "baseline/*_${CI_COMMIT_TAG}.pdf"

      - "baseline/*_${CI_COMMIT_TAG}.epub"

#

#	checking_conflicts.sh

#

#	Script to check potential conflicts with existing merge requests 

#

#	(c) 2024 by Miguel Angel Reina Ortega

#	License: BSD 3-Clause License. See the LICENSE file for further details.

#

#!/bin/bash

#Parameters

#${CI_API_V4_URL} -> 1

echo "CI_API_V4_URL:" $1

#${CI_MERGE_REQUEST_PROJECT_ID} -> 2

echo "CI_MERGE_REQUEST_PROJECT_ID:" $2

#${CI_MERGE_REQUEST_MILESTONE} -> 3

echo "CI_MERGE_REQUEST_MILESTONE:" $3

#${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} -> 4

echo "CI_MERGE_REQUEST_TARGET_BRANCH_NAME:" $4

#${CI_MERGE_REQUEST_IID} -> 5

echo "CI_MERGE_REQUEST_IID:" $5

#${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} -> 6

echo "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:" $6

#${CI_PROJECT_NAME} -> 7

echo "CI_PROJECT_NAME:" $7

#"$GITLAB_USER_NAME" -> 8

echo "GITLAB_USER_NAME:" $8

#$GITLAB_USER_EMAIL -> 9

echo "GITLAB_USER_EMAIL:" $9

#echo "------ Removing previous outputs --------"

rm **/*.txt

#echo "------ Parsing repo URL --------"

#HOST_URL=$(echo $1 | cut -d'/' -f 1-3)

#PROJECT_NAME=$(echo $1 | cut -d'/' -f 6- | cut -d'.' -f 1)

#echo "HOST URL:" $HOST_URL

#echo "PROJECT NAME:" $PROJECT_NAME

#echo "PROJECT ID:" $2

#echo "MERGE IID:" $3

#echo "\n------ Install necessary packages --------"

#pip install -q mkdocs-material mike rich mkdocs-rss-plugin

echo "------ Config git and checkout project actual merge request source branch --------"

#git clone "https://oauth2:${10}@git.onem2m.org/specifications/ts/${7}.git" ${7}

cd ${7}

git config --global --replace-all user.name "$8"

git config --global --replace-all user.email $9

echo "Checkout merge request source branch ${6} ..."

git checkout -b "${6}" "origin/${6}" #Git checkout actual merge requests SOURCE BRANCHE

git status

echo "\n------ Getting all potential conflicting merge requests --------"

# Get a list of open merge requests: Filter by the same Milestone and Target Branch as the actual merge request

#curl --header "PRIVATE-TOKEN: ${10}" "${1}/projects/${2}/merge_requests?state=opened&milestone=${3}&target_branch=${4}" |jq -c '.[] | {iid: .iid, source_branch: .source_branch, title: .title}'

#curl --header "PRIVATE-TOKEN: ${10}" "${1}/projects/${2}/merge_requests?state=opened&milestone=${3}&target_branch=${4}" | jq -c '.[]' | while read mr; do

#merge_requests=$(curl --header "PRIVATE-TOKEN: ${10}" "${1}/projects/${2}/merge_requests?state=opened&milestone=${3}&target_branch=${4}")

# Initialize an empty array to store conflicting merge requests

conflicting_merge_request_ids=()

if [ ${3} == "all" ] ; then

  merge_requests=$(curl "${1}/projects/${2}/merge_requests?state=opened&target_branch=${4}")

else

  merge_requests=$(curl "${1}/projects/${2}/merge_requests?state=opened&milestone=${3}&target_branch=${4}")

fi

printf '%s' "${merge_requests}" | jq -c '.[]' | while read -r mr; do

  mr_id=$(printf '%s' "$mr" | jq -r '.iid')

  result_id=$?

  if [ ! $result_id == 0 ] ; then

    echo "ERROR: When retrieving merge request id"

    continue

  fi

  mr_title=$(printf '%s' "$mr" | jq -r '.title')

  result_title=$?

  if [ ! $result_title == 0 ] ; then

    echo "ERROR: When retrieving merge request title"

    continue

  fi

  mr_source_branch=$(printf '%s' "$mr" | jq -r '.source_branch')

  result_source_branch=$?

  if [ ! $result_source_branch == 0 ] ; then

    echo "ERROR: When retrieving merge request source branch"

    continue

  fi

  if [ ${mr_id} != ${5} ] && [ ${mr_id} -lt ${5} ]; then

    echo "Checkout potential merge request source branch ${mr_source_branch} ..."

    git checkout -b "${mr_source_branch}" "origin/${mr_source_branch}" #Git checkout other merge requests SOURCE BRANCHES

    echo "Run dry merge from potential merge request source branch ${mr_source_branch} into actual merge request source branch ${6} ..."

    git merge --no-commit --no-ff "${6}" #SOURCE BRANCH of the merge request

    result=$?

    if [ ! $result == 0 ] ; then

      echo "Merge request title: ${mr_title}, Merge Request ID: ${mr_id}" >> conflicting_merge_requests.txt

      spec=$(git diff --name-only --diff-filter=U | grep -E "(TS|TR)")

      sed -n '

        /^<<<<<<< /,/^>>>>>>> / {

          /^<<<<<<<\|^>>>>>>>/!{=;p}

          /^>>>>>>> /a\

        }

      ' ${spec} >> conflicting_merge_requests.txt

      conflicting_merge_request_ids+=("${mr_id}")

      # Setting label to the merge request which is conflicting with

      echo "Setting label to the merge request which is conflicting with: ${mr_id}"

      #Creating label colored orange

      curl --request POST \

        --header "PRIVATE-TOKEN: ${10}" \

        --data "name=Conflict with merge request ${5}&color=#ed9121" \

        "${1}/projects/${2}/labels"

      #Assign label to the conflicting merge request

      curl --request PUT \

        --silent \

        --header "PRIVATE-TOKEN: ${10}" \

        --data "add_labels=Conflict with merge request ${5}" \

        "${1}/projects/${2}/merge_requests/${mr_id}"

    else

      echo "No conflict with Merge request title: ${mr_title}, Merge Request ID: ${mr_id}"

    fi

    git merge --abort

  fi

done

if [ -f "conflicting_merge_requests.txt" ]; then

  echo "List of conflicting merge requests:"

  cat conflicting_merge_requests.txt

  mv conflicting_merge_requests.txt ../conflicting_merge_requests.txt

  exit 1

else

  exit 0

fi

exit 0

#

#	generate_changemarks.sh

#

#	Script to generate CR Word documents from merge requests. Outputs:

#       - 1 - Word CR document containing the git diff 

#       - 2 - Word CR document containing tracked changes (informative) - Deprecated 

#	(c) 2024 by Miguel Angel Reina Ortega

#	License: BSD 3-Clause License. See the LICENSE file for further details.

#

#!/bin/bash

FORGELIB_DOCKER_IMAGE=forge.3gpp.org:5050/tools/3gpp-scripts/forgelib:v2.21.0

GENERATE_CHANGEMARKS_DOCKER_IMAGE=generatechangemarks:master

DOCKER_IMAGE=pandoc/core:3.1.1.0

#Parameters

#${CI_PROJECT_URL} -> 1

echo "CI_PROJECT_URL:" $1

#${CI_MERGE_REQUEST_PROJECT_ID} -> 2

echo "CI_MERGE_REQUEST_PROJECT_ID:" $2

#${CI_MERGE_REQUEST_IID} -> 3

echo "CI_MERGE_REQUEST_IID:" $3

#${mec_start_delimiter} -> 4

echo "mec_start_delimiter:" $4

#${mec_coversheet_template} -> 5

echo "mec_coversheet_template:" $5

#"$MIKE_ACCESS_TOKEN" -> 6

echo "MIKE_ACCESS_TOKEN:" $6

echo "\n------ Checking for docker image --------"

docker pull "$DOCKER_IMAGE"

echo "------ Removing previous outputs --------"

rm **/*.docx

rm out/*.md

echo "------ Parsing repo URL --------"

HOST_URL=$(echo $1 | cut -d'/' -f 1-4)

PROJECT_NAME=$(echo $1 | grep -o '[^/]*$' | cut -d'.' -f 1)

echo "HOST URL:" $HOST_URL

echo "PROJECT NAME:" $PROJECT_NAME

echo "PROJECT ID:" $2

echo "MERGE IID:" $3

echo "\n------ Generating change marks MD --------"

#docker container run --rm -v $(pwd):/tmp/ "$DOCKER_IMAGE" forgelib-changedocs -vv --combine --diffs --preprocessor onem2m --outPath=/tmp/docs --startdelimiter "/tmp/$4" --enddelimiter "/tmp/$5" --coversheet "/tmp/$6" "$HOST_URL" "$2" "$PROJECT_NAME" "$3"

docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" changemarks -o "/tmp/out" "$HOST_URL" "$2" "$3"

#echo "\n------ Generating changemarks docx --------"

#for i in out/*.md ;  do

#    DOCUMENT_NAME=$(echo $i | cut -d'/' -f 2)

#	echo "\n------ Preparaing spec --------" 

#	docker run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" pandocFilter -o "/tmp/out" "/tmp/$i"

#	echo "\n------ Publishing spec --------" 

#	docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -f markdown -t docx --reference-doc "/data/${7}" -o "/data/out/${DOCUMENT_NAME}.docx"

#   #docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -f markdown -t html -o "/data/out/${DOCUMENT_NAME}.html"

#    #docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/out/${DOCUMENT_NAME}.html" -f html -t docx --reference-doc "Spec-template.docx" -o "/data/out/${DOCUMENT_NAME}.docx"

#done

#echo "\n------ Combining docx --------"

##docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$FORGELIB_DOCKER_IMAGE" forgelib-changedocs -vv -sf "/tmp/out/" --preprocessor onem2m --outPath=/tmp/docs --startdelimiter "/tmp/$4" --enddelimiter "/tmp/$5" --coversheet "/tmp/$6" "$HOST_URL" "$2" "$3"

#docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) -w "/tmp" "$FORGELIB_DOCKER_IMAGE" forgelib-changedocs -vv -sf "out/" --preprocessor onem2m --token ${8} --outPath=docs/ --startdelimiter "$4" --enddelimiter "$5" --coversheet "$6" "$HOST_URL" "$2" "$3"

#echo "\n------ Converting to tracked changes docx --------"

#for i in docs/*.docx ; do

#    DOCUMENT_NAME=$(echo $i | cut -d'/' -f 2)

#	docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" addTrackedChanges -o "/tmp/docs/$DOCUMENT_NAME" "/tmp/$i" "$HOST_URL" "$2" "$3"

#done

#for filename in docs/*.docx; do

#    mv $filename ${filename%.*}_revisionMarks\(informative\).docx; done;

#    #mv "$filename" "prefix_${filename}"; done;

echo "\n------ Generating change marks --------"

docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) -w "/tmp" "$FORGELIB_DOCKER_IMAGE" forgelib-changedocs -vv --combine --diffs --linenumbers --extension "png" --extension "jpg" --extension "svg" --extension "md" --preprocessor mec --token ${6} --outPath=docs/ --startdelimiter "$4" --coversheet "$5" "$HOST_URL" "$2" "$3"

exit 0