Commit 89e36bd1 authored by Miguel Angel Reina Ortega's avatar Miguel Angel Reina Ortega
Browse files

Add functionality to upgrade index.html in all projects

parent 5ea48eaa

.gitlab-ci.yml

+8 −89
Original line number Diff line number Diff line
@@ -6,20 +6,13 @@ 


workflow:

  rules:

    - if: $CI_PIPELINE_SOURCE == "merge_request_event"

    - if: $CI_COMMIT_TAG

    - if: $CI_COMMIT_BRANCH && $CI_PROJECT_NAME == "scripts"

    - if: $CI_PIPELINE_SOURCE == "web"

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS

      when: never

    - if: $CI_COMMIT_BRANCH



variables:



stages:

  - build

  - generation

  - publication

  - web

  - upgrade



Build pythonForPandocFilter docker image:

  stage: build
@@ -35,85 +28,11 @@ Build pythonForPandocFilter docker image: 
        - generateChangemarks/changemarks.py

        - generateChangemarks/addTrackedChanges.py

        

Word CR text:

  stage: generation

  only:

    - merge_requests

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh

    - chmod +x generate_changemarks.sh

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx

  script:

    - echo 'CR contribution'

    - mkdir docs

    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx

  artifacts:

    paths:

      - docs/

    expose_as: 'Word CR text'



Protect branch:

  stage: generation

  when: on_success

  needs: ["Word CR text"]

  only:

    - merge_requests

  script:

    - |

     curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"





Publish spec:

  stage: publication

  only:

    - tags

Upgrade index for pages:

  stage: upgrade

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh

    - chmod +x publish_spec.sh

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx

    - |

     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)

  script:

    - echo 'Publishing spec'

    - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME

  artifacts:

    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"

    paths:

      - "*_${CI_COMMIT_TAG}.docx"



pages:

  stage: web

  when: on_success

    - ./upgrade_pages.sh ${ACCESS_TOKEN}

  rules:

    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG

      variables:

        TAG_NAME: $CI_COMMIT_TAG

    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"

      variables:

        TAG_NAME: $CLEAN_WEB_PAGES

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh

    - chmod +x publish_on_pages.sh

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html

    - |

     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)

  script:

    - echo 'Publishing on pages'

    - ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME

  artifacts:

    paths:

      - public
 
 No newline at end of file
 
    - changes:

        - index.html
 
 No newline at end of file

generateChangemarks/.gitlab-ci.yml

0 → 100644
+109 −0
Original line number Diff line number Diff line 
# CI/CD:

#

# GENERATION triggered by:

#  - merge request

#



workflow:

  rules:

    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Creation of Merge Request (generation of CR)

    - if: $CI_COMMIT_TAG # Creation of tag (publish spec)

    - if: $CI_PIPELINE_SOURCE == "web" # Removal of a specific entry

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS

      when: never # Pipeline to be run manually as defined in the contribution procedure as it is not known when the CR is ready

    - if: $CI_PIPELINE_SOURCE == "trigger" # Upgrade of the index.html page



variables:



stages:

  - generation

  - publication

  - web



Word CR:

  stage: generation

  only:

    - merge_requests

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/generate_changemarks%2Esh/raw?ref=master" >> generate_changemarks.sh

    - chmod +x generate_changemarks.sh

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_start%2Edocx/raw?ref=master" >> onem2m_delimiter_start.docx

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_delimiter_end%2Edocx/raw?ref=master" >> onem2m_delimiter_end.docx

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/onem2m_coversheet_template%2Edocx/raw?ref=master" >> onem2m_coversheet_template.docx

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx

  script:

    - echo 'CR contribution'

    - mkdir docs

    - ./generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID} onem2m_delimiter_start.docx onem2m_delimiter_end.docx onem2m_coversheet_template.docx onem2m_spec_template.docx

  artifacts:

    paths:

      - docs/

    expose_as: 'Word CR'



Protect branch:

  stage: generation

  when: on_success

  needs: ["Word CR"]

  only:

    - merge_requests

  script:

    - |

     curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}&merge_access_level=30"





Publish spec:

  stage: publication

  only:

    - tags

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh

    - chmod +x publish_spec.sh

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx

    - |

     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)

  script:

    - echo 'Publishing spec'

    - ./publish_spec.sh ${CI_PROJECT_URL} ${CI_COMMIT_TAG} onem2m_spec_template.docx $SPEC_NAME

  artifacts:

    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"

    paths:

      - "*_${CI_COMMIT_TAG}.docx"



pages:

  stage: web

  when: on_success

  rules:

    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG

      variables:

        TAG_NAME: $CI_COMMIT_TAG

    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"

      variables:

        TAG_NAME: $CLEAN_WEB_PAGES

    - if: $CI_PIPELINE_SOURCE == "trigger"

      variables:

        TAG_NAME: "upgrade"

  before_script:

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=master" >> publish_on_pages.sh

    - chmod +x publish_on_pages.sh

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=master" >> updateIndex.py

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html

    - |

     export SPEC_NAME=$(ls | grep *.md | cut -d'.' -f1)

    - |

     sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' index.html

  script:

    - echo 'Publishing on pages'

    -  ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME;

  artifacts:

    paths:

      - public
 
 No newline at end of file

publish_on_pages.sh

+7 −2
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ DOCKER_IMAGE=python:3.9-slim-bullseye 
echo "\n------ Checking for docker image --------"

docker pull "$DOCKER_IMAGE"

echo "Pages URL:" $2

echo "PROJECT NAME:" $3

echo "PROJECT NAME:" $3 # Not used anymore, consider to remove it

echo "TAG NAME:" $4



echo "------ Get the previous GitLab Pages content --------"
@@ -15,7 +15,10 @@ if [ -e "content.zip" ]; then unzip "content.zip"; rm "content.zip"; fi 
set -e



echo "------ Add/update content --------"

if [ $6 == 'true' ]; then 

if [ $4 == 'upgrade' ]; then

    echo 'Upgrading index.html...'

    docker run --rm -v $(pwd):/tmp -w /tmp "$DOCKER_IMAGE" pip install bs4 && python3 $1 -u "$7_$4" "$4/$7_$4.docx" "$5"

elif [ $6 == 'true' ]; then

    echo 'Removing all web pages content...'

	rm -r public/*;

elif [ $6 == 'false' ]; then
@@ -41,3 +44,5 @@ exit 0 
#$CI_PROJECT_NAME

#$CI_COMMIT_REF_NAME

#index.html

#CLEAN_WEB_PAGES

#SPEC_NAME

updateIndex.py

+53 −1
Original line number Diff line number Diff line
@@ -9,6 +9,53 @@ 
from bs4 import BeautifulSoup

import argparse, os





def upgradeIndex(args: argparse.Namespace) -> None:

    # Check if index.htm exists

    if not os.path.exists('public/index.html'):

        print("'index.html' does not exist yet, Upgrade is not needed")

        exit()

    else:

        # Read the source HTML file

        with open('public/index.html', 'r', encoding='utf-8') as source_file:

            source_html = source_file.read()



        # Read the destination HTML file

        with open(args.indexTemplate, 'r', encoding='utf-8') as dest_file:

            dest_html = dest_file.read()



        print(source_html)

        print(dest_html)

        # Parse both HTML files

        source_soup = BeautifulSoup(source_html, 'html.parser')

        dest_soup = BeautifulSoup(dest_html, 'html.parser')



        print(source_soup)

        print(dest_soup)

        # Find the <ul> element in both the source and destination HTML

        source_ul = source_soup.find('ul', id='document-list')

        dest_ul = dest_soup.find('ul', id='document-list')



        print(source_ul)

        print(dest_ul)

        # Copy the contents of the source <ul> to the destination <ul>

        if dest_ul and source_ul:

            dest_ul.contents.extend(source_ul.contents)

        else:

            print("ERROR: List not found in source and destination index files...")

            exit(1)



        # Save the updated destination HTML

        with open('public/index.html', 'w', encoding='utf-8') as dest_updated_file:

            dest_updated_file.write(str(dest_soup))





        # Read index HTML

        with open('public/index.html', 'r', encoding='utf-8') as file:

            html_content = file.read()

        print("Index.html upgraded")





def updateIndex(args:argparse.Namespace) -> None:

    

    # Check if index.htm exists
@@ -63,10 +110,15 @@ if __name__ == '__main__': 
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('-re', '--removeEntry', action='store_true', required=False, default=False,

                        help="Indicate to remove the given entry")

    parser.add_argument('-u', '--upgrade', action='store_true', required=False, default=False,

                        help="Upgrade the index.html")

    parser.add_argument('documentName', help = 'Document name to add to the list')

    parser.add_argument('documentLink', help = 'Document location to add to the list')

    parser.add_argument('indexTemplate', help = 'Index template document')

    args = parser.parse_args()



    if args.upgrade:

        upgradeIndex(args)

    else:

        updateIndex(args)

upgrade_pages.sh

0 → 100755
+30 −0
Original line number Diff line number Diff line 
#!/bin/bash



GITLAB_HOST="http://git.onem2m.org"

SPECIFICATIONS_GROUP="2501"

NS_GITLAB_HOST=$(echo $GITLAB_HOST | cut -d'/' -f3)



echo "\n------ Getting all projects IDs --------"

# Get a list of projects in the group using GitLab API

#PROJECT_IDS=$(curl --header "PRIVATE-TOKEN: $1" "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2)

PROJECT_IDS=$(curl "$GITLAB_HOST/api/v4/groups/$SPECIFICATIONS_GROUP/projects" | grep -oP '"id":(?!'$SPECIFICATIONS_GROUP')\d+' | cut -d':' -f2)

echo $PROJECT_IDS



#PROJECT_IDS="41" # Use TS-0019 for debugging

# Trigger pipeline for each project. Triggered pipeline pages job will handle the upgrade of the index.html

for PROJECT_ID in $PROJECT_IDS; do

	echo "------ Preparing specification project --------"

	#WORKAROUND UNTIL ACCESS TOKEN WORKS FOR TRIGGERING A PIPELINE (right now, it is allowed but it seems NOT to work)

	#create a pipeline trigger token first

	echo "------ Creating pipeline trigger token --------"

	PIPELINE_TRIGGER_TOKEN_ID=$(curl --request POST --header "PRIVATE-TOKEN:$1" --form description="Temporary pipeline trigger token for tools/scripts" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers | grep -o '"id":[^,]*,"token"'|cut -d',' -f1 | cut -d':' -f2)

    PIPELINE_TRIGGER_TOKEN=$(curl --header "PRIVATE-TOKEN:$1" https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID | grep -o '"token":[^,]*'|cut -d'"' -f4)

    echo "------ Triggering pipeline --------"

	#trigger the pipeline

    curl -X POST --fail -F token=$PIPELINE_TRIGGER_TOKEN -F ref=main https://git.onem2m.org/api/v4/projects/$PROJECT_ID/trigger/pipeline

    echo "------ Deleting pipeline trigger token (NOTE that information about the token that caused the pipeline will NOT be available for the pipeline) --------"

	#remove the pipeline trigger token

    curl --request DELETE --header "PRIVATE-TOKEN:$1" "https://git.onem2m.org/api/v4/projects/$PROJECT_ID/triggers/$PIPELINE_TRIGGER_TOKEN_ID"

done



exit 0
 
 No newline at end of file