Commit 401284c8 authored by ankraft's avatar ankraft
Browse files

Merge branch 'toMkdocs' of https://git.onem2m.org/tools/scripts into toMkdocs 

# Conflicts:
#	generateChangemarks/.gitlab-ci.yml
#	spec_on_pages.sh
#	toMkdocs/index.md
parents d6278b9a 209a2b68

generateChangemarks/.gitlab-ci.yml

+41 −95
Original line number Original line Diff line number Diff line
@@ -45,16 +45,18 @@ Word CR: 
      - docs/

      - docs/

    expose_as: 'Word CR'

    expose_as: 'Word CR'





Protect branch:

# Do not protect branch until creation and update of merge request can be differenciated for triggering pipeline, otherwise branch is protected_branches

  stage: generation

# with a single update of a merge request (which is not desired). This should be run only for a creation of merge request

  when: on_success

# Protect branch:

  needs: ["Word CR"]

#   stage: generation

  only:

#   when: on_success

    - merge_requests

#   needs: ["Word CR"]

  script:

#   only:

    - |

#     - merge_requests

     curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"

#   script:



#     - |

#      curl --request POST --header "PRIVATE-TOKEN: ${ACCESS_TOKEN}" "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/protected_branches?name=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"

#





Publish spec:

Publish spec:

  stage: publication

  stage: publication
@@ -62,7 +64,7 @@ Publish spec: 
    - tags

    - tags

  before_script:

  before_script:

    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=master" >> publish_spec.sh

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_spec%2Esh/raw?ref=toMkdocs" >> publish_spec.sh

    - chmod +x publish_spec.sh

    - chmod +x publish_spec.sh

    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/Spec-template%2Edocx/raw?ref=master" >> onem2m_spec_template.docx
@@ -75,84 +77,44 @@ Publish spec: 
    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"

    name: "${SPEC_NAME}_${CI_COMMIT_TAG}"

    paths:

    paths:

      - "*_${CI_COMMIT_TAG}.docx"

      - "*_${CI_COMMIT_TAG}.docx"

      - "*_${CI_COMMIT_TAG}.pdf"

      - "*_${CI_COMMIT_TAG}.epub"





# pages:

#   stage: web

#   when: on_success

#   rules:

#     - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG

#       variables:

#         TAG_NAME: $CI_COMMIT_TAG

#     - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"

#       variables:

#         TAG_NAME: $CLEAN_WEB_PAGES

#     - if: $CI_PIPELINE_SOURCE == "trigger"

#       variables:

#         TAG_NAME: "upgrade"

#   before_script:

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/publish_on_pages%2Esh/raw?ref=toMkdocs" >> publish_on_pages.sh

#     - chmod +x publish_on_pages.sh

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/updateIndex%2Epy/raw?ref=toMkdocs" >> updateIndex.py

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/index%2Ehtml/raw?ref=master" >> index.html

#     - |

#      export SPEC_NAME=$(ls | grep T*.md | cut -d'.' -f1)

#     - |

#      sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' index.html

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Ffavicon%2Eico/raw?ref=toMkdocs" >> favicon.html

#     - mkdir -p docs/images && mv favicon.html docs/images/

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Fonem2m%2Epng/raw?ref=toMkdocs" >> onem2m.png

#     - mv onem2m.png docs/images/

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Fonem2m_sq%2Epng/raw?ref=toMkdocs" >> onem2m_sq.png

#     - mv onem2m_sq.png docs/images/

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fstylesheets%2Fextra%2Ecss/raw?ref=toMkdocs" >> extra.css

#     - mkdir -p docs/stylesheets && mv extra.css docs/stylesheets/

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fmkdocs%2Eyml/raw?ref=toMkdocs" >> mkdocs.yml

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Findex%2Emd/raw?ref=toMkdocs" >> index.md

#     - mv index.md docs/

#     - |

#      curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FtoMkdocs%2Epy/raw?ref=toMkdocs" >> toMkdocs.py

#   script:

#     - echo 'Publishing on pages'

#     - ./publish_on_pages.sh updateIndex.py $CI_PAGES_URL ${CI_PROJECT_NAME} $TAG_NAME index.html $CLEAN_WEB_PAGES $SPEC_NAME toMkdocs.py;

#   artifacts:

#     paths:

#       - public

#

pages:

pages:

  stage: web

  stage: web

  image: python:3.9.18-slim-bullseye

  image: python:3.9.18-slim-bullseye

  tags:

  tags:

    - docker

    - docker

  #needs: ["Publish spec"] Uncomment when finishing testing

  variables:

  variables:

    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

    PAGES_BRANCH: gl-pages

    PAGES_BRANCH: gl-pages

    HTTPS_REMOTE: https://gitlab-ci-token:${MIKE_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git

    HTTPS_REMOTE: https://gitlab-ci-token:${MIKE_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git

    TAG_NAME: ""

    TAG_NAME: ""

    TMPTAG: "none"

    TMPTAG: "none" # Remove/comment when finishing testing





  rules:

  rules:

    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $TMPTAG ## Replace this by CI_COMMIT_TAG when test finished

    # Simulated creation of tag pipeline

    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $TMPTAG != "none" ## Replace this by CI_COMMIT_TAG when test finished

      variables:

        TAG_NAME: $TMPTAG

    # Normal creation of tag pipeline

    - if: ($CLEAN_WEB_PAGES == 'false' || $CLEAN_WEB_PAGES == 'true') && $CI_COMMIT_TAG ## Replace this by CI_COMMIT_TAG when test finished

      variables:

      variables:

        TAG_NAME: $CI_COMMIT_TAG

        TMPTAG: $CI_COMMIT_TAG

    # Pipeline triggered from web to remove some docs versions -> CLEAN_WEB_PAGES set to an existing tag name

    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"

    - if: ($CLEAN_WEB_PAGES != "false" && $CLEAN_WEB_PAGES != "true") && $CI_PIPELINE_SOURCE == "web"

      variables:

      variables:

        TAG_NAME: $CLEAN_WEB_PAGES

        TAG_NAME: $CLEAN_WEB_PAGES

    # Upgrade of index.html (not used anymore)

    - if: $CI_PIPELINE_SOURCE == "trigger"

    - if: $CI_PIPELINE_SOURCE == "trigger"

      variables:

      variables:

        TAG_NAME: "upgrade"

        TAG_NAME: "upgrade"





  before_script:

  before_script:

#    - pip install -q mkdocs-material mike rich mkdocs-rss-plugin

    # Installation of required software

    - apt-get update -qq && apt-get -qq install -y git curl > /dev/null

    - apt-get update -qq && apt-get -qq install -y git curl > /dev/null

    # Retrieval of required files from tools/scripts

    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Ffavicon%2Eico/raw?ref=toMkdocs" >> favicon.html

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Ffavicon%2Eico/raw?ref=toMkdocs" >> favicon.html

    - mkdir -p docs/images && mv favicon.html docs/images/

    - mkdir -p docs/images && mv favicon.html docs/images/
@@ -162,14 +124,23 @@ pages: 
    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Fonem2m_sq%2Epng/raw?ref=toMkdocs" >> onem2m_sq.png

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2Fonem2m_sq%2Epng/raw?ref=toMkdocs" >> onem2m_sq.png

    - mv onem2m_sq.png docs/images/

    - mv onem2m_sq.png docs/images/

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FWord%2Epng/raw?ref=toMkdocs" >> Word.png

    - mkdir -p docs/download/images && mv Word.png docs/download/images/

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FEpub%2Epng/raw?ref=toMkdocs" >> Epub.png

    - mv Epub.png docs/download/images/

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fimages%2FPdf%2Epng/raw?ref=toMkdocs" >> Pdf.png

    - mv Pdf.png docs/download/images/

    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fstylesheets%2Fextra%2Ecss/raw?ref=toMkdocs" >> extra.css

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fstylesheets%2Fextra%2Ecss/raw?ref=toMkdocs" >> extra.css

    - mkdir -p docs/stylesheets && mv extra.css docs/stylesheets/

    - mkdir -p docs/stylesheets && mv extra.css docs/stylesheets/

    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fmkdocs%2Eyml/raw?ref=toMkdocs" >> mkdocs.yml

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Fmkdocs%2Eyml/raw?ref=toMkdocs" >> mkdocs.yml

    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2Findex%2Emd/raw?ref=toMkdocs" >> index.md

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FindexDownload%2Emd/raw?ref=toMkdocs" >> indexDownload.md

    - mv index.md docs/

    - mkdir -p docs/download && mv indexDownload.md docs/download/index.md

    - |

    - |

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FtoMkdocs%2Epy/raw?ref=toMkdocs" >> toMkdocs.py

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/toMkdocs%2FtoMkdocs%2Epy/raw?ref=toMkdocs" >> toMkdocs.py

    - |

    - |
@@ -178,35 +149,10 @@ pages: 
     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/spec_on_pages%2Esh/raw?ref=toMkdocs" >> spec_on_pages.sh

     curl "${CI_API_V4_URL}/projects/$TOOLS_SCRIPTS_PROJECT_ID/repository/files/spec_on_pages%2Esh/raw?ref=toMkdocs" >> spec_on_pages.sh

    - chmod +x spec_on_pages.sh

    - chmod +x spec_on_pages.sh

    - echo "'$TAG_NAME'"

    - echo "'$TAG_NAME'"

#    - echo $GITLAB_USER_NAME



#    - echo $GITLAB_USER_EMAIL

#    - git config --global --replace-all user.name "$GITLAB_USER_NAME"

#    - git config --global --replace-all user.email $GITLAB_USER_EMAIL

#    - git fetch origin $PAGES_BRANCH && git checkout $PAGES_BRANCH || git checkout -b $PAGES_BRANCH origin/$PAGES_BRANCH || echo "Pages branch not deployed yet."

#    - git checkout $CI_COMMIT_SHA

  script:

  script:

 #    - python3 toMkdocs.py --title ${CI_PROJECT_NAME^^} "./$SPEC_NAME.md"

 #    - mv ${CI_PROJECT_NAME^^}/ docs/

 #   - cat _nav.yml >> mkdocs.yml

#     - |

#       if [ -n "$TAG_NAME" ]; then

#         mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u $TAG_NAME latest

#         mike set-default --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH latest

#       elif [ "$CI_COMMIT_REF_NAME" == "develop" ]; then

#         mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u $CI_COMMIT_BRANCH

#       fi

 #   - mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u $TAG_NAME latest

 #   - mike set-default --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH latest

 #   - git checkout $PAGES_BRANCH -- public/

#    - ./spec_on_pages.sh toMkdocs.py $MIKE_ACCESS_TOKEN $CI_SERVER_HOST $CI_PROJECT_PATH "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $CI_COMMIT_SHA $SPEC_NAME $CI_PROJECT_NAME $TAG_NAME $CLEAN_WEB_PAGES

#    - ./spec_on_pages.sh toMkdocs.py $MIKE_ACCESS_TOKEN $CI_SERVER_HOST $CI_PROJECT_PATH "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $CI_COMMIT_SHA $SPEC_NAME $CI_PROJECT_NAME $TAG_NAME $CLEAN_WEB_PAGES

    - ./spec_on_pages.sh toMkdocs.py $MIKE_ACCESS_TOKEN $CI_SERVER_HOST $CI_PROJECT_PATH "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $CI_COMMIT_SHA $SPEC_NAME $CI_PROJECT_NAME $TMPTAG $CLEAN_WEB_PAGES

    - ./spec_on_pages.sh toMkdocs.py $MIKE_ACCESS_TOKEN $CI_SERVER_HOST $CI_PROJECT_PATH "$GITLAB_USER_NAME" $GITLAB_USER_EMAIL $CI_COMMIT_SHA $SPEC_NAME $CI_PROJECT_NAME $TMPTAG $CLEAN_WEB_PAGES

  artifacts:

  artifacts:

    paths:

    paths:

      - public/
 
      - public/
 
 No newline at end of file
 
#   rules:

#     - if: '$CI_PIPELINE_SOURCE == "web"'

#       when: always

#     - if: '$CI_COMMIT_TAG'

#     - if: '$CI_COMMIT_REF_NAME == "develop"'

#       when: always

#     - when: never

generateChangemarks/dockerfile.pandoc

+4 −1
Original line number Original line Diff line number Diff line
@@ -2,7 +2,10 @@ FROM pandoc/core:3-ubuntu 




RUN apt-get update -y && \

RUN apt-get update -y && \

    apt-get install -y npm &&\

    apt-get install -y npm &&\

    npm install --global mermaid-filter

    npm install --global mermaid-filter &&\

    apt-get install -y texlive-xetex



#texlive-latex-base texlive-latex-recommended texlive-fonts-recommended





CMD ["/bin/sh"]

CMD ["/bin/sh"]

generateChangemarks/generateTOC.py

+3 −2
Original line number Original line Diff line number Diff line
@@ -61,7 +61,8 @@ def processDocument(args:argparse.Namespace) -> None: 




	# Read the document

	# Read the document

	headers:list[Tuple[str, int]] = []

	headers:list[Tuple[str, int]] = []

	with open(args.document, 'r') as f:

	# Note: We use utf-8 and replace errors to avoid problems with special or unknown characters.

	with open(args.document, 'r', encoding='utf-8', errors='replace') as f:

		document = f.readlines()

		document = f.readlines()

		for line in document:

		for line in document:

			_l = line.strip()

			_l = line.strip()
@@ -95,7 +96,7 @@ def processDocument(args:argparse.Namespace) -> None: 
		backupFile(outDocument)

		backupFile(outDocument)

		tocDone = False

		tocDone = False

		# Add the TOC to the document

		# Add the TOC to the document

		with open(outDocument, 'w') as f:

		with open(outDocument, 'w', encoding='utf-8', errors='replace') as f:

			inToc = False

			inToc = False

			for line in document:

			for line in document:

				# Skip the old TOC when writing

				# Skip the old TOC when writing

generate_changemarks.sh

+1 −1
Original line number Original line Diff line number Diff line
@@ -45,7 +45,7 @@ for i in docs/*.docx ; do 
	docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" addTrackedChanges -o "/tmp/docs/$DOCUMENT_NAME" "/tmp/$i" "$HOST_URL" "$2" "$3"

	docker container run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$GENERATE_CHANGEMARKS_DOCKER_IMAGE" addTrackedChanges -o "/tmp/docs/$DOCUMENT_NAME" "/tmp/$i" "$HOST_URL" "$2" "$3"

done

done

for filename in docs/*.docx; do

for filename in docs/*.docx; do

    mv $filename ${filename%.*}_Word.docx; done;

    mv $filename ${filename%.*}_revisionMarks\(informative\).docx; done;

    #mv "$filename" "prefix_${filename}"; done;

    #mv "$filename" "prefix_${filename}"; done;





echo "\n------ Generating change marks --------"

echo "\n------ Generating change marks --------"

publish_spec.sh

+9 −2
Original line number Original line Diff line number Diff line 
#!/bin/bash

#!/bin/bash





DOCKER_IMAGE=pandoc:master

DOCKER_IMAGE=pandoc:toMkdocs

TOOLS_DOCKER_IMAGE=generatechangemarks:master

TOOLS_DOCKER_IMAGE=generatechangemarks:toMkdocs





echo "\n------ Checking for docker image --------"

echo "\n------ Checking for docker image --------"

docker pull "$DOCKER_IMAGE"

docker pull "$DOCKER_IMAGE"
@@ -34,7 +34,14 @@ for i in *.md ; do 
	    echo "\n------ Preparaing spec --------"

	    echo "\n------ Preparaing spec --------"

	    docker run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$TOOLS_DOCKER_IMAGE" pandocFilter -o "/tmp/" "/tmp/$i"

	    docker run --rm -v $(pwd):/tmp/ -u $(id -u):$(id -g) "$TOOLS_DOCKER_IMAGE" pandocFilter -o "/tmp/" "/tmp/$i"

	    echo "\n------ Publishing spec --------"

	    echo "\n------ Publishing spec --------"

	    # Word output

	    docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -F mermaid-filter -f markdown+escaped_line_breaks -t docx --reference-doc "/data/${3}" -o "${4}_${2}.docx"

	    docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -F mermaid-filter -f markdown+escaped_line_breaks -t docx --reference-doc "/data/${3}" -o "${4}_${2}.docx"

	    #echo 'docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -F mermaid-filter -f markdown+escaped_line_breaks -t pdf -o "${4}_${2}.pdf"'

	    # PDF Output

	    docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -F mermaid-filter -f markdown+escaped_line_breaks -t pdf -o "${4}_${2}.pdf"

	    # EPUB Output

	    docker run --rm -v $(pwd):/data -u $(id -u):$(id -g) "$DOCKER_IMAGE" "/data/$i" -f markdown+escaped_line_breaks -t epub3 -o "${4}_${2}.epub" --metadata title="${PROJECT_NAME}_${2}" --metadata creator="oneM2M Partnership Project" --metadata rights="Copyright 2024 oneM2M Partners

hip Project"

	fi

	fi

done

done