Loading .gitlab-ci.yml +6 −0 Original line number Diff line number Diff line Loading @@ -64,10 +64,16 @@ Build generateSpecWebSite docker image: - generateSpecWebSite/gridTableFilter.py - generateSpecWebSite/gridTableTools.py - generateSpecWebSite/indexDownload.md - generateSpecWebSite/indexReleaseNotes.md - generateSpecWebSite/markdownTools.py - generateSpecWebSite/regexMatches.py - generateSpecWebSite/spec_on_pages.sh - generateSpecWebSite/toMkdocs.py - generateSpecWebSite/create_frontmatter_table.py - markdownTools/dockerfile - markdownTools/setup.py - markdownTools/requirements.txt - markdownTools/processMDSpec.py Build markdownTools docker image: stage: build Loading generateSpecWebSite/create_frontmatter_table.py 0 → 100644 +100 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # create_frontmatter_table.py # # Creates a markdown table from frontmatter data # - Removes headings # - Removes <br> tags # - Formats as a clean markdown table # # (c) 2025 by Miguel Angel Reina Ortega # License: BSD 3-Clause License. See the LICENSE file for further details. import sys import yaml import re def remove_br_tags(text): """Remove <br> and <br/> tags from text.""" if not text: return text # Replace <br>, <br/>, <br /> with space text = re.sub(r'<br\s*/?>', ' ', text, flags=re.IGNORECASE) # Clean up multiple spaces text = re.sub(r'\s+', ' ', text) return text.strip() def create_table_from_frontmatter(frontmatter_file, output_file=None): """ Create a markdown table from frontmatter data. Args: frontmatter_file: Path to the frontmatter.md file output_file: Optional path to output file (if None, prints to stdout) """ # Read frontmatter with open(frontmatter_file, 'r', encoding='utf-8') as f: content = f.read() frontmatter_content = [] # Split into lines and skip first line if it looks like a filename lines = content.split('\n') for line in lines: # Check if it's a filename line: ends with .md: or ends with : and no space before colon if line.strip().endswith('.md:'): # Skip first line continue else: frontmatter_content.append(line) # Parse YAML frontmatter = yaml.safe_load('\n'.join(frontmatter_content)) if not frontmatter: frontmatter = {} # Generate markdown table (no header row) table_lines = [] table_lines.append("") table_lines.append("| | |") table_lines.append(f"|:-|:-|") # Process each field dynamically for key, value in frontmatter.items(): # Convert value to string and clean it if value is None: value_str = "" elif isinstance(value, (list, dict)): # For lists and dicts, convert to YAML string value_str = yaml.dump(value, default_flow_style=False).strip() else: value_str = str(value) # Remove <br> tags #value_str = remove_br_tags(value_str) # Escape pipe characters in values (they break markdown tables) value_str = value_str.replace('|', '\\|') # Add row to table table_lines.append(f"| **{key}** | {value_str} |") # Join all lines table_lines.append("") table_content = '\n'.join(table_lines) # Output if output_file: with open(output_file, 'w', encoding='utf-8') as f: f.write(table_content) else: print(table_content) if __name__ == '__main__': if len(sys.argv) < 2: print("Usage: create_frontmatter_table.py <frontmatter_file> [output_file]") sys.exit(1) frontmatter_file = sys.argv[1] output_file = sys.argv[2] if len(sys.argv) > 2 else None create_table_from_frontmatter(frontmatter_file, output_file) generateSpecWebSite/dockerfile +7 −0 Original line number Diff line number Diff line Loading @@ -7,3 +7,10 @@ RUN apt-get update -y && \ rm -rf /var/lib/apt/lists/* &&\ pip install -e generateSpecWebSite/ &&\ pip install -r generateSpecWebSite/requirements.txt ADD /markdownTools/ /markdownTools/ RUN apt-get update -y && \ apt-get install -y libcairo2 && \ rm -rf /var/lib/apt/lists/* &&\ pip install -e markdownTools/ &&\ pip install -r markdownTools/requirements.txt generateSpecWebSite/indexReleaseNotes.md 0 → 100644 +6 −0 Original line number Diff line number Diff line Download Release Notes for PROJECT version VERSION: [](SPECWORD) generateSpecWebSite/spec_on_pages.sh +102 −80 Original line number Diff line number Diff line Loading @@ -14,97 +14,119 @@ #toMkdocs.py -> 1 echo "toMkdocs.py:" $1 #$MIKE_ACCESS_TOKEN -> 2 echo "MIKE_ACCESS_TOKEN:" $2 MIKE_ACCESS_TOKEN=${2} echo "MIKE_ACCESS_TOKEN:" ${MIKE_ACCESS_TOKEN} #CI_SERVER_HOST -> 3 echo "CI_SERVER_HOST:" $3 CI_SERVER_HOST=${3} echo "CI_SERVER_HOST:" ${CI_SERVER_HOST} #CI_PROJECT_PATH -> 4 echo "CI_PROJECT_PATH:" $4 CI_PROJECT_PATH=${4} echo "CI_PROJECT_PATH:" ${CI_PROJECT_PATH} #GITLAB_USER_NAME -> 5 echo "GITLAB_USER_NAME:" $5 GITLAB_USER_NAME=${5} echo "GITLAB_USER_NAME:" ${GITLAB_USER_NAME} #GITLAB_USER_EMAIL -> 6 echo "GITLAB_USER_EMAIL:" $6 GITLAB_USER_EMAIL=${6} echo "GITLAB_USER_EMAIL:" ${GITLAB_USER_EMAIL} #CI_COMMIT_SHA -> 7 echo "CI_COMMIT_SHA:" $7 CI_COMMIT_SHA=${7} echo "CI_COMMIT_SHA:" ${CI_COMMIT_SHA} #$SPEC_NAME -> 8 echo "SPEC_NAME:" $8 SPEC_NAME=${8} echo "SPEC_NAME:" ${SPEC_NAME} #$CI_PROJECT_NAME -> 9 echo "CI_PROJECT_NAME:" $9 CI_PROJECT_NAME=${9} echo "CI_PROJECT_NAME:" ${CI_PROJECT_NAME} #$TAG_NAME ->10 echo "TAG_NAME:" ${10} TAG_NAME=${10} echo "TAG_NAME:" ${TAG_NAME} #$CLEAN_WEB_PAGES ->11 echo "CLEAN_WEB_PAGES:" ${11} CLEAN_WEB_PAGES=${11} echo "CLEAN_WEB_PAGES:" ${CLEAN_WEB_PAGES} #BACKUP=false PAGES_BRANCH=gl-pages HTTPS_REMOTE=https://gitlab-ci-token:${2}@${3}/${4}.git HTTPS_REMOTE=https://gitlab-ci-token:${MIKE_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git echo "\n------ Install necessary packages --------" pip install -q mkdocs-material mike rich mkdocs-rss-plugin echo "------ Config git and get the previous GitLab Pages content --------" git config --global --replace-all user.name "$5" git config --global --replace-all user.email $6 git config --global --replace-all user.name "${GITLAB_USER_NAME}" git config --global --replace-all user.email "${GITLAB_USER_EMAIL}" git fetch origin $PAGES_BRANCH && git checkout $PAGES_BRANCH || git checkout -b $PAGES_BRANCH origin/$PAGES_BRANCH || echo "Pages branch not deployed yet." git checkout $7 git checkout ${CI_COMMIT_SHA} ####### GENERATE NAV SPEC ####### echo "------ Process MD Spec --------" frontmatterClause=False python3 /markdownTools/processMDSpec.py -fmo "./${SPEC_NAME}.md" -fmf frontmatter.md if [ -f "frontmatter.md" ] && [ "$(cat frontmatter.md | tr -d ' \n\t')" != "{}" ]; then echo "------ Adding frontmatter to spec.md --------" python3 /generateSpecWebSite/create_frontmatter_table.py frontmatter.md spec.md echo "------ Content of spec.md --------" cat spec.md frontmatterClause=True fi python3 /markdownTools/processMDSpec.py "./${SPEC_NAME}.md" > combined.md echo "------ Generate input for mkdocs --------" #python3 $1 -it -ihp --title ${9^^} "./$8.md" python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${9^^} "./combined.md" ls ${9^^} if [ $frontmatterClause == True ]; then cat ./combined.md >> ./spec.md python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./spec.md" else python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./${SPEC_NAME}.md" fi ls ${CI_PROJECT_NAME^^} echo "------ Move to docs folder --------" mv ${9^^}/* docs/ sed -i 's/'${9^^}'\///g' _nav.yml mv ${CI_PROJECT_NAME^^}/* docs/ sed -i 's/'${CI_PROJECT_NAME^^}'\///g' _nav.yml cat _nav.yml echo " - 'Home': 'index.md'" >> mkdocs.yml cat _nav.yml >> mkdocs.yml cp docs/0.md docs/index.md repo_url="https://${3}/${4}.git" echo ${repo_url} sed -i 's/##PROJECT/'${9^^}'/g' mkdocs.yml repo_url=$(echo ${repo_url} | sed 's/\//\\\//g') sed -i 's/##REPO_URL/'"${repo_url}"'/g' mkdocs.yml sed -i 's/##REPO_NAME/'${9^^}'/g' mkdocs.yml sed --version cat mkdocs.yml ## Create download tab if official baseline version if [[ ${10} == v* ]]; then echo "Adding download tab for version ${10}..." if [[ ${TAG_NAME} == v* ]]; then echo "\nAdding download tab for version ${TAG_NAME}..." mkdir -p "docs/download" mv "baseline/$8_${10}.docx" docs/download spec_name=$(echo ${8/&/and}) mv "baseline/${SPEC_NAME}_${TAG_NAME}.docx" docs/download spec_name=$(echo ${SPEC_NAME/&/and}) file_name=$(ls docs/download/*.docx) filename_corrected=$(echo ${file_name/&/and}) mv docs/download/"$8_${10}.docx" ${filename_corrected} mv "baseline/$8_${10}.pdf" docs/download spec_name=$(echo ${8/&/and}) mv docs/download/"${SPEC_NAME}_${TAG_NAME}.docx" ${filename_corrected} mv "baseline/${SPEC_NAME}_${TAG_NAME}.pdf" docs/download spec_name=$(echo ${SPEC_NAME/&/and}) file_name=$(ls docs/download/*.pdf) filename_corrected=$(echo ${file_name/&/and}) mv docs/download/"$8_${10}.pdf" ${filename_corrected} mv "baseline/$8_${10}.epub" docs/download spec_name=$(echo ${8/&/and}) mv docs/download/"${SPEC_NAME}_${TAG_NAME}.pdf" ${filename_corrected} mv "baseline/${SPEC_NAME}_${TAG_NAME}.epub" docs/download spec_name=$(echo ${SPEC_NAME/&/and}) file_name=$(ls docs/download/*.epub) filename_corrected=$(echo ${file_name/&/and}) mv docs/download/"$8_${10}.epub" ${filename_corrected} mv docs/download/"${SPEC_NAME}_${TAG_NAME}.epub" ${filename_corrected} echo ${spec_name} #for x in docs/download/*"&"*; do mv -- "$x" "${x//&/and}"; done sed -i 's/PROJECT/'${9^^}'/g' docs/download/index.md sed -i 's/SPECWORD/'"${spec_name}_${10}.docx"'/g' docs/download/index.md sed -i 's/SPECPDF/'"${spec_name}_${10}.pdf"'/g' docs/download/index.md sed -i 's/SPECEPUB/'"${spec_name}_${10}.epub"'/g' docs/download/index.md sed -i 's/VERSION/'${10}'/g' docs/download/index.md sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' docs/download/index.md sed -i 's/SPECWORD/'"${spec_name}_${TAG_NAME}.docx"'/g' docs/download/index.md sed -i 's/SPECPDF/'"${spec_name}_${TAG_NAME}.pdf"'/g' docs/download/index.md sed -i 's/SPECEPUB/'"${spec_name}_${TAG_NAME}.epub"'/g' docs/download/index.md sed -i 's/VERSION/'${TAG_NAME}'/g' docs/download/index.md echo " - Download: 'download/index.md'" >> mkdocs.yml echo "\nAdding release notes tab for version ${TAG_NAME}..." mkdir -p "docs/release_notes" mv "baseline/Release_Notes_${TAG_NAME}.docx" docs/release_notes sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' docs/release_notes/index.md sed -i 's/VERSION/'${TAG_NAME}'/g' docs/release_notes/index.md sed -i 's/SPECWORD/'"Release_Notes_${TAG_NAME}.docx"'/g' docs/release_notes/index.md echo " - Release Notes: 'release_notes/index.md'" >> mkdocs.yml fi if [ ${11} == "false" ]; then if [ ${CLEAN_WEB_PAGES} == "false" ]; then echo "------ Generating site --------" echo "Generating docs version ${10}..." majorVersion=$(echo ${10} | grep -o v[0-9]*\. | grep -o [0-9]*) interVersion=$(echo ${10} | cut -d. -f2) minorVersion=$(echo ${10} | cut -d. -f3) echo "Generating docs version ${TAG_NAME}..." majorVersion=$(echo ${TAG_NAME} | grep -o v[0-9]*\. | grep -o [0-9]*) interVersion=$(echo ${TAG_NAME} | cut -d. -f2) minorVersion=$(echo ${TAG_NAME} | cut -d. -f3) echo "Major version is ${majorVersion}" if [ -n "${majorVersion}" ]; then releaseVersion="latestR${majorVersion}" Loading @@ -113,11 +135,11 @@ if [ ${11} == "false" ]; then if [ -z "${majorVersion}" ] || [ -z "${interVersion}" ] || [ -z "${minorVersion}" ]; then echo "Deploying non-versioned documents..." mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${10} mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${TAG_NAME} else isPagesSet=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH) if [ -z "${isPagesSet}" ]; then mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest mike set-default --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH latest #That's only needed the first time gl-pages is created else latestVersionDeployed=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep ${releaseVersion} | cut -d" " -f1) Loading @@ -129,51 +151,51 @@ if [ ${11} == "false" ]; then ##Handling versions: update latestRelease and latest (majorVersion updated) if [[ "${releaseVersion}" > "${latestRelease}" ]]; then echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest elif [[ "${releaseVersion}" == "${latestRelease}" ]]; then if [[ ${latestVersionDeployed_inter} == ${interVersion} ]]; then if [[ ${latestVersionDeployed_minor} < ${minorVersion} || ${latestVersionDeployed_minor} == ${minorVersion} ]]; then echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest if [[ ${latestVersionDeployed_minor} < ${minorVersion} ]]; then echo "Hidding version ${latestVersionDeployed}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${latestVersionDeployed}" fi else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Hidding version ${10}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${10}" echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} echo "Hidding version ${TAG_NAME}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${TAG_NAME}" fi elif [[ ${latestVersionDeployed_inter} < ${interVersion} ]]; then echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} fi else if [[ ${latestVersionDeployed_inter} == ${interVersion} ]]; then if [[ ${latestVersionDeployed_minor} < ${minorVersion} || ${latestVersionDeployed_minor} == ${minorVersion} ]]; then echo "Deploying version ${10} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} echo "Deploying version ${TAG_NAME} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} if [[ ${latestVersionDeployed_minor} < ${minorVersion} ]]; then echo "Hidding version ${latestVersionDeployed}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${latestVersionDeployed}" fi else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Hidding version ${10}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${10}" echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} echo "Hidding version ${TAG_NAME}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${TAG_NAME}" fi elif [[ ${latestVersionDeployed_inter} < ${interVersion} ]]; then echo "Deploying version ${10} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} echo "Deploying version ${TAG_NAME} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} fi fi Loading @@ -187,17 +209,17 @@ if [ ${11} == "false" ]; then fi fi else echo "------ Removing docs version ${11}..." majorVersion=$(echo ${11} | grep -o v[0-9]*\. | grep -o [0-9]*) echo "------ Removing docs version }..." majorVersion=$(echo ${CLEAN_WEB_PAGES} | grep -o v[0-9]*\. | grep -o [0-9]*) if [ -n "${majorVersion}" ]; then previousVersion=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep v${majorVersion} | grep -A1 ${11} | grep -v ${11}) previousVersion=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep v${majorVersion} | grep -A1 ${CLEAN_WEB_PAGES} | grep -v ${CLEAN_WEB_PAGES}) releaseVersion="latestR${majorVersion}" echo "Major version is ${majorVersion}" echo "Previous version is ${previousVersion}" fi isLatest=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${11} | grep -o latest\, ) isLatestRelease=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${11} | grep -o ${releaseVersion} ) isLatest=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} | grep -o latest\, ) isLatestRelease=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} | grep -o ${releaseVersion} ) echo "This is isLatest: $isLatest" echo "This is isLatestRelease: $isLatestRelease" ## Deal with alias latest Loading Loading @@ -227,7 +249,7 @@ else fi #Deal with moving down a release fi mike delete --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${11} mike delete --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} fi git checkout $PAGES_BRANCH -- public/ Loading Loading
.gitlab-ci.yml +6 −0 Original line number Diff line number Diff line Loading @@ -64,10 +64,16 @@ Build generateSpecWebSite docker image: - generateSpecWebSite/gridTableFilter.py - generateSpecWebSite/gridTableTools.py - generateSpecWebSite/indexDownload.md - generateSpecWebSite/indexReleaseNotes.md - generateSpecWebSite/markdownTools.py - generateSpecWebSite/regexMatches.py - generateSpecWebSite/spec_on_pages.sh - generateSpecWebSite/toMkdocs.py - generateSpecWebSite/create_frontmatter_table.py - markdownTools/dockerfile - markdownTools/setup.py - markdownTools/requirements.txt - markdownTools/processMDSpec.py Build markdownTools docker image: stage: build Loading
generateSpecWebSite/create_frontmatter_table.py 0 → 100644 +100 −0 Original line number Diff line number Diff line #!/usr/bin/env python3 # # create_frontmatter_table.py # # Creates a markdown table from frontmatter data # - Removes headings # - Removes <br> tags # - Formats as a clean markdown table # # (c) 2025 by Miguel Angel Reina Ortega # License: BSD 3-Clause License. See the LICENSE file for further details. import sys import yaml import re def remove_br_tags(text): """Remove <br> and <br/> tags from text.""" if not text: return text # Replace <br>, <br/>, <br /> with space text = re.sub(r'<br\s*/?>', ' ', text, flags=re.IGNORECASE) # Clean up multiple spaces text = re.sub(r'\s+', ' ', text) return text.strip() def create_table_from_frontmatter(frontmatter_file, output_file=None): """ Create a markdown table from frontmatter data. Args: frontmatter_file: Path to the frontmatter.md file output_file: Optional path to output file (if None, prints to stdout) """ # Read frontmatter with open(frontmatter_file, 'r', encoding='utf-8') as f: content = f.read() frontmatter_content = [] # Split into lines and skip first line if it looks like a filename lines = content.split('\n') for line in lines: # Check if it's a filename line: ends with .md: or ends with : and no space before colon if line.strip().endswith('.md:'): # Skip first line continue else: frontmatter_content.append(line) # Parse YAML frontmatter = yaml.safe_load('\n'.join(frontmatter_content)) if not frontmatter: frontmatter = {} # Generate markdown table (no header row) table_lines = [] table_lines.append("") table_lines.append("| | |") table_lines.append(f"|:-|:-|") # Process each field dynamically for key, value in frontmatter.items(): # Convert value to string and clean it if value is None: value_str = "" elif isinstance(value, (list, dict)): # For lists and dicts, convert to YAML string value_str = yaml.dump(value, default_flow_style=False).strip() else: value_str = str(value) # Remove <br> tags #value_str = remove_br_tags(value_str) # Escape pipe characters in values (they break markdown tables) value_str = value_str.replace('|', '\\|') # Add row to table table_lines.append(f"| **{key}** | {value_str} |") # Join all lines table_lines.append("") table_content = '\n'.join(table_lines) # Output if output_file: with open(output_file, 'w', encoding='utf-8') as f: f.write(table_content) else: print(table_content) if __name__ == '__main__': if len(sys.argv) < 2: print("Usage: create_frontmatter_table.py <frontmatter_file> [output_file]") sys.exit(1) frontmatter_file = sys.argv[1] output_file = sys.argv[2] if len(sys.argv) > 2 else None create_table_from_frontmatter(frontmatter_file, output_file)
generateSpecWebSite/dockerfile +7 −0 Original line number Diff line number Diff line Loading @@ -7,3 +7,10 @@ RUN apt-get update -y && \ rm -rf /var/lib/apt/lists/* &&\ pip install -e generateSpecWebSite/ &&\ pip install -r generateSpecWebSite/requirements.txt ADD /markdownTools/ /markdownTools/ RUN apt-get update -y && \ apt-get install -y libcairo2 && \ rm -rf /var/lib/apt/lists/* &&\ pip install -e markdownTools/ &&\ pip install -r markdownTools/requirements.txt
generateSpecWebSite/indexReleaseNotes.md 0 → 100644 +6 −0 Original line number Diff line number Diff line Download Release Notes for PROJECT version VERSION: [](SPECWORD)
generateSpecWebSite/spec_on_pages.sh +102 −80 Original line number Diff line number Diff line Loading @@ -14,97 +14,119 @@ #toMkdocs.py -> 1 echo "toMkdocs.py:" $1 #$MIKE_ACCESS_TOKEN -> 2 echo "MIKE_ACCESS_TOKEN:" $2 MIKE_ACCESS_TOKEN=${2} echo "MIKE_ACCESS_TOKEN:" ${MIKE_ACCESS_TOKEN} #CI_SERVER_HOST -> 3 echo "CI_SERVER_HOST:" $3 CI_SERVER_HOST=${3} echo "CI_SERVER_HOST:" ${CI_SERVER_HOST} #CI_PROJECT_PATH -> 4 echo "CI_PROJECT_PATH:" $4 CI_PROJECT_PATH=${4} echo "CI_PROJECT_PATH:" ${CI_PROJECT_PATH} #GITLAB_USER_NAME -> 5 echo "GITLAB_USER_NAME:" $5 GITLAB_USER_NAME=${5} echo "GITLAB_USER_NAME:" ${GITLAB_USER_NAME} #GITLAB_USER_EMAIL -> 6 echo "GITLAB_USER_EMAIL:" $6 GITLAB_USER_EMAIL=${6} echo "GITLAB_USER_EMAIL:" ${GITLAB_USER_EMAIL} #CI_COMMIT_SHA -> 7 echo "CI_COMMIT_SHA:" $7 CI_COMMIT_SHA=${7} echo "CI_COMMIT_SHA:" ${CI_COMMIT_SHA} #$SPEC_NAME -> 8 echo "SPEC_NAME:" $8 SPEC_NAME=${8} echo "SPEC_NAME:" ${SPEC_NAME} #$CI_PROJECT_NAME -> 9 echo "CI_PROJECT_NAME:" $9 CI_PROJECT_NAME=${9} echo "CI_PROJECT_NAME:" ${CI_PROJECT_NAME} #$TAG_NAME ->10 echo "TAG_NAME:" ${10} TAG_NAME=${10} echo "TAG_NAME:" ${TAG_NAME} #$CLEAN_WEB_PAGES ->11 echo "CLEAN_WEB_PAGES:" ${11} CLEAN_WEB_PAGES=${11} echo "CLEAN_WEB_PAGES:" ${CLEAN_WEB_PAGES} #BACKUP=false PAGES_BRANCH=gl-pages HTTPS_REMOTE=https://gitlab-ci-token:${2}@${3}/${4}.git HTTPS_REMOTE=https://gitlab-ci-token:${MIKE_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git echo "\n------ Install necessary packages --------" pip install -q mkdocs-material mike rich mkdocs-rss-plugin echo "------ Config git and get the previous GitLab Pages content --------" git config --global --replace-all user.name "$5" git config --global --replace-all user.email $6 git config --global --replace-all user.name "${GITLAB_USER_NAME}" git config --global --replace-all user.email "${GITLAB_USER_EMAIL}" git fetch origin $PAGES_BRANCH && git checkout $PAGES_BRANCH || git checkout -b $PAGES_BRANCH origin/$PAGES_BRANCH || echo "Pages branch not deployed yet." git checkout $7 git checkout ${CI_COMMIT_SHA} ####### GENERATE NAV SPEC ####### echo "------ Process MD Spec --------" frontmatterClause=False python3 /markdownTools/processMDSpec.py -fmo "./${SPEC_NAME}.md" -fmf frontmatter.md if [ -f "frontmatter.md" ] && [ "$(cat frontmatter.md | tr -d ' \n\t')" != "{}" ]; then echo "------ Adding frontmatter to spec.md --------" python3 /generateSpecWebSite/create_frontmatter_table.py frontmatter.md spec.md echo "------ Content of spec.md --------" cat spec.md frontmatterClause=True fi python3 /markdownTools/processMDSpec.py "./${SPEC_NAME}.md" > combined.md echo "------ Generate input for mkdocs --------" #python3 $1 -it -ihp --title ${9^^} "./$8.md" python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${9^^} "./combined.md" ls ${9^^} if [ $frontmatterClause == True ]; then cat ./combined.md >> ./spec.md python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./spec.md" else python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./${SPEC_NAME}.md" fi ls ${CI_PROJECT_NAME^^} echo "------ Move to docs folder --------" mv ${9^^}/* docs/ sed -i 's/'${9^^}'\///g' _nav.yml mv ${CI_PROJECT_NAME^^}/* docs/ sed -i 's/'${CI_PROJECT_NAME^^}'\///g' _nav.yml cat _nav.yml echo " - 'Home': 'index.md'" >> mkdocs.yml cat _nav.yml >> mkdocs.yml cp docs/0.md docs/index.md repo_url="https://${3}/${4}.git" echo ${repo_url} sed -i 's/##PROJECT/'${9^^}'/g' mkdocs.yml repo_url=$(echo ${repo_url} | sed 's/\//\\\//g') sed -i 's/##REPO_URL/'"${repo_url}"'/g' mkdocs.yml sed -i 's/##REPO_NAME/'${9^^}'/g' mkdocs.yml sed --version cat mkdocs.yml ## Create download tab if official baseline version if [[ ${10} == v* ]]; then echo "Adding download tab for version ${10}..." if [[ ${TAG_NAME} == v* ]]; then echo "\nAdding download tab for version ${TAG_NAME}..." mkdir -p "docs/download" mv "baseline/$8_${10}.docx" docs/download spec_name=$(echo ${8/&/and}) mv "baseline/${SPEC_NAME}_${TAG_NAME}.docx" docs/download spec_name=$(echo ${SPEC_NAME/&/and}) file_name=$(ls docs/download/*.docx) filename_corrected=$(echo ${file_name/&/and}) mv docs/download/"$8_${10}.docx" ${filename_corrected} mv "baseline/$8_${10}.pdf" docs/download spec_name=$(echo ${8/&/and}) mv docs/download/"${SPEC_NAME}_${TAG_NAME}.docx" ${filename_corrected} mv "baseline/${SPEC_NAME}_${TAG_NAME}.pdf" docs/download spec_name=$(echo ${SPEC_NAME/&/and}) file_name=$(ls docs/download/*.pdf) filename_corrected=$(echo ${file_name/&/and}) mv docs/download/"$8_${10}.pdf" ${filename_corrected} mv "baseline/$8_${10}.epub" docs/download spec_name=$(echo ${8/&/and}) mv docs/download/"${SPEC_NAME}_${TAG_NAME}.pdf" ${filename_corrected} mv "baseline/${SPEC_NAME}_${TAG_NAME}.epub" docs/download spec_name=$(echo ${SPEC_NAME/&/and}) file_name=$(ls docs/download/*.epub) filename_corrected=$(echo ${file_name/&/and}) mv docs/download/"$8_${10}.epub" ${filename_corrected} mv docs/download/"${SPEC_NAME}_${TAG_NAME}.epub" ${filename_corrected} echo ${spec_name} #for x in docs/download/*"&"*; do mv -- "$x" "${x//&/and}"; done sed -i 's/PROJECT/'${9^^}'/g' docs/download/index.md sed -i 's/SPECWORD/'"${spec_name}_${10}.docx"'/g' docs/download/index.md sed -i 's/SPECPDF/'"${spec_name}_${10}.pdf"'/g' docs/download/index.md sed -i 's/SPECEPUB/'"${spec_name}_${10}.epub"'/g' docs/download/index.md sed -i 's/VERSION/'${10}'/g' docs/download/index.md sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' docs/download/index.md sed -i 's/SPECWORD/'"${spec_name}_${TAG_NAME}.docx"'/g' docs/download/index.md sed -i 's/SPECPDF/'"${spec_name}_${TAG_NAME}.pdf"'/g' docs/download/index.md sed -i 's/SPECEPUB/'"${spec_name}_${TAG_NAME}.epub"'/g' docs/download/index.md sed -i 's/VERSION/'${TAG_NAME}'/g' docs/download/index.md echo " - Download: 'download/index.md'" >> mkdocs.yml echo "\nAdding release notes tab for version ${TAG_NAME}..." mkdir -p "docs/release_notes" mv "baseline/Release_Notes_${TAG_NAME}.docx" docs/release_notes sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' docs/release_notes/index.md sed -i 's/VERSION/'${TAG_NAME}'/g' docs/release_notes/index.md sed -i 's/SPECWORD/'"Release_Notes_${TAG_NAME}.docx"'/g' docs/release_notes/index.md echo " - Release Notes: 'release_notes/index.md'" >> mkdocs.yml fi if [ ${11} == "false" ]; then if [ ${CLEAN_WEB_PAGES} == "false" ]; then echo "------ Generating site --------" echo "Generating docs version ${10}..." majorVersion=$(echo ${10} | grep -o v[0-9]*\. | grep -o [0-9]*) interVersion=$(echo ${10} | cut -d. -f2) minorVersion=$(echo ${10} | cut -d. -f3) echo "Generating docs version ${TAG_NAME}..." majorVersion=$(echo ${TAG_NAME} | grep -o v[0-9]*\. | grep -o [0-9]*) interVersion=$(echo ${TAG_NAME} | cut -d. -f2) minorVersion=$(echo ${TAG_NAME} | cut -d. -f3) echo "Major version is ${majorVersion}" if [ -n "${majorVersion}" ]; then releaseVersion="latestR${majorVersion}" Loading @@ -113,11 +135,11 @@ if [ ${11} == "false" ]; then if [ -z "${majorVersion}" ] || [ -z "${interVersion}" ] || [ -z "${minorVersion}" ]; then echo "Deploying non-versioned documents..." mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${10} mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${TAG_NAME} else isPagesSet=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH) if [ -z "${isPagesSet}" ]; then mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest mike set-default --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH latest #That's only needed the first time gl-pages is created else latestVersionDeployed=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep ${releaseVersion} | cut -d" " -f1) Loading @@ -129,51 +151,51 @@ if [ ${11} == "false" ]; then ##Handling versions: update latestRelease and latest (majorVersion updated) if [[ "${releaseVersion}" > "${latestRelease}" ]]; then echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest elif [[ "${releaseVersion}" == "${latestRelease}" ]]; then if [[ ${latestVersionDeployed_inter} == ${interVersion} ]]; then if [[ ${latestVersionDeployed_minor} < ${minorVersion} || ${latestVersionDeployed_minor} == ${minorVersion} ]]; then echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest if [[ ${latestVersionDeployed_minor} < ${minorVersion} ]]; then echo "Hidding version ${latestVersionDeployed}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${latestVersionDeployed}" fi else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Hidding version ${10}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${10}" echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} echo "Hidding version ${TAG_NAME}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${TAG_NAME}" fi elif [[ ${latestVersionDeployed_inter} < ${interVersion} ]]; then echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} fi else if [[ ${latestVersionDeployed_inter} == ${interVersion} ]]; then if [[ ${latestVersionDeployed_minor} < ${minorVersion} || ${latestVersionDeployed_minor} == ${minorVersion} ]]; then echo "Deploying version ${10} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} echo "Deploying version ${TAG_NAME} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} if [[ ${latestVersionDeployed_minor} < ${minorVersion} ]]; then echo "Hidding version ${latestVersionDeployed}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${latestVersionDeployed}" fi else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Hidding version ${10}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${10}" echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} echo "Hidding version ${TAG_NAME}..." mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${TAG_NAME}" fi elif [[ ${latestVersionDeployed_inter} < ${interVersion} ]]; then echo "Deploying version ${10} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} echo "Deploying version ${TAG_NAME} as ${releaseVersion}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} else echo "Deploying version ${10}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} echo "Deploying version ${TAG_NAME}" mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} fi fi Loading @@ -187,17 +209,17 @@ if [ ${11} == "false" ]; then fi fi else echo "------ Removing docs version ${11}..." majorVersion=$(echo ${11} | grep -o v[0-9]*\. | grep -o [0-9]*) echo "------ Removing docs version }..." majorVersion=$(echo ${CLEAN_WEB_PAGES} | grep -o v[0-9]*\. | grep -o [0-9]*) if [ -n "${majorVersion}" ]; then previousVersion=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep v${majorVersion} | grep -A1 ${11} | grep -v ${11}) previousVersion=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep v${majorVersion} | grep -A1 ${CLEAN_WEB_PAGES} | grep -v ${CLEAN_WEB_PAGES}) releaseVersion="latestR${majorVersion}" echo "Major version is ${majorVersion}" echo "Previous version is ${previousVersion}" fi isLatest=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${11} | grep -o latest\, ) isLatestRelease=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${11} | grep -o ${releaseVersion} ) isLatest=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} | grep -o latest\, ) isLatestRelease=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} | grep -o ${releaseVersion} ) echo "This is isLatest: $isLatest" echo "This is isLatestRelease: $isLatestRelease" ## Deal with alias latest Loading Loading @@ -227,7 +249,7 @@ else fi #Deal with moving down a release fi mike delete --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${11} mike delete --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} fi git checkout $PAGES_BRANCH -- public/ Loading