Commit a55b507e authored by Miguel Angel Reina Ortega's avatar Miguel Angel Reina Ortega
Browse files

Handling of frontmatter and release notes

parent ef35ef2d
Loading
Loading
Loading
Loading
Loading
+6 −0
Original line number Diff line number Diff line
@@ -64,10 +64,16 @@ Build generateSpecWebSite docker image:
        - generateSpecWebSite/gridTableFilter.py
        - generateSpecWebSite/gridTableTools.py
        - generateSpecWebSite/indexDownload.md
        - generateSpecWebSite/indexReleaseNotes.md
        - generateSpecWebSite/markdownTools.py
        - generateSpecWebSite/regexMatches.py
        - generateSpecWebSite/spec_on_pages.sh
        - generateSpecWebSite/toMkdocs.py
        - generateSpecWebSite/create_frontmatter_table.py
        - markdownTools/dockerfile
        - markdownTools/setup.py
        - markdownTools/requirements.txt
        - markdownTools/processMDSpec.py

Build markdownTools docker image:
  stage: build
+100 −0
Original line number Diff line number Diff line
#!/usr/bin/env python3
#
#	create_frontmatter_table.py
#
#	Creates a markdown table from frontmatter data
#	- Removes headings
#	- Removes <br> tags
#	- Formats as a clean markdown table
#
#	(c) 2025 by Miguel Angel Reina Ortega
#	License: BSD 3-Clause License. See the LICENSE file for further details.

import sys
import yaml
import re

def remove_br_tags(text):
    """Remove <br> and <br/> tags from text."""
    if not text:
        return text
    # Replace <br>, <br/>, <br /> with space
    text = re.sub(r'<br\s*/?>', ' ', text, flags=re.IGNORECASE)
    # Clean up multiple spaces
    text = re.sub(r'\s+', ' ', text)
    return text.strip()

def create_table_from_frontmatter(frontmatter_file, output_file=None):
    """
    Create a markdown table from frontmatter data.
    
    Args:
        frontmatter_file: Path to the frontmatter.md file
        output_file: Optional path to output file (if None, prints to stdout)
    """
    
    # Read frontmatter
    with open(frontmatter_file, 'r', encoding='utf-8') as f:
        content = f.read()
    frontmatter_content = []
    # Split into lines and skip first line if it looks like a filename
    lines = content.split('\n')
    for line in lines:
        # Check if it's a filename line: ends with .md: or ends with : and no space before colon
        if line.strip().endswith('.md:'):
            # Skip first line
            continue
        else:
            frontmatter_content.append(line)
    
    # Parse YAML
    frontmatter = yaml.safe_load('\n'.join(frontmatter_content))
    if not frontmatter:
        frontmatter = {}
    
    # Generate markdown table (no header row)
    table_lines = []
    table_lines.append("")
    table_lines.append("|&nbsp; |&nbsp; |")
    table_lines.append(f"|:-|:-|")
    # Process each field dynamically
    for key, value in frontmatter.items():
        # Convert value to string and clean it
        if value is None:
            value_str = ""
        elif isinstance(value, (list, dict)):
            # For lists and dicts, convert to YAML string
            value_str = yaml.dump(value, default_flow_style=False).strip()
        else:
            value_str = str(value)
        
        # Remove <br> tags
        #value_str = remove_br_tags(value_str)
        
        # Escape pipe characters in values (they break markdown tables)
        value_str = value_str.replace('|', '\\|')
        
        # Add row to table
        table_lines.append(f"| **{key}** | {value_str} |")
    
    # Join all lines
    table_lines.append("")
    table_content = '\n'.join(table_lines)
    
    # Output
    if output_file:
        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(table_content)
    else:
        print(table_content)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("Usage: create_frontmatter_table.py <frontmatter_file> [output_file]")
        sys.exit(1)
    
    frontmatter_file = sys.argv[1]
    output_file = sys.argv[2] if len(sys.argv) > 2 else None
    
    create_table_from_frontmatter(frontmatter_file, output_file)
+7 −0
Original line number Diff line number Diff line
@@ -7,3 +7,10 @@ RUN apt-get update -y && \
    rm -rf /var/lib/apt/lists/* &&\
    pip install -e generateSpecWebSite/ &&\
	pip install -r generateSpecWebSite/requirements.txt

ADD /markdownTools/ /markdownTools/
RUN apt-get update -y && \
    apt-get install -y libcairo2 && \
    rm -rf /var/lib/apt/lists/* &&\
    pip install -e markdownTools/ &&\
	pip install -r markdownTools/requirements.txt
+6 −0
Original line number Diff line number Diff line


Download Release Notes for PROJECT version VERSION:

[![Word](images/Word.png)](SPECWORD)
+102 −80
Original line number Diff line number Diff line
@@ -14,97 +14,119 @@
#toMkdocs.py -> 1
echo "toMkdocs.py:" $1
#$MIKE_ACCESS_TOKEN -> 2
echo "MIKE_ACCESS_TOKEN:" $2
MIKE_ACCESS_TOKEN=${2}
echo "MIKE_ACCESS_TOKEN:" ${MIKE_ACCESS_TOKEN}
#CI_SERVER_HOST -> 3
echo "CI_SERVER_HOST:" $3
CI_SERVER_HOST=${3}
echo "CI_SERVER_HOST:" ${CI_SERVER_HOST}
#CI_PROJECT_PATH -> 4
echo "CI_PROJECT_PATH:" $4
CI_PROJECT_PATH=${4}
echo "CI_PROJECT_PATH:" ${CI_PROJECT_PATH}
#GITLAB_USER_NAME -> 5
echo "GITLAB_USER_NAME:" $5
GITLAB_USER_NAME=${5}
echo "GITLAB_USER_NAME:" ${GITLAB_USER_NAME}
#GITLAB_USER_EMAIL -> 6
echo "GITLAB_USER_EMAIL:" $6
GITLAB_USER_EMAIL=${6}
echo "GITLAB_USER_EMAIL:" ${GITLAB_USER_EMAIL}
#CI_COMMIT_SHA -> 7
echo "CI_COMMIT_SHA:" $7
CI_COMMIT_SHA=${7}
echo "CI_COMMIT_SHA:" ${CI_COMMIT_SHA}
#$SPEC_NAME -> 8
echo "SPEC_NAME:" $8
SPEC_NAME=${8}
echo "SPEC_NAME:" ${SPEC_NAME}
#$CI_PROJECT_NAME -> 9
echo "CI_PROJECT_NAME:" $9
CI_PROJECT_NAME=${9}
echo "CI_PROJECT_NAME:" ${CI_PROJECT_NAME}
#$TAG_NAME ->10
echo "TAG_NAME:" ${10}
TAG_NAME=${10}
echo "TAG_NAME:" ${TAG_NAME}
#$CLEAN_WEB_PAGES ->11
echo "CLEAN_WEB_PAGES:" ${11}
CLEAN_WEB_PAGES=${11}
echo "CLEAN_WEB_PAGES:" ${CLEAN_WEB_PAGES}

#BACKUP=false
PAGES_BRANCH=gl-pages
HTTPS_REMOTE=https://gitlab-ci-token:${2}@${3}/${4}.git
HTTPS_REMOTE=https://gitlab-ci-token:${MIKE_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git

echo "\n------ Install necessary packages --------"
pip install -q mkdocs-material mike rich mkdocs-rss-plugin

echo "------ Config git and get the previous GitLab Pages content --------"
git config --global --replace-all user.name "$5"
git config --global --replace-all user.email $6
git config --global --replace-all user.name "${GITLAB_USER_NAME}"
git config --global --replace-all user.email "${GITLAB_USER_EMAIL}"
git fetch origin $PAGES_BRANCH && git checkout $PAGES_BRANCH || git checkout -b $PAGES_BRANCH origin/$PAGES_BRANCH || echo "Pages branch not deployed yet."
git checkout $7
git checkout ${CI_COMMIT_SHA}

####### GENERATE NAV SPEC #######
echo "------ Process MD Spec --------"
frontmatterClause=False
python3 /markdownTools/processMDSpec.py -fmo "./${SPEC_NAME}.md" -fmf frontmatter.md
if [ -f "frontmatter.md" ] && [ "$(cat frontmatter.md | tr -d ' \n\t')" != "{}" ]; then
  echo "------ Adding frontmatter to spec.md --------"
  python3 /generateSpecWebSite/create_frontmatter_table.py frontmatter.md spec.md
  echo "------ Content of spec.md --------"
  cat spec.md
  frontmatterClause=True
fi
python3 /markdownTools/processMDSpec.py "./${SPEC_NAME}.md" > combined.md
echo "------ Generate input for mkdocs --------"
#python3 $1 -it -ihp --title ${9^^} "./$8.md"
python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${9^^} "./combined.md"
ls ${9^^}
if [ $frontmatterClause == True ]; then
  cat ./combined.md >> ./spec.md
  python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./spec.md"
else
  python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./${SPEC_NAME}.md"
fi
ls ${CI_PROJECT_NAME^^}
echo "------ Move to docs folder --------"
mv ${9^^}/* docs/
sed -i 's/'${9^^}'\///g' _nav.yml
mv ${CI_PROJECT_NAME^^}/* docs/
sed -i 's/'${CI_PROJECT_NAME^^}'\///g' _nav.yml
cat _nav.yml
echo "  - 'Home': 'index.md'" >> mkdocs.yml
cat _nav.yml >> mkdocs.yml
cp docs/0.md docs/index.md
repo_url="https://${3}/${4}.git"
echo ${repo_url}
sed -i 's/##PROJECT/'${9^^}'/g' mkdocs.yml
repo_url=$(echo ${repo_url} | sed 's/\//\\\//g')
sed -i 's/##REPO_URL/'"${repo_url}"'/g' mkdocs.yml
sed -i 's/##REPO_NAME/'${9^^}'/g' mkdocs.yml

sed --version

cat mkdocs.yml

## Create download tab if official baseline version
if [[ ${10} == v* ]]; then
    echo "Adding download tab for version ${10}..."
if [[ ${TAG_NAME} == v* ]]; then
    echo "\nAdding download tab for version ${TAG_NAME}..."
    mkdir -p "docs/download"
    mv "baseline/$8_${10}.docx" docs/download
    spec_name=$(echo ${8/&/and})
    mv "baseline/${SPEC_NAME}_${TAG_NAME}.docx" docs/download
    spec_name=$(echo ${SPEC_NAME/&/and})
    file_name=$(ls docs/download/*.docx)
    filename_corrected=$(echo ${file_name/&/and})
    mv docs/download/"$8_${10}.docx" ${filename_corrected}
    mv "baseline/$8_${10}.pdf" docs/download
    spec_name=$(echo ${8/&/and})
    mv docs/download/"${SPEC_NAME}_${TAG_NAME}.docx" ${filename_corrected}
    mv "baseline/${SPEC_NAME}_${TAG_NAME}.pdf" docs/download
    spec_name=$(echo ${SPEC_NAME/&/and})
    file_name=$(ls docs/download/*.pdf)
    filename_corrected=$(echo ${file_name/&/and})
    mv docs/download/"$8_${10}.pdf" ${filename_corrected}
    mv "baseline/$8_${10}.epub" docs/download
    spec_name=$(echo ${8/&/and})
    mv docs/download/"${SPEC_NAME}_${TAG_NAME}.pdf" ${filename_corrected}
    mv "baseline/${SPEC_NAME}_${TAG_NAME}.epub" docs/download
    spec_name=$(echo ${SPEC_NAME/&/and})
    file_name=$(ls docs/download/*.epub)
    filename_corrected=$(echo ${file_name/&/and})
    mv docs/download/"$8_${10}.epub" ${filename_corrected}
    mv docs/download/"${SPEC_NAME}_${TAG_NAME}.epub" ${filename_corrected}
    echo ${spec_name}
    #for x in docs/download/*"&"*; do   mv -- "$x" "${x//&/and}"; done
    sed -i 's/PROJECT/'${9^^}'/g' docs/download/index.md
    sed -i 's/SPECWORD/'"${spec_name}_${10}.docx"'/g' docs/download/index.md
    sed -i 's/SPECPDF/'"${spec_name}_${10}.pdf"'/g' docs/download/index.md
    sed -i 's/SPECEPUB/'"${spec_name}_${10}.epub"'/g' docs/download/index.md
    sed -i 's/VERSION/'${10}'/g' docs/download/index.md
    sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' docs/download/index.md
    sed -i 's/SPECWORD/'"${spec_name}_${TAG_NAME}.docx"'/g' docs/download/index.md
    sed -i 's/SPECPDF/'"${spec_name}_${TAG_NAME}.pdf"'/g' docs/download/index.md
    sed -i 's/SPECEPUB/'"${spec_name}_${TAG_NAME}.epub"'/g' docs/download/index.md
    sed -i 's/VERSION/'${TAG_NAME}'/g' docs/download/index.md
    echo "  - Download: 'download/index.md'" >> mkdocs.yml
    echo "\nAdding release notes tab for version ${TAG_NAME}..."
    mkdir -p "docs/release_notes"
    mv "baseline/Release_Notes_${TAG_NAME}.docx" docs/release_notes
    sed -i 's/PROJECT/'${CI_PROJECT_NAME^^}'/g' docs/release_notes/index.md
    sed -i 's/VERSION/'${TAG_NAME}'/g' docs/release_notes/index.md
    sed -i 's/SPECWORD/'"Release_Notes_${TAG_NAME}.docx"'/g' docs/release_notes/index.md
    echo "  - Release Notes: 'release_notes/index.md'" >> mkdocs.yml
fi

if [ ${11} == "false" ]; then
if [ ${CLEAN_WEB_PAGES} == "false" ]; then
    echo "------ Generating site --------"
    echo "Generating docs version ${10}..."
    majorVersion=$(echo ${10} | grep -o v[0-9]*\. | grep -o [0-9]*)
    interVersion=$(echo ${10} | cut -d. -f2)
    minorVersion=$(echo ${10} | cut -d. -f3)
    echo "Generating docs version ${TAG_NAME}..."
    majorVersion=$(echo ${TAG_NAME} | grep -o v[0-9]*\. | grep -o [0-9]*)
    interVersion=$(echo ${TAG_NAME} | cut -d. -f2)
    minorVersion=$(echo ${TAG_NAME} | cut -d. -f3)
    echo "Major version is ${majorVersion}"
    if [ -n "${majorVersion}" ]; then
        releaseVersion="latestR${majorVersion}"
@@ -113,11 +135,11 @@ if [ ${11} == "false" ]; then

    if [ -z "${majorVersion}" ] || [ -z "${interVersion}" ] || [ -z "${minorVersion}" ]; then
        echo "Deploying non-versioned documents..."
        mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${10}
        mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${TAG_NAME}
    else
      isPagesSet=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH)
      if [ -z "${isPagesSet}" ]; then
          mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest
          mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest
          mike set-default --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH latest #That's only needed the first time gl-pages is created
      else
          latestVersionDeployed=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep ${releaseVersion} | cut -d" " -f1)
@@ -129,51 +151,51 @@ if [ ${11} == "false" ]; then

          ##Handling versions: update latestRelease and latest (majorVersion updated)
          if [[ "${releaseVersion}" > "${latestRelease}" ]]; then
              echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}"
              mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest
              echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}"
              mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest
          elif [[ "${releaseVersion}" == "${latestRelease}" ]]; then
              if [[ ${latestVersionDeployed_inter} == ${interVersion} ]]; then
                if [[ ${latestVersionDeployed_minor} < ${minorVersion} || ${latestVersionDeployed_minor} == ${minorVersion} ]]; then
                  echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}"
                  mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest
                  echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}"
                  mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest
                  if [[ ${latestVersionDeployed_minor} < ${minorVersion} ]]; then
                    echo "Hidding version ${latestVersionDeployed}..."
                    mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${latestVersionDeployed}"
                  fi
                else
                  echo "Deploying version ${10}"
                  mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10}
                  echo "Hidding version ${10}..."
                  mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${10}"
                  echo "Deploying version ${TAG_NAME}"
                  mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME}
                  echo "Hidding version ${TAG_NAME}..."
                  mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${TAG_NAME}"
                fi
              elif [[ ${latestVersionDeployed_inter} < ${interVersion} ]]; then
                echo "Upgrade latest release...deploying version ${10} as latest and ${releaseVersion}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion} latest
                echo "Upgrade latest release...deploying version ${TAG_NAME} as latest and ${releaseVersion}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion} latest
              else
                echo "Deploying version ${10}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10}
                echo "Deploying version ${TAG_NAME}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME}
              fi
          else
            if [[ ${latestVersionDeployed_inter} == ${interVersion} ]]; then
              if [[ ${latestVersionDeployed_minor} < ${minorVersion} || ${latestVersionDeployed_minor} == ${minorVersion} ]]; then
                echo "Deploying version ${10} as ${releaseVersion}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion}
                echo "Deploying version ${TAG_NAME} as ${releaseVersion}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion}
                if [[ ${latestVersionDeployed_minor} < ${minorVersion} ]]; then
                  echo "Hidding version ${latestVersionDeployed}..."
                  mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${latestVersionDeployed}"
                fi
              else
                echo "Deploying version ${10}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10}
                echo "Hidding version ${10}..."
                mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${10}"
                echo "Deploying version ${TAG_NAME}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME}
                echo "Hidding version ${TAG_NAME}..."
                mike props --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --set hidden=true "${TAG_NAME}"
              fi
            elif [[ ${latestVersionDeployed_inter} < ${interVersion} ]]; then
                echo "Deploying version ${10} as ${releaseVersion}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10} ${releaseVersion}
                echo "Deploying version ${TAG_NAME} as ${releaseVersion}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME} ${releaseVersion}
            else
                echo "Deploying version ${10}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${10}
                echo "Deploying version ${TAG_NAME}"
                mike deploy --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH -u ${TAG_NAME}
            fi
          fi

@@ -187,17 +209,17 @@ if [ ${11} == "false" ]; then
      fi
    fi
else
    echo "------ Removing docs version ${11}..."
    majorVersion=$(echo ${11} | grep -o v[0-9]*\. | grep -o [0-9]*)
    echo "------ Removing docs version  }..."
    majorVersion=$(echo ${CLEAN_WEB_PAGES} | grep -o v[0-9]*\. | grep -o [0-9]*)
    if [ -n "${majorVersion}" ]; then
        previousVersion=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep v${majorVersion} | grep -A1 ${11} | grep -v ${11})
        previousVersion=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep v${majorVersion} | grep -A1 ${CLEAN_WEB_PAGES} | grep -v ${CLEAN_WEB_PAGES})
        releaseVersion="latestR${majorVersion}"
        echo "Major version is ${majorVersion}"
        echo "Previous version is ${previousVersion}"
    fi

    isLatest=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${11} | grep -o latest\, )
    isLatestRelease=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${11} | grep -o ${releaseVersion} )
    isLatest=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} | grep -o latest\, )
    isLatestRelease=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH ${CLEAN_WEB_PAGES} | grep -o ${releaseVersion} )
    echo "This is isLatest: $isLatest"
    echo "This is isLatestRelease: $isLatestRelease"
    ## Deal with alias latest
@@ -227,7 +249,7 @@ else
        fi #Deal with moving down a release
    fi

    mike delete --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${11}
    mike delete --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${CLEAN_WEB_PAGES}
fi

git checkout $PAGES_BRANCH -- public/