Commit 6486c74b authored by Jonas Schüppen's avatar Jonas Schüppen
Browse files

Merge remote-tracking branch 'origin/master' into format_update_EQ_in_NO 

# Conflicts:
#	generateBaseline/postprocessing.py
parents 3cecb405 905e98a9

.gitignore

+2 −0
Original line number Diff line number Diff line
@@ -4,3 +4,5 @@ 
*/.python-version

.python-version

toMkdocs/__pycache__

__pycache__

.idea/

.gitlab-ci.yml

+1 −0
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ Build generateSpecWebSite docker image: 
        - generateSpecWebSite/spec_on_pages.sh

        - generateSpecWebSite/toMkdocs.py

        - generateSpecWebSite/create_frontmatter_table.py

        - generateSpecWebSite/stylesheets/extra.css

        - markdownTools/dockerfile

        - markdownTools/setup.py

        - markdownTools/requirements.txt

generateBaseline/postprocessing.py

+36 −16
Original line number Diff line number Diff line
@@ -1933,10 +1933,10 @@ def update_notes(docx_input, docx_output): 
        all_children = [child for child in para if child.tag != f"{{{ns['w']}}}pPr"]

        

        # Check if text matches NOTE pattern: NOTE\s*\d*:

        note_pattern = re.compile(r'NOTE\s*(\d*)\s*:\s*')

        note_pattern = re.compile(r'NOTES?\s*\d*:?\s+')

        note_match = note_pattern.search(full_text)

        # Check if text matches EXAMPLE pattern: EXAMPLE\s*\d*:

        example_pattern = re.compile(r'EXAMPLE\s*(\d*)\s*:\s*')

        example_pattern = re.compile(r'EXAMPLES?\s*\d*:?\s+')

        example_match = example_pattern.search(full_text)

        if note_match:

            # Change style to NO
@@ -1972,11 +1972,7 @@ def update_notes(docx_input, docx_output): 
            # Create new runs: r1="NOTE:" r2="tab" and append all original children

            r1 = OxmlElement('w:r')

            t1 = OxmlElement('w:t')

            number = note_match.group(1)  # "" oder z.B. "3"

            if number:

                t1.text = f"NOTE {number}:"

            else:

                t1.text = "NOTE:"

            t1.text = matched_note

            r1.append(t1)

            para.append(r1)
@@ -2019,11 +2015,7 @@ def update_notes(docx_input, docx_output): 
            # Create new runs: r1="EXAMPLE:" r2="tab" and append all original children

            r1 = OxmlElement('w:r')

            t1 = OxmlElement('w:t')

            number = example_match.group(1)  # "" oder z.B. "3"

            if number:

                t1.text = f"EXAMPLE {number}:"

            else:

                t1.text = "EXAMPLE:"

            t1.text = matched_example

            r1.append(t1)

            para.append(r1)
@@ -2036,6 +2028,34 @@ def update_notes(docx_input, docx_output): 
            # Re-append all original children (runs and hyperlinks) in their original order

            for child in all_children:

                para.append(child)

        else:

            # Check whether BlockText is preceded by paragraph style "EW".

            # Rules:

            # - if preceding paragraph style is EW -> change to PL and stop

            # - if preceding paragraph has a style and it is not EW -> stop (no change)

            # - if no pPr or no pStyle -> continue searching backwards

            # - if not a paragraph -> continue searching backwards

            prev_para = para.getprevious()

            while prev_para is not None:

                if prev_para.tag != f"{{{ns['w']}}}p":

                    prev_para = prev_para.getprevious()

                    continue



                prev_pPr = prev_para.xpath('./w:pPr', namespaces=ns)

                if not prev_pPr:

                    prev_para = prev_para.getprevious()

                    continue



                prev_pStyle = prev_pPr[0].xpath('./w:pStyle', namespaces=ns)

                if not prev_pStyle:

                    prev_para = prev_para.getprevious()

                    continue



                prev_pStyle_val = prev_pStyle[0].get(f"{{{ns['w']}}}val")

                if prev_pStyle_val == example_style:

                    pStyle_elem.set(f"{{{ns['w']}}}val", "PL")

                    print(f'Changed style "BlockText" to "PL" because it is preceded by a paragraph with style "EW" (line {prev_para.sourceline})')

                break

            

    print(f'Changed style "BlockText/Notes" to "NO" {counter_no} times and "BlockText/Examples" to "EW" {counter_ew} times')
@@ -2233,7 +2253,7 @@ def update_references(docx_input, docx_output): 
        # Keep Editor's Notes untouched here; they are formatted later in

        # update_body_text_style (including highlighting).

        para_text = _paragraph_text_from_runs(para)

        if re.search(r'Editor[\'’]s\s+Note:', para_text, re.IGNORECASE):

        if re.search(r'Editor(?:[\'’]?s)?\s+Note:', para_text, re.IGNORECASE):

            continue



        # A reference is a paragraph that contains:
@@ -3027,6 +3047,7 @@ def update_format_styles_cli(): 
        ("update_table_captions", update_table_captions),

        ("update_abbreviations", update_abbreviations),

        ("update_table_rows", update_table_rows),

        ("update_notes", update_notes),

        ("update_references", update_references),

        ("update_lists", update_lists),

        ("repair_invalid_numbering_references", repair_invalid_numbering_references),
@@ -3036,8 +3057,7 @@ def update_format_styles_cli(): 
        ("update_source_code_style", update_source_code_style),

        ("add_break_after_code_blocks_and_tables", add_break_after_code_blocks_and_tables),

        ("update_equation_style", update_equation_style),

        ("correct_quotes_docx", correct_quotes_docx),

        ("update_notes", update_notes)

        ("correct_quotes_docx", correct_quotes_docx)

    ]



    for step_name, step_func in format_style_steps:

generateSpecWebSite/gridTableTools.py

+5 −5
Original line number Diff line number Diff line
@@ -10,9 +10,9 @@ from typing import Optional, Callable 
from regexMatches import *





_alignLeft = 'align="left"'

_alignRight = 'align="right"'

_alignCenter = 'align="center"'

_alignLeft = 'style="text-align:left"'

_alignRight = 'style="text-align:right"'

_alignCenter = 'style="text-align:center"'

_nextListElementMark = '∆'	# Marks a continuing list in the line before. !!! Must be a single character
@@ -724,7 +724,7 @@ def generateHtmlTableWithSpans(gridTable:str) -> str: 


					rowspan = f' rowspan="{cell.rowspan}"' if cell.rowspan > 1 else ''

					colspan = f' colspan="{cell.colspan}"' if cell.colspan > 1 else ''

					html +=   f'<th{rowspan}{colspan} {cell.alignment} markdown="1">{cell.content}</th>\n'

					html +=   f'<th{rowspan}{colspan} {cell.alignment} markdown="block">{cell.content}</th>\n'

			html += '</tr>\n'

		html += '</thead>\n'
@@ -765,7 +765,7 @@ def generateHtmlTableWithSpans(gridTable:str) -> str: 
				rowspan = f' rowspan="{cell.rowspan}"' if cell.rowspan > 1 else ''

				colspan = f' colspan="{cell.colspan}"' if cell.colspan > 1 else ''

				#html = '\n'.join([html, f'<td{rowspan}{colspan} {cell.alignment} markdown="1">','','', multilineContent,'','', '</td>',''])

				html +=   f'<td{rowspan}{colspan} {cell.alignment} markdown="1">\n\n{cell.content}\n\n</td>\n'

				html +=   f'<td{rowspan}{colspan} {cell.alignment} markdown="block">\n\n{cell.content}\n\n</td>\n'

		html += '</tr>\n'



	html += '</tbody>\n'

generateSpecWebSite/spec_on_pages.sh

+46 −12
Original line number Diff line number Diff line
@@ -16,9 +16,9 @@ echo "toMkdocs.py:" $1 
#$MIKE_ACCESS_TOKEN -> 2

MIKE_ACCESS_TOKEN=${2}

echo "MIKE_ACCESS_TOKEN:" ${MIKE_ACCESS_TOKEN}

#CI_SERVER_HOST -> 3

CI_SERVER_HOST=${3}

echo "CI_SERVER_HOST:" ${CI_SERVER_HOST}

#CI_SERVER_FQDN -> 3

CI_SERVER_FQDN=${3}

echo "CI_SERVER_FQDN:" ${CI_SERVER_FQDN}

#CI_PROJECT_PATH -> 4

CI_PROJECT_PATH=${4}

echo "CI_PROJECT_PATH:" ${CI_PROJECT_PATH}
@@ -46,7 +46,7 @@ echo "CLEAN_WEB_PAGES:" ${CLEAN_WEB_PAGES} 


#BACKUP=false

PAGES_BRANCH=gl-pages

HTTPS_REMOTE=https://gitlab-ci-token:${MIKE_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git

HTTPS_REMOTE=https://gitlab-ci-token:${MIKE_ACCESS_TOKEN}@${CI_SERVER_FQDN}/${CI_PROJECT_PATH}.git



echo "\n------ Install necessary packages --------"

pip install -q mkdocs-material mike rich mkdocs-rss-plugin
@@ -60,7 +60,7 @@ git checkout ${CI_COMMIT_SHA} 
####### GENERATE NAV SPEC #######

echo "------ Process MD Spec --------"

frontmatterClause=False

python3 /markdownTools/processMDSpec.py -fmo "./${SPEC_NAME}.md" -fmf frontmatter.md

python3 /markdownTools/processMDSpec.py -fmo "${SPEC_NAME}.md" -fmf frontmatter.md

if [ -f "frontmatter.md" ] && [ "$(cat frontmatter.md | tr -d ' \n\t')" != "{}" ]; then

  echo "------ Adding frontmatter to spec.md --------"

  python3 /generateSpecWebSite/create_frontmatter_table.py frontmatter.md spec.md
@@ -68,14 +68,14 @@ if [ -f "frontmatter.md" ] && [ "$(cat frontmatter.md | tr -d ' \n\t')" != "{}" 
  cat spec.md

  frontmatterClause=True

fi

python3 /markdownTools/processMDSpec.py --no-expand-paths "./${SPEC_NAME}.md" > combined.md

python3 /markdownTools/processMDSpec.py "${SPEC_NAME}.md" > combined.md

echo "------ Generate input for mkdocs --------"

#python3 $1 -it -ihp --title ${9^^} "./$8.md"

if [ $frontmatterClause == True ]; then

  cat ./combined.md >> ./spec.md

  python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./spec.md"

  python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "spec.md"

else

  python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "./${SPEC_NAME}.md"

  python3 /generateSpecWebSite/toMkdocs.py -it -ihp --title ${CI_PROJECT_NAME^^} "${SPEC_NAME}.md"

fi

ls ${CI_PROJECT_NAME^^}

echo "------ Move to docs folder --------"
@@ -84,8 +84,18 @@ sed -i 's/'${CI_PROJECT_NAME^^}'\///g' _nav.yml 
cat _nav.yml

echo "  - 'Home': 'index.md'" >> mkdocs.yml

cat _nav.yml >> mkdocs.yml

if [ -f docs/0.md ]; then

    python3 -c "

import re

from pathlib import Path

p = Path('docs/0.md')

text = p.read_text(encoding='utf-8')

p.write_text(re.sub(r'<!--.*?-->', '', text, flags=re.DOTALL), encoding='utf-8')

"

fi

cat docs/0.md

cp docs/0.md docs/index.md

repo_url="https://${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"

repo_url="https://${CI_SERVER_FQDN}/${CI_PROJECT_PATH}.git"

echo ${repo_url}

sed -i 's/##PROJECT/'${CI_PROJECT_NAME^^}'/g' mkdocs.yml

repo_url=$(echo ${repo_url} | sed 's/\//\\\//g')
@@ -94,7 +104,13 @@ sed -i 's/##REPO_NAME/'${CI_PROJECT_NAME^^}'/g' mkdocs.yml 


sed --version



echo "------ Content of mkdocs.yml before adding download and release notes tabs--------"

cat mkdocs.yml

echo "------ Validate mkdocs.yml before adding download and release notes tabs--------"

python3 -c "from mkdocs.config import load_config; load_config('mkdocs.yml')" || {

    echo "mkdocs.yml failed validation; see output above."

    exit 1

}

## Create download tab if official baseline version

if [[ ${TAG_NAME} == v* ]]; then

    echo "\nAdding download tab for version ${TAG_NAME}..."
@@ -131,10 +147,19 @@ if [[ ${TAG_NAME} == v* ]]; then 
    echo "  - Release Notes: 'release_notes/index.md'" >> mkdocs.yml

fi





echo "------ Content of mkdocs.yml after adding download and release notes tabs--------"

cat mkdocs.yml

echo "------ Validate mkdocs.yml after adding download and release notes tabs--------"

python3 -c "from mkdocs.config import load_config; load_config('mkdocs.yml')" || {

    echo "mkdocs.yml failed validation; see output above."

    exit 1

}



if [ ${CLEAN_WEB_PAGES} == "false" ]; then

    echo "------ Generating site --------"

    echo "Generating docs version ${TAG_NAME}..."

    majorVersion=$(echo ${TAG_NAME} | grep -o v[0-9]*\. | grep -o [0-9]*)

    majorVersion=$(echo ${TAG_NAME} | grep -oE v[0-9]+\. | grep -oE [0-9]+)

    interVersion=$(echo ${TAG_NAME} | cut -d. -f2)

    minorVersion=$(echo ${TAG_NAME} | cut -d. -f3)

    echo "Major version is ${majorVersion}"
@@ -220,7 +245,7 @@ if [ ${CLEAN_WEB_PAGES} == "false" ]; then 
    fi

else

    echo "------ Removing docs version  }..."

    majorVersion=$(echo ${CLEAN_WEB_PAGES} | grep -o v[0-9]*\. | grep -o [0-9]*)

    majorVersion=$(echo ${CLEAN_WEB_PAGES} | grep -oE v[0-9]+\. | grep -oE [0-9]+)

    if [ -n "${majorVersion}" ]; then

        previousVersion=$(mike list --deploy-prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH | grep v${majorVersion} | grep -A1 ${CLEAN_WEB_PAGES} | grep -v ${CLEAN_WEB_PAGES})

        releaseVersion="latestR${majorVersion}"
@@ -262,10 +287,19 @@ else 
    mike delete --deploy-prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH ${CLEAN_WEB_PAGES}

fi



git checkout $PAGES_BRANCH -- public/

git checkout "${PAGES_BRANCH}" -- public/

#if git show-ref --verify --quiet "refs/heads/${PAGES_BRANCH}" \

#    || git show-ref --verify --quiet "refs/remotes/origin/${PAGES_BRANCH}"; then

#    git fetch origin "${PAGES_BRANCH}" 2>/dev/null || true

#    git checkout "${PAGES_BRANCH}" -- public/ 2>/dev/null \

#        || echo "public/ not present on ${PAGES_BRANCH} yet."

#else

#    echo "No ${PAGES_BRANCH} branch yet; skipping public/ checkout."

#fi



echo "Content of public ..."

ls public/

#ls public/ 2>/dev/null || echo "public/ not available"



exit 0