Script to split markdown files into clauses. Generate the mkdocs navigation...

#

#	toMkdocs.py

#

#	(c) 2024 by Andreas Kraft

#

#	This script converts oneM2M spec markdown file to a mkdocs compatible

#	directory structure.

#

import argparse, re, os, shutil

from dataclasses import dataclass

from rich import print

@dataclass

class Clause:

	"""	Represents a clause in the markdown file. """

	level:int

	title:str

	lines:list[str]

	onlyNav:bool = False

_matchHeader = re.compile(r'(#+)\s+(.*)', re.IGNORECASE)

_matchCodefence = re.compile(r'\s*```\s?.*', re.IGNORECASE)

_match2spaceListIndention = re.compile(r'^\s{2}-', re.IGNORECASE)

# TODO handle multiple nav levels (left bar) better (make conifgurable)

# TODO Update links in the markdown files to the new structure

def analyseMarkdown(filename:str) -> list[Clause]:

	"""	Analyse the markdown file and split it into clauses.

		Args:

			filename: The name of the markdown file.

		Returns:

			The list of clauses.

	"""

	with open(filename, 'r') as file:

		inLines = file.readlines()

	outLines:list[Clause] = [Clause(0, '', [])]

	# Go through the lines and detect headers and codefences

	inCodefence = False

	for line in inLines:

		# Detect codefences

		if _matchCodefence.match(line):

			inCodefence = not inCodefence

		if inCodefence:

			outLines[-1].lines.append(line)

			continue

		# Detect headers

		if (m := _matchHeader.match(line)):

			level = len(m.groups()[0])

			clauseTitle = m.groups()[1].strip()

			outLines.append(Clause(level, clauseTitle, []))

		outLines[-1].lines.append(line)

	return outLines

def splitMarkdownDocument(clauses:list[Clause], 

						  ignoreTitles:list[str] = [], 

						  splitLevel:int = 1,

						  ignoreUntilFirstHeading:bool = True) -> list[Clause]:

	"""	Split the clauses at a certain level. This is used to create separate

		markdown files for MkDocs.

		Args:

			clauses: The list of clauses.

			ignoreTitles: A list of titles that should be ignored. They are not included in the output.

			splitLevel: The level at which the clauses should be split.

			ignoreUntilFirstHeader: Ignore all clauses until the first heading.

		Returns:

			The list of clauses.

	"""

	outLines:list[Clause] = [Clause(0, '', [])]

	for clause in clauses:

		level = clause.level

		# Check if the current clause should be ignored

		if clause.title.casefold() in ignoreTitles:

			continue

		# Add a new output clause if the current clause's level is 

  		# equal or less than the split level

		if clause.level <= splitLevel:

			outLines.append(Clause(level, clause.title, []))

		# Add the lines to the output clause

		outLines[-1].lines.extend(clause.lines)

	# Remove the first clause if it has no title

	if ignoreUntilFirstHeading:

		while len(outLines[0].title) == 0:

			outLines.pop(0)

	return outLines

def prepareForMkdocs(clauses:list[Clause]) -> list[Clause]:

	"""	Prepare the clauses for MkDocs. This includes removing the heading

		from the clauses and marking the clauses that are only for navigation.

		Args:

			clauses: The list of clauses.

		Returns:

			The list of clauses.

	"""

	# Remove the heading from the lines. The heading is the first line

	# in the clause. This is done because MkDocs repeats the heading when

	# displaying the page.

	for clause in clauses:

		if len(clause.lines) > 0:

			clause.lines.pop(0)

			# Also, remove the first empty lines if they exist

			while len(clause.lines) > 0 and clause.lines[0].strip() == '':

				clause.lines.pop(0)

	# Mark the whole clause if it is the first AND NOT only clause

	# for a parent clause. Then it is usually empty except the heading.

	# We still need it for navigation, so we mark it as onlyNav

	for clause in clauses:

		if len(''.join(clause.lines).strip()) == 0 and clause.level > 0:

			clause.onlyNav = True

	# Repair wrong markdown for indented lines.

	# Add 2 spaces to existing 2-space indentions

	for clause in clauses:

		for i, line in enumerate(clause.lines):

			if _match2spaceListIndention.match(line):

				clause.lines[i] = '  ' + line

	return clauses

def writeClauses(outLines:list[Clause], filename:str, navTitle:str) -> None:

	"""	Write the clauses to separate files and create a navigation file.

		Args:

			outLines: The list of clauses.

			filename: The name of the original markdown file.

			navTitle: The title of the navigation entry. This is used to determine the directories.

	"""

	# Write the files

	# create directory first

	os.makedirs(f'{os.path.dirname(filename)}/{navTitle}', exist_ok = True)

	for i, f in enumerate(outLines):

		if len(f.lines) == 0:	# ignore empty clauses

			continue

		# Don't write content files that are only for navigation

		if f.onlyNav:

			continue

		# write to single files

		with open(f'{os.path.dirname(filename)}/{navTitle}/{i}.md', 'w') as file:

			file.writelines(f.lines)

			print(f'[green]File "{i}.md" written')

	# write nav.yml file

	with open(f'{os.path.dirname(filename)}/_nav.yml', 'w') as file:

		file.write(f'  - {navTitle}:\n')

		for i, f in enumerate(outLines):

			if f.onlyNav:

				file.write(f"  {'  '*f.level}- '{f.title}':\n")

			else:

				if len(f.lines) == 0:

					continue

				file.write(f"  {'  '*f.level}- '{f.title}': '{navTitle}/{i}.md'\n")

	print(f'[green]File "_nav.yml" written')

def copyMediaFiles(filename:str, navTitle:str, mediaDirectory:str = 'media') -> None:

	"""	Copy media files from the source directory to the target directory.

		Args:

			filename: The name of the markdown file.

			navTitle: The title of the navigation entry.

			mediaDirectory: The name of the media directory.

	"""

	sourceDirectory = f'{os.path.dirname(filename)}/{mediaDirectory}'

	targetDirectory = f'{os.path.dirname(filename)}/{navTitle}/{mediaDirectory}'

	if os.path.exists(sourceDirectory):

		print(f'[green]Copying media files from "{sourceDirectory}" to "{targetDirectory}"')

		shutil.copytree(sourceDirectory, targetDirectory, dirs_exist_ok = True)

	else:

		print(f'[red]Media directory "{sourceDirectory}" does not exist')

def processDocument(args:argparse.Namespace) -> None:

	clauses = analyseMarkdown(args.document)

	clauses = splitMarkdownDocument(clauses, [ t.casefold() for t in args.ignore_clause ], args.split_level)

	clauses = prepareForMkdocs(clauses)

	writeClauses(clauses, args.document, args.title)

	copyMediaFiles(args.document, args.title, args.media_directory)

if __name__ == '__main__':

	parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)

	parser.add_argument('--title', '-t', metavar = 'title', required = True, help = 'mkdocs navigation tile')

	parser.add_argument('--ignore-clause', '-i', metavar = 'clause', nargs = '+', default = [ 'Contents', 'History' ], help = 'ignore headers in the markdown document')

	parser.add_argument('--split-level', '-sl', metavar = 'level', type = int, default = 2, help = 'split clauses on which level')

	parser.add_argument('--media-directory', '-md', metavar = 'media-directory', default = 'media', help = 'directory where media files are stored')

	parser.add_argument('document', type = str, help = 'a oneM2M markdown specification document to process')

	args = parser.parse_args()

	processDocument(args)