Commit 0dbbb63b authored by Miguel Angel Reina Ortega's avatar Miguel Angel Reina Ortega
Browse files

Several fixes: 

- Get the indexes of the change
- Detect the changed clause
- Correct generation of the MD with changes marks
parent c3a3eb3d

generateChangemarks/changemarks.py

+76 −36
Original line number Diff line number Diff line
@@ -92,14 +92,15 @@ def find_all_clauses(progress:Progress, mdLines:list[str]): 
    empty = ""

    clause = Clause(empty,0,0,"0")



    for line in mdLines:

        if line.startswith('#'):

            # Clause 0 (from start to first clause) found

            clause.to_id = index - 1

            clauses.append(clause)

            break

        index = index + 1

    #for line in mdLines:

    #    if line.startswith('#'):

    #        # Clause 0 (from start to first clause) found

    #        clause.to_id = index - 1

    #        clauses.append(clause)

    #        break

    #    index = index + 1



    index = 1

    for line in mdLines:

        if line.startswith('#'):

            matches = re.findall(clauseregex, line)	# Match heading
@@ -121,7 +122,12 @@ def find_all_clauses(progress:Progress, mdLines:list[str]): 
    clause.to_id = index - 1

    clauses.append(clause)



    logging.debug("Number of clauses: {len(clauses)}")

    logging.debug(f"Number of clauses: {len(clauses)}")

    for clause in clauses:

        logging.debug(clause.clause_nr)

        logging.debug(clause.from_id)

        logging.debug(clause.to_id)



        return clauses



class MR:
@@ -163,25 +169,28 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st 
            logging.debug(f"Looking at changes in {patched_file.source_file}")

            lines_added = 0

            lines_removed = 0

            previous_change_lines_added = 0

            for change in patched_file:

                logging.debug(f'Change from patch details: source_start: {change.source_start} - target_start: {change.target_start}')

                change_start_line, change_end_line, change_lines_added, change_lines_removed = changeDetails(change)

                logging.debug(f"Change starting at line {change_start_line} and ending at line {change_end_line}")

                logging.debug(f"Change containing {change_lines_added} added lines and {change_lines_removed} lines removed")

                lines_added = lines_added + change_lines_added

                lines_removed = lines_removed + change_lines_removed

                # Check the previous changed_clause

                if (changed_clause.from_id <= change_start_line) and (changed_clause.to_id + lines_added - lines_removed >= change_end_line):

                    generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, True)

                if (changed_clause.from_id <= change_start_line) and (changed_clause.to_id >= change_end_line):

                    generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, True, previous_change_lines_added)

                    continue

                i = 0 

                # Check all clauses

                for clause in clauses:

                    if (clause.from_id <= change_start_line) and (clause.to_id + lines_added - lines_removed >= change_end_line):

                    if (clause.from_id <= change_start_line) and (clause.to_id >= change_end_line):

                        changed_clause = clauses.pop(i)

                        changed_clauses.append(clause)

                        generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, False)

                        generateMDforChange(progress, mdLines, changed_clause, change, outDirectory, False, previous_change_lines_added)

                        break

                    i = i + 1

                previous_change_lines_added = change_lines_added

                logging.info(f"Total number of lines added is {lines_added}")

                logging.info(f"Total number of lines removed is {lines_removed}")
@@ -194,10 +203,17 @@ def changeDetails(change) -> (int, int, int, int): 
    i = 0

    lines_added = 0

    lines_removed = 0

    change_start_line = change.target_start

    change_start_line = change.source_start

    change_end_line = change_start_line

    for line in change:

        #print(vars(line))

        if line.is_added:

            logging.debug(f'Line of change: (ADDED) source_line_no: {line.source_line_no} - target_line_no: {line.target_line_no} - value: {line.value}')

        elif line.is_removed:

            logging.debug(

            f'Line of change: (REMOVED) source_line_no: {line.source_line_no} - target_line_no: {line.target_line_no} - value: {line.value}')

        else:

            logging.debug(f'Line of change: source_line_no: {line.source_line_no} - target_line_no: {line.target_line_no} - value: {line.value}')



        if line.is_added or line.is_removed:

            #if change_start_line == change.target_start:

            #    change_start_line = change.target_start + i
@@ -209,16 +225,21 @@ def changeDetails(change) -> (int, int, int, int): 
                lines_added = lines_added + 1

            elif line.is_removed:

                lines_removed = lines_removed + 1

            if change_start_line == change.target_start:

            if change_start_line == change.source_start:

                if line.is_added:

                    change_start_line = line.target_line_no

                    change_end_line = change_start_line

                    if change.source_start > change.target_start:  # there are more lines removed than lines added

                        change_start_line = line.target_line_no - 1 + change.source_start - change.target_start

                    else:

                        change_start_line = line.target_line_no - 1 + change.target_start - change.source_start  # There are more lines added than removed lines

                elif line.is_removed:

                    change_start_line = line.source_line_no

                    change_start_line = line.source_line_no - 1

                change_end_line = change_start_line

            else:

                if line.is_added:

                    change_end_line = line.target_line_no

                    if change.source_start > change.target_start:

                        change_end_line = line.target_line_no + change.source_start - change.target_start + lines_removed - lines_added  # There are more lines added than removed lines

                    else:

                        change_end_line = line.target_line_no + change.target_start - change.source_start + lines_removed - lines_added

                elif line.is_removed:

                    change_end_line = line.source_line_no

        #i = i + 1
@@ -226,7 +247,7 @@ def changeDetails(change) -> (int, int, int, int): 
    #change_end_line = change_end_line - lines_removed

    return change_start_line, change_end_line, lines_added, lines_removed



def generateMDforChange(progress:Progress, mdLines:list[str],changed_clause:Clause, change, outDirectory:str, existing_clause:bool):

def generateMDforChange(progress:Progress, mdLines:list[str],changed_clause:Clause, change, outDirectory:str, existing_clause:bool, previous_change_lines_added:int):

    '''

    Generate the MD for the clauses that have been modified by the merge request

https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.stfubuntu    Returns a list of Clauses, start index and end index
@@ -243,15 +264,22 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st 
    else:

        clauseMDlines = readMDFile(progress, outDirectory + '/' + changed_clause.clause_nr +'.md')



    logging.info(f"Lines in clause {changed_clause.clause_nr} is {len(clauseMDlines)}")

    if change.source_start >= changed_clause.from_id:

        j = change.source_start - changed_clause.from_id

        index_gap = change.target_start - change.source_start

    elif change.source_start < changed_clause.from_id:

        j = changed_clause.from_id - change.source_start

        index_gap = change.source_start - change.target_start



    j = change.target_start - changed_clause.from_id + index_gap

    if existing_clause:

        j = j + 1

    logging.info(f"Change.source_start {change.source_start}")

    logging.info(f"Changed_clause.from_id {changed_clause.from_id}")



    for line in change:

        if line.source_line_no != None and line.source_line_no < changed_clause.from_id:

            j = 0

            j = j # It should deal with the case where source line number is not within the changed clause index (too many removed lines in other clauses, i.e.)

            continue

        else:

            #if (not (line.value.strip() == '') and (line.is_added)):
@@ -265,16 +293,22 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st 
                            modifiedElements.append("<span class=\"underline\">" + element.strip() + "</span> ")

                    #modifiedRow = "|" + "|".join(modifiedElements) + "|" + "\n"

                        else:

                            modifiedElements.append(element)

                    modifiedRow = "|".join(modifiedElements) + "\n"

                    clauseMDlines.insert(j,modifiedRow)

                            modifiedElements.append(" ")

                    modifiedRow = "|".join(modifiedElements)

                    clauseMDlines.insert(j,modifiedRow + "\n")

                    clauseMDlines.pop(j + 1) # Todo Check what happens when modifying last row of a table

                else:

                    if not line.value.strip() == '':

                        clauseMDlines.insert(j, "<span class=\"underline\">" + line.value.strip() + "</span>\n\n")

                        if line.value.startswith("!["):  # It is a figure

                            clauseMDlines.insert(j, line.value.strip() + "\n")

                            # clauseMDlines.insert(j, "<span class=\"underline\">" + line.value.strip() + "</span>\n") #Track change OK Caption Not OK

                        else:

                            clauseMDlines.insert(j, "<span class=\"underline\">" + line.value.strip() + "</span>\n") #it works for simple lines, not for lines in a list

                        #clauseMDlines.insert(j, "<mark>" + line.value.strip("\n") + "</mark>\n\n")

                        #if (j + 1) <= len(clauseMDlines):

                        #    clauseMDlines.pop(j+1)

                    else:

                       clauseMDlines.insert(j, "<span class=\"underline\">\t<\span>\n") #Add tab to simulate a added line - Not working in lines in list

            #elif (not (line.value.strip() == '') and (line.is_removed)):

            elif line.is_removed:

                if line.value.strip().startswith("|"):  # It is a table
@@ -285,12 +319,18 @@ https://forge.etsi.org/rep/cdm/pipeline-scripts/-/blob/main/common/Dockerfile.st 
                            modifiedElements.append("~~" + element.strip() + "~~ ")

                    #modifiedRow = "|" + "|".join(modifiedElements) + "|" + "\n"

                        else:

                            modifiedElements.append(element)

                    modifiedRow = "|".join(modifiedElements) + "\n"

                    clauseMDlines.insert(j, modifiedRow)

                            modifiedElements.append(" ")

                    modifiedRow = "|".join(modifiedElements)

                    clauseMDlines.insert(j, modifiedRow + "\n")

                    if (j + 1) <= len(clauseMDlines):

                        clauseMDlines.pop(j+1)

                else:

                    logging.debug(f"Index to delete is {j}")

                    if not line.value.strip() == '':

                        clauseMDlines.insert(j, "~~" + line.value.strip() + "~~\n")

                    else:

                        # clauseMDlines.insert(j, "~~\t~~\n")

                        clauseMDlines.insert(j, "\n")

                    if (j + 1) <= len(clauseMDlines):

                        clauseMDlines.pop(j+1)

            j = j + 1
@@ -318,7 +358,7 @@ def main(args=None): 
    parser.add_argument('mergeID',      help="Merge IID")



    pargs = parser.parse_args()

    logging.basicConfig(level=logging.DEBUG)

    logging.basicConfig(level=logging.INFO)



    # Process documents and print output