build_parameters.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    This script aims to provide multiple parameters source files for each vehicle based on the versions available at
    https://firmware.ardupilot.org

    It is intended to be run on the main wiki server. TO-DO: run locally within the project's Vagrant environment.

    Build notes:

    * Before start:
        * Folder and file management tested only in linux;
        * It is supposed to have the wiki repo in a same level of an ArduPilot repo and two other folders
        named new_params_mvesion/ and old_params_mversion/

    * First step is go to each vehicle on firmware.ardupilot.org and get all available versions namely as stable/beta/latest.
        * For each version it gets a board and get git_version.txt file;
        * It parsers that to get the version and commit hash;
        * It creates a dictionary with vehicles, versions and commit hashes.

    * Second step is use the dict to navigates on a ArduPilot Repo, changing checkouts to desidered hashes.
        * Relies on ArduPilotRepoFolder/Tools/autotest/param_metadata/param_parse.py to generate the parameters files;
        * It renames the anchors for all files, except for latest versions.

    * Third step: create the json files and move all generated files.

"""

import os
import shutil  # noqa: F401
import sys
import time  # noqa: F401
from html.parser import HTMLParser
import urllib.request
import re
import glob
import argparse

parser = argparse.ArgumentParser(description="python3 build_parameters.py [options]")
parser.add_argument("--verbose", dest='verbose', action='store_false', default=True, help="show debugging output")
parser.add_argument("--ardupilotRepoFolder", dest='gitFolder', default="../ardupilot", help="Ardupilot git folder. ")
parser.add_argument("--destination", dest='destFolder', default="../../../../new_params_mversion", help="Parameters*.rst destination folder.")  # noqa: E501
parser.add_argument('--vehicle', dest='single_vehicle', help="If you just want to copy to one vehicle, you can do this. Otherwise it will work for all vehicles (Copter, Plane, Rover, AntennaTracker, Sub, Blimp)")  # noqa: E501
args = parser.parse_args()

error_count = 0

# Parameters
COMMITFILE = "git-version.txt"
BASEURL = "https://firmware.ardupilot.org/"
ALLVEHICLES = ["AntennaTracker", "Copter" , "Plane", "Rover", "Blimp"]
VEHICLES = ALLVEHICLES

BASEPATH = ""

# Dicts for name replacing
vehicle_new_to_old_name = { # Used because "param_parse.py" args expect old names
    "Rover": "APMrover2",
    "Sub": "ArduSub",
    "Copter": "ArduCopter",
    "Plane": "ArduPlane",
    "AntennaTracker": "AntennaTracker",  # firmware server calls Tracker as AntennaTracker
    "Blimp": "Blimp",
}

vehicle_old_to_new_name = { # Used because git-version.txt use APMVersion with old names
    "APMrover2": "Rover",
    "ArduRover": "Rover",
    "ArduSub": "Sub",
    "ArduCopter": "Copter",
    "ArduPlane": "Plane",
    "AntennaTracker": "AntennaTracker",  # firmware server calls Tracker/atennatracker as AntennaTracker
    "Blimp": "Blimp",
}


def debug(str_to_print):
    """Debug output if verbose is set."""
    if args.verbose:
        print("[build_parameters.py] " + str(str_to_print))


def error(str_to_print):
    """Show and count the errors."""
    global error_count
    error_count += 1
    print("[build_parameters.py][error]: " + str(str_to_print))


def check_temp_folders():
    """Creates temporaty subfolders IFF not exists  """

    os.chdir("../")
    if not os.path.exists("new_params_mversion"):
        os.makedirs("new_params_mversion")
    os.chdir("new_params_mversion")
    for vehicle in ALLVEHICLES:
        if not os.path.exists(vehicle_new_to_old_name[vehicle]):
            os.makedirs(vehicle_new_to_old_name[vehicle])

    os.chdir("../")
    if not os.path.exists("old_params_mversion"):
        os.makedirs("old_params_mversion")
    os.chdir("old_params_mversion")
    for vehicle in ALLVEHICLES:
        if not os.path.exists(vehicle_new_to_old_name[vehicle]):
            os.makedirs(vehicle_new_to_old_name[vehicle])


def setup():
    """
    Goes to the work folder, clean it and update.
    """

    # Test for a single vehicle run
    if args.single_vehicle in ALLVEHICLES:
        global VEHICLES
        VEHICLES = [args.single_vehicle]
        print("[build_parameters.py] Running only for " + str(args.single_vehicle))
    else:
        print("[build_parameters.py] Vehicle %s not recognized, running for all vehicles." % str(args.single_vehicle))

    try:
        # Goes to ardupilot folder and clean it and update to make sure that is the most recent one.
        debug("Recovering from a previous run...")
        os.chdir(args.gitFolder)
        os.system("git reset --hard HEAD")
        os.system("git clean -f -d")
        os.system("git checkout -f master")
        os.system("git fetch origin master")
        os.system("git reset --hard origin/master")
        os.system("git pull")
        os.system("git submodule update --init --recursive")
        global BASEPATH
        BASEPATH = os.getcwd()
        check_temp_folders()
    except Exception as e:
        error("ArduPilot Repo folder not found (cd %s failed)" % args.gitFolder)
        error(e)
        sys.exit(1)
    finally:
        debug("\nThe current working directory is %s" % BASEPATH)


def fetch_releases(firmware_url, vehicles):
    """
    Select folders with desidered releases for the vehicles

    """

    def fetch_vehicle_subfolders(firmware_url):
        """
        Fetch firmware.ardupilot.org/baseURL all first level folders for a given base URL.

        """
        links = []
        # Define HTML Parser

        class parseText(HTMLParser):
            def handle_starttag(self, tag, attrs):
                if tag != 'a':
                    return
                attr = dict(attrs)
                links.append(attr)
        # Create instance of HTML parser
        lParser = parseText()
        # Feed HTML file into parsers
        try:
            debug("Fetching " + firmware_url)
            lParser.feed(urllib.request.urlopen(firmware_url).read().decode('utf8'))
        except Exception as e:
            error("Folders list download error: " + e)
            sys.exit(1)
        finally:
            lParser.links = []
            lParser.close()
            return links
    ######################################################################################

    debug("Cleaning fetched links for wanted folders")
    stableFirmwares = []
    for f in vehicles:
        page_links = fetch_vehicle_subfolders(firmware_url + f)
        for folder in page_links:  # Non clever way to filter the strings insert by makehtml.py, unwanted folders, and so.
            version_folder = str(folder)
            if version_folder.find("stable") > 0 and not version_folder.endswith("stable"): # If finish with
                stableFirmwares.append(firmware_url[:-1] + version_folder[10:-2])
            elif version_folder.find("latest") > 0 :
                stableFirmwares.append(firmware_url[:-1] + version_folder[10:-2])
            elif version_folder.find("beta") > 0:
                stableFirmwares.append(firmware_url[:-1] + version_folder[10:-2])

    return stableFirmwares # links for the firmwares folders


def get_commit_dict(releases_parsed):
    """
    For informed releases, return a dict git hashs of its build.

    """
    def get_last_board_folder(url):
        """
        For given URL returns the last folder which should be a board name.

        """
        links = []
        # Define HTML Parser

        class parseText(HTMLParser):
            def handle_starttag(self, tag, attrs):
                if tag != 'a':
                    return
                attr = dict(attrs)
                links.append(attr)
        # Create instance of HTML parser
        lParser = parseText()
        # Feed HTML file into parsers
        try:
            debug("Fetching " + url)
            lParser.feed(urllib.request.urlopen(url).read().decode('utf8'))
        except Exception as e:
            error("Folders list download error:" + e)
        finally:
            lParser.links = []
            lParser.close()
            last_item = links.pop()
            last_folder = last_item['href']
            debug("Returning link of the last board folder (" + last_folder[last_folder.rindex('/')+1:] + ")")
            return last_folder[last_folder.rindex('/')+1:]  # clean the partial link
    ####################################################################################################

    def fetch_commit_hash(version_link, board, file):
        """
        For a binnary folder, gets a git hash of its build.

        """
        fetch_link = version_link + '/' + board + '/' + file

        print("Processing link...\t" + fetch_link)

        try:
            fecth_response = ""
            with urllib.request.urlopen(fetch_link) as response:
                fecth_response = response.read().decode("utf-8")

            commit_details = fecth_response.split("\n")
            commit_hash = commit_details[0][7:]
            # version =  commit_details[6] the sizes cary
            version = commit_details.pop(-2)

            version_number = version.split(" ")[2]
            vehicle = version.split(" ")[1]

            regex = re.compile('[@_!#$%^&*()<>?/\|}{~:]')  # noqa: W605

            if (regex.search(vehicle) is None):  # there are some non standart names
                vehicle = vehicle_old_to_new_name[vehicle.strip()]   # Names may not be standart as expected
            else:
                # tries to fix automatically
                if re.search('copter', vehicle, re.IGNORECASE):
                    vehicle = "Copter"
                    debug("Bad vehicle name auto fixed to COPTER on:\t" + fetch_link)

                elif re.search('plane', vehicle, re.IGNORECASE):
                    vehicle = "Plane"
                    debug("Bad vehicle name auto fixed to PLANE on:\t" + fetch_link)

                elif re.search('rover', vehicle, re.IGNORECASE):
                    vehicle = "Rover"
                    debug("Bad vehicle name auto fixed to ROVER on:\t" + fetch_link)

                elif re.search('sub', vehicle, re.IGNORECASE):
                    vehicle = "Sub"
                    debug("Bad vehicle name auto fixed to SUB on:\t" + fetch_link)

                elif re.search('racker', vehicle, re.IGNORECASE):
                    vehicle = "Tracker"
                    debug("Bad vehicle name auto fixed to TRACKER on:\t" + fetch_link)
                elif re.search('blimp', vehicle, re.IGNORECASE):
                    vehicle = "Blimp"
                    debug("Bad vehicle name auto fixed to BLIMP on:\t" + fetch_link)
                else:
                    error("Nomenclature exception found in a vehicle name:\t" + vehicle + "\tLink with the exception:\t" + fetch_link)  # noqa: E501

            if "beta" in fetch_link:
                version_number = "beta-" + version_number

            if "latest" in fetch_link:
                version_number = "latest-" + version_number

            return vehicle, version_number, commit_hash
        except Exception as e:
            error("An exception occurred: " + file + " DECODE ERROR. Link: " + fetch_link)
            error(e)
            # sys.exit(1) #comment to make easer debug
            return "error", "error", "error"
    ####################################################################################################

    commits_and_codes = {}
    commite_and_codes_cleanned = {}

    for j in range(0, len(releases_parsed)):
        commits_and_codes[j] = fetch_commit_hash(releases_parsed[j], get_last_board_folder(releases_parsed[j]), COMMITFILE)

    for i in commits_and_codes:
        if commits_and_codes[i][0] != 'error':
            commite_and_codes_cleanned[i] = commits_and_codes[i]
    return commite_and_codes_cleanned


def generate_rst_files(commits_to_checkout_and_parse):
    """
    For each git hash it generates its Parameters file.
    """

    def replace_anchors(source_file, dest_file, version_tag):
        """
        For each parameter file generate by param_parse.py, it inserts a version tag in anchors
        to do not make confusing in sphinx toctrees.
        """
        file_in = open(source_file, "r")
        file_out = open(dest_file, "w")
        found_original_title = False
        if "latest" not in version_tag:
            file_out.write(':orphan:\n\n')

        for line in file_in:
            if (re.match("(^.. _)(.*):$", line)) and ("latest" not in version_tag):
                file_out.write(line[0:-2] + version_tag + ":\n")  # renames the anchors, but leave latest anchors "as-is" to maintaim compatibility with all links across the wiki  # noqa: E501

            elif "Complete Parameter List" in line:
                # Adjusting the page title
                out_line = "Complete Parameter List\n=======================\n\n"
                out_line += "\n.. raw:: html\n\n"
                out_line += "   <h2>Full Parameter List of " + version_tag[1:].replace("-", " ") + "</h2>\n\n"  # rename the page identifier to insert the version  # noqa: E501

                # Pigbacking and inserting the javascript selector
                out_line += "\n.. raw:: html\n   :file: ../_static/parameters_versioning_script.inc\n\n"
                file_out.write(out_line)

            elif ("=======================" in line) and (not found_original_title): # Ignores the original mark
                found_original_title = True

            else:
                file_out.write(line)

    for i in commits_to_checkout_and_parse:
        vehicle = str(commits_to_checkout_and_parse[i][0])
        version = str(commits_to_checkout_and_parse[i][1])
        commit_id = str(commits_to_checkout_and_parse[i][2])

        # Not elegant workaround:
        # These versions present errors when parsing using param_parser.py. Needs more investigation?
        if (
            "3.2.1" in version or # last stable APM Copte
            "3.4.0" in version or # last stable APM Plane
            "3.4.6" in version or # Copter
            "2.42" in version or  # last stable APM Rover?
            "2.51" in version or  # last beta APM Rover?
            "0.7.2" in version    # Antennatracker
        ):
            debug("Ignoring APM version:\t" + vehicle + "\t" + version)
            continue

        # Checkout an Commit ID in order to get its parameters
        try:
            debug("Git checkout on " + vehicle + " version " + version + " id " + commit_id)
            os.system("git checkout --force " + commit_id)

        except Exception as e:
            error("GIT checkout error: " + e)
            sys.exit(1)
        debug("")

        # Run param_parse.py tool from Autotest set in the desidered commit id
        try:
            os.chdir(BASEPATH + "/Tools/autotest/param_metadata")
            if ('rover' in vehicle.lower()) and ('v3.' not in version.lower()) and ('v4.0' not in version.lower()): # Workaround the vehicle renaming (Rover, APMRover2 ArduRover...)  # noqa: E501
                os.system("python3 ./param_parse.py --vehicle " + 'Rover')
            else: # regular case
                os.system("python3 ./param_parse.py --vehicle " + vehicle_new_to_old_name[vehicle])  # option "param_parse.py --format rst" is not available in all commits where param_parse.py is found  # noqa: E501

            # create a filename for new parameters file
            filename = "parameters-" + vehicle
            if ("beta" in version or "rc" in version): # Plane uses BETA, Copter and Rover uses RCn
                filename += "-" + version  + ".rst"
            elif ("latest" in version):
                filename += "-" + version + ".rst"
            else:
                filename += "-stable-" + version + ".rst"

            # Generate new anchors names in files to avoid toctree problems and links in sphinx.
            if os.path.exists("Parameters.rst"):
                replace_anchors("Parameters.rst", filename, filename[10:-4])
                os.remove("Parameters.rst")
                debug("File " + filename + " generated. ")
            else:
                error("Parameters.rst not found to rename to  %s" % filename)

            os.chdir(BASEPATH)
        except Exception as e:
            error("Error while parsing \"Parameters.rst\" | details:\t" + vehicle + "\t" + version  + "\t" + commit_id)
            error(e)
            # sys.exit(1)
        debug("")

    return 0


def generate_json(vehicles):
    """
        Generates a JSON with all parameters page to be live consumed by a javascript
    """

    os.chdir(BASEPATH + "/Tools/autotest/param_metadata")

    for vehicle in vehicles:

        debug("Creating JSON files for " + vehicle)

        # Creates the JSON lines from available rst files
        parameters_files = [f for f in glob.glob("parameters-" + vehicle + "*.rst")]
        parameters_files.sort(reverse=True)

        json_lines = []
        json_lines.append("{")
        json_lines.append("\"Click here to change\" : \"\"")

        for filename in parameters_files:
            if ("beta" in filename or "rc" in filename): # Plane uses BETA, Copter and Rover uses RCn
                json_lines.append(",\"" + vehicle + " beta " + filename[(len("parameters-" + vehicle + "-beta")+1):-4] + "\" : \"" + filename[:-3] + "html\"")  # noqa: E501
            elif ("latest" in filename):
                # json_lines.append(",\"" +  vehicle + " latest " + filename[(len("parameters-" + vehicle + "-latest")+1):-4] + "\" : \"" + filename[:-3] + "html\"")  # noqa: E501
                json_lines.append(",\"" + vehicle + " latest " + filename[(len("parameters-" + vehicle + "-latest")+1):-4] + "\" : \"" + ("parameters.html\""))  # Trying to re-enable toc list on the left bar on the wiki by forcing latest file name.  # noqa: E501

            else:
                json_lines.append(",\"" + vehicle + " stable " + filename[(len("parameters-" + vehicle + "-stable")+1):-4] + "\" : \"" + filename[:-3] + "html\"")  # noqa: E501

        json_lines.append("}")

        # Saves the JSON
        json_filename = "parameters-" + vehicle + ".json"
        try:
            with open(json_filename, 'w') as f:
                for lines in json_lines:
                    f.write("%s\n" % lines)
        except Exception as e:
            error("Error while creating the JSON file " + vehicle + " in folder " + str(os.getcwd()))  # noqa: E501
            error(e)
            # sys.exit(1)
        debug("")


def move_results(vehicles):
    """
        Once all parameters files are created, moves for "new_params_mversion" as the last execution result
    """

    os.chdir(BASEPATH + "/Tools/autotest/param_metadata")

    for vehicle in vehicles:
        debug("Moving created files for " + vehicle)
        try:
            folder = args.destFolder + "/" + vehicle_new_to_old_name[vehicle] + "/"

            # touch the folders
            if not os.path.exists(args.destFolder):
                os.makedirs(args.destFolder)
            if not os.path.exists(folder):
                os.makedirs(folder)

            # Cleanning last run, iff exists
            files_to_delete = [f for f in glob.glob(folder + "*")]
            for old_file in files_to_delete:
                os.remove(old_file)

            # Moving files.
            files_to_move = [f for f in glob.glob("parameters-" + vehicle + "*")]
            for file in files_to_move:
                if ("latest" not in file):  # Trying to re-enable toc list on the left bar on the wiki by forcing latest file name.  # noqa: E501
                    os.rename(file, folder + file)
                else:
                    os.rename(file, str((folder + "parameters.rst")))

        except Exception as e:
            error("Error while moving result files of vehicle " + vehicle + " pwd: " + str(os.getcwd()))
            error(e)
            # sys.exit(1)


def print_versions(commits_to_checkout_and_parse):
    """ Partial results: present all vechicles, versions and commits selected to generate parameters """
    debug("\n\tList of parameters files to generate:\n")
    for i in commits_to_checkout_and_parse:
        debug(commits_to_checkout_and_parse[i][0] + ' - ' + commits_to_checkout_and_parse[i][1] + ' - ' + commits_to_checkout_and_parse[i][2])  # noqa: E501
    debug("")


# Step 1 - Select the versions for generate parameters
setup()                                                             # Reset the ArduPilot folder/repo
feteched_releases = fetch_releases(BASEURL, VEHICLES)               # All folders/releases.
commits_to_checkout_and_parse = get_commit_dict(feteched_releases)  # Parse names, and git hashes.
print_versions(commits_to_checkout_and_parse)                       # Present work dict.

# Step 2 - Generates them in ArdupilotRepoFolder/Tools/autotest/param_metadata
generate_rst_files(commits_to_checkout_and_parse)

# Step 3 - Generates a JOSN file for each vehicle and mode files to folder new_params_mversion
generate_json(VEHICLES)
move_results(VEHICLES)

# sys.exit(error_count)