update-docs.py

#!/usr/bin/python
# coding=utf-8
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse, glob, json, os, re, codecs

DEST_DIR = os.path.dirname(__file__)
OVERLAY_DIR = os.path.join(DEST_DIR, "overlay")
EXAMPLE_DIR = os.path.join(DEST_DIR, "examples")

current_namespace = None
all_namespaces = []

# enums have been moved inline and are no longer referenced
# unique_id is no longer needed
unique_id = 1
additional_type_defs = {}
additional_type_used = []

def merge_objects(a, b):
    if isinstance(a, list):
        for c in a:
            merged = False
            if isinstance(c, dict):
                name = c.get("namespace", c.get("name", c.get("id", c.get("$extend"))))
                if name is not None:
                    for d in b:
                        if d.get("namespace", d.get("name", d.get("id", d.get("$extend")))) == name:
                            merge_objects(c, d)
                            merged = True
            if not merged:
                b.append(c)
                continue
    elif isinstance(a, dict):
        for [e, f] in a.iteritems():
            # choices will be replaced completely as specified
            if e in ["choices"]:
                b[e] = f;
                continue

            # merge existing dicts and lists (former restrictions are a subset)
            if e in b and (isinstance(f, list) or isinstance(f, dict)): # and e not in ["namespace", "name", "id", "$extend"]
                merge_objects(f, b[e])
                continue

            # allow new entries or overrides of descriptions and types
            if e not in b or e in ["description", "$ref", "type"]:
                if (e in b and e in ["description"]):
                    print("Replacing Description")
                    print("  comm-central: " + b[e])
                    print("  overlay file: " + f)
                    print("")
                
                # allow to replace a $ref by a type
                if e == "type" and "$ref" in b:
                    del b["$ref"]

                # allow to replace a type by a ref
                if e == "$ref" and "type" in b:
                    del b["type"]
                
                b[e] = f
                continue
                
    else:
        print "Unexpected item:", a


def replace_code(string):
    # Dirty quick fix, should be solved by a regexp.
    string = string.replace("</literalinclude> ", "\n\n")
    
    replacements = {
        "<em>": "*",
        "</em>": "*",
        "<b>": "**",
        "</b>": "**",
        "<code>":":code:`",
        "</code>":"`",
        "<codeblock>": "\n\n::\n\n  ",
        "</codeblock>": "\n\n",
        "<literalinclude>": "\n\n.. literalinclude:: ",
        "</literalinclude>": "\n\n",
        "<lang>": "\n  :language: ",
        "</lang>": "",
        "<var>":":value:`",
        "</var>":"`",
        "<permission>":":permission:`",
        "</permission>":"`",
        "<val>":":value:`",
        "</val>":"`",
        "&mdash;": u"—",
        "\n": "\n\n",
        "<li>": "\n\n* ",
    }
    for [s, r] in replacements.items():
        string = string.replace(s, r)

    string = re.sub(r'\$\(ref:(.*?)\)', ':ref:`\\1`', string)
    string = re.sub(r'\$\(doc:(.*?)\)', ':doc:`\\1`', string)
    string = re.sub(r'<a href="(.*?)">(.*?)</a>', '`\\2 <\\1>`__', string)
    string = re.sub(r"<a href='(.*?)'>(.*?)</a>", '`\\2 <\\1>`__', string)
    return string


def get_type(obj, name):
    if "type" in obj:
        if obj.get("enum") is not None:
            # enums have been moved inline and are no longer referenced
            #return "`%s <enum_%s_%d_>`__" % (obj["type"], name, unique_id)
            return "`%s`" % (obj["type"])
        elif obj["type"] == "array":
            if "items" in obj:
                if "choices" in obj["items"]:
                    choices = []
                    for choice in obj["items"]["choices"]:
                        choices.append(get_type(choice, name))
                    return "array of %s" % " or ".join(choices)
                else:
                    return "array of %s" % get_type(obj["items"], name)
            else:
                return "array"
        elif "isInstanceOf" in obj:
            return "`%s <https://developer.mozilla.org/en-US/docs/Web/API/%s>`__" % \
                (obj["isInstanceOf"], obj["isInstanceOf"])
        else:
            return obj["type"]

    elif "$ref" in obj:
        return link_ref(obj["$ref"])


def link_ref(ref):
    global additional_type_used
    if ref == "extensionTypes.File":
        return "`File <https://developer.mozilla.org/en-US/docs/Web/API/File>`__"
    if ref == "extensionTypes.Date":
        return "`Date <https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date>`__"
    if ref == "runtime.Port":
        return "`Port <https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/runtime/Port>`__"
    if ref.startswith("manifest."):
        ref = ref[9:]
    if ref == "IconPath" or ref.endswith(".IconPath"):
        ref = "IconPath"

    for additional_type in additional_type_defs:
        if additional_type['id'] == ref:
            if not ref in additional_type_used:
                additional_type_used.append(ref)
            return ":ref:`%s.%s`" % (current_namespace["namespace"], ref)
    
    for moz_namespace in ["extension.", "extensionTypes."]:
        if ref.startswith(moz_namespace):
            name = ref[len(moz_namespace):]
            url = "https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/%s/%s"
            url = url % (moz_namespace[:-1], name)
            return "`%s <%s>`__" % (name, url)
    if "manifest." in ref:
        # manifest types are not global and need to be prepended by the current namespace
        return ":ref:`%s.%s`" % (current_namespace["namespace"], ref.replace("manifest.",""))
    elif "." in ref or current_namespace["namespace"] is None:
        return ":ref:`%s`" % ref
    else:
        return ":ref:`%s.%s`" % (current_namespace["namespace"], ref)


def format_addition(obj):
    if "backported" in obj:
        return "-- [Added in TB %s, backported to TB %s]" % (obj["added"], obj["backported"])
    if "added" in obj:
        return "-- [Added in TB %s]" % obj["added"]
    return ""

def format_changes(obj, inline = False):
    lines = []
    if "changed" in obj:
        for k, v in obj['changed'].items():
            if inline:
                lines.extend([
                    ".. container:: api-member-inline-changes",
                    "",
                    "   :Changes in TB " + k + ": " + replace_code(v),
                    ""])
            else:
                lines.extend(api_header("Changes in Thunderbird " + k, api_member(name=replace_code(v))))
    return lines

def format_hints(obj):
    lines = []
    if "hints" in obj:
        lines.extend(replace_code(obj['hints']).split("\n"))
    return lines

def get_api_member_parts(name, value):
    parts = {
        "name" : "",
        "type" : "",
        "annotation" : "",
        "description" : [],
        "enum" : [],
    }

    # The return element is using a fake "_returns" name to get here.
    type_string = "%s"
    if name == "_returns":
        if value.get("optional", False):
            # type_string = "[%s]" activate not yet
            type_string = "%s"
    elif name:
        type_string = "(%s)"
        if value.get("optional", False):
            parts['name'] = "[``%s``]" % name
            type_string = "(%s, optional)"
        else:
            parts['name'] = "``%s``" % name

    if "unsupported" in value:
        type_string += " **Unsupported.**"
    elif "deprecated" in value:
        type_string += " **Deprecated.**"

    if "type" in value or "$ref" in value:
        parts['type'] = type_string % get_type(value, name)
    elif "choices" in value:
        choices = []
        for choice in value["choices"]:
            choices.append(get_type(choice, name))
        parts['type'] = type_string % " or ".join(choices)

    if "description" in value:
        parts['description'].append("")
        parts['description'].extend(replace_code(value["description"]).split("\n"))
    
    if "changed" in value:
        parts['description'].append("")
        parts['description'].extend(format_changes(value, inline=True))
    
    parts['enum'].extend(format_enum(name, value))
    
    if "added" in value or "changed" in value:
        parts['annotation'] = format_addition(value)

    return parts


def format_enum(name, value):
    if value.get("enum") is None:
        if value.get("items") is not None:
            return format_enum(name, value["items"])
        return []
    
    enum_lines = [""]
    enum_lines.append("Supported values:")

    enum_changes = value.get("enumChanges", None)
    for enum_value in value.get("enum"):       
        enum_annotation = None
        enum_description = []
        if enum_changes and enum_value in enum_changes:
            enum_change = enum_changes.get(enum_value)
            enum_annotation = format_addition(enum_change)
            if "description" in enum_change:
                enum_description.extend(replace_code(enum_change.get("description")).split("\n"))
        enum_lines.extend(api_member(name=":value:`" + enum_value + "`", annotation=enum_annotation, description=enum_description))

    return enum_lines
    
def format_object(name, obj, print_description_only = False, print_enum_only = False, enumChanges = None):
    global unique_id
    
    # If we have received an enumChanges object and we do not already have one,
    # add it to the object.
    if obj.get("enumChanges") == None and enumChanges != None:
        obj["enumChanges"] = enumChanges

    parts =  get_api_member_parts(name, obj)
  
    #enum_only:        fake header + enum
    #description_only: fake header + description + enum + nested
    #default:          standard header + description enum + nested

    fakeHeader = []
    content = []
    lines = []

    if print_enum_only or print_description_only:
        # fake api-member div structure, so style sheets continue to work
        indent = "      "
        fakeHeader.extend([
            "",
            ".. container:: api-member-node",
            "",
            "   .. container:: api-member-description-only",
        ])
    else:
        indent = "   "
        content.extend(api_member(name=parts['name'], type=parts['type'], annotation=parts['annotation']))

    nested_content = []
    if obj.get("type") == "object" and "properties" in obj:
        items = sorted(obj["properties"].items())
        for [key, value] in items:
            if value.get("ignore", False):
                continue
            if not value.get("optional", False):
                nested_content.extend(format_object(key, value))

        for [key, value] in items:
            if value.get("ignore", False):
                continue
            if value.get("optional", False):
                nested_content.extend(format_object(key, value))

    if print_enum_only:
        content.extend([indent + sub for sub in parts['enum']])
    else:
        content.extend([indent + sub for sub in parts['description']])
        content.extend([indent + sub for sub in parts['enum']])
        content.extend([indent + sub for sub in nested_content])

    if len(content) > 0:
        lines.extend(fakeHeader)
        lines.extend(content)
        lines.append("")

    return lines


def format_params(function, callback=None):
    params = []
    for param in function.get("parameters", []):
        if param["name"] == callback:
            continue
        if param.get("optional", False):
            params.append("[%s]" % param["name"])
        else:
            params.append(param["name"])
    return ", ".join(params)


def format_permissions(obj, namespace_obj = None):
    name = obj.get("namespace", obj.get("name"))
    entries = {
        "manifest" : {
            "single" : "A manifest entry named %s is required to use ``messenger.%s.*``.",
            "multiple" : "One of the manifest entries %s or %s is required to use ``messenger.%s.*``.",
            "entries" : [],
            },
        "permissions" : {
            "single" : "The permission %s is required to use ``messenger.%s.*``.",
            "multiple" : "One of the permissions %s or %s is required to use ``messenger.%s.*``.",
            "entries" : []
            },
    }
    
    # read the global permissions first (if provided)
    if namespace_obj and "permissions" in namespace_obj:
        permissions = list(dict.fromkeys(namespace_obj["permissions"]))
        permissions.sort()
        for i in range(0, len(permissions)):
            permission = permissions[i]
            if permission.startswith("manifest:"):
                continue
            else:
                entries['permissions']['entries'].append(":permission:`%s`" % permission)

    if obj and "permissions" in obj:
        permissions = list(dict.fromkeys(obj["permissions"]))
        permissions.sort()
        for i in range(0, len(permissions)):
            permission = permissions[i]
            if permission.startswith("manifest:"):
                entries['manifest']['entries'].append(":value:`%s`" % permission[9:])
            else:
                entries['permissions']['entries'].append(":permission:`%s`" % permission)

    
    lines = []
    # if a namespace_obj is provided, we only print a list of required permissions
    # including the permission required by the namespace itself
    # manifest entries are ignored
    if namespace_obj:
        content = []
        for permission in entries['permissions']['entries']:
            content.append("- %s" % permission)
        if len(content)>0:
            lines.extend(api_header("Required permissions", content))
    else:
        for entrytype in ['manifest', 'permissions']:
            entry = entries[entrytype]
            text = ""
            if len(entry['entries']) == 0:
                continue
            elif len(entry['entries']) == 1:
                text = entry['single'] % (entry['entries'][0], name)
            else:
                last = entry['entries'].pop()
                text = entry['multiple'] % (", ".join(entry['entries']), last, name)
                
            lines.extend([
                "",
                ".. rst-class:: api-permission-info",
                "",
                ".. note::",
                "",
                "   " + text,
                "",
            ])        
        
    return lines


def header_1(string):
    return [
        "=" * len(string),
        string,
        "=" * len(string),
        "",
    ]

def header_2(string, classnames=""):
    return [
        ".. rst-class:: " + classnames,
        "",
        string,
        "=" * len(string),
        "",
    ]

def header_3(string, label=None, info=""):
    # The api-section-annotation-hack directive attaches the anotation to the preeding section
    # header, closes standard section div and opens api-section-body div
    return reference(label) + [
        string,
        "-" * len(string),
        "",
        ".. api-section-annotation-hack:: " + info,
        "",
    ]

def api_member(name=None, type=None, annotation=None, description=[]):
    lines = [
        "",
        ".. api-member::",
    ]    
    if name:
        lines.append("   :name: " + name)
    if type:
        lines.append("   :type: " + type)
    if annotation:
        lines.append("   :annotation: " + annotation)
    if description and len(description)>0:
        lines.append("")
        for line in description:
            lines.append("   " + line)
    return lines

def api_header(label, content = [], annotation=None):
    lines = [
        "",
        ".. api-header::",
        "   :label: " + label,
    ]
    if annotation:
        lines.append("   :annotation: " + annotation)
    
    lines.append("")
    if len(content):
        for line in content:
            lines.append("   " + line)
        lines.append("")

    return lines
    
def reference(label):
    if label is None:
        return []

    return [
        ".. _%s:" % label,
        "",
    ]


def format_namespace(manifest, namespace):
    global unique_id, additional_type_used

    lines = []
    sidebartoc = [
        ".. container:: sticky-sidebar",
        "",
        "  " + u'\u2261' + " " + namespace["namespace"] + " API",
        "",
    ]

    #unique_id = 1
    preamble = os.path.join(OVERLAY_DIR, namespace["namespace"] + ".rst")
    if os.path.exists(preamble):
        with open(preamble) as fp_preamble:
            lines.extend(map(lambda l: l.rstrip("\n").decode("utf-8"), fp_preamble.readlines()))
            lines.append("")
    else:
        lines.extend(header_1(namespace["namespace"] + " API"))

    lines.extend([
        "",
        ".. role:: permission",
        ""]);

    lines.extend([
        "",
        ".. role:: value",
        ""]);

    lines.extend([
        "",
        ".. role:: code",
        ""]);

    if "description" in namespace:
        lines.extend(replace_code(namespace["description"]).split("\n"))
        lines.append("")

    if manifest is not None:
        lines.extend(format_manifest_namespace(manifest, namespace, sidebartoc))

    lines.extend(format_permissions(namespace))

    if "functions" in namespace:
        sidebartoc.append("  * `Functions`_")
        lines.append("")
        lines.extend(header_2("Functions", "api-main-section"))
        for function in sorted(namespace["functions"], key=lambda t: t["sort"] + t["name"] if "sort" in t else "0" + t["name"]):
            support = function.get("support")
            if support and "version_added" in support:
                if support.get("version_added") == False:
                    continue

            async = function.get("async")
            lines.extend(header_3(
                "%s(%s)" % (function["name"], format_params(function, callback=async)),
                label="%s.%s" % (namespace["namespace"], function["name"]),
                info=format_addition(function)
            ))

            if "description" in function:
                lines.extend(replace_code(function["description"]).split("\n"))
                lines.append("")
                
            if "changed" in function:
                lines.extend(format_changes(function))

            if len(function.get("parameters", [])) > 0:
                content = []
                for param in function["parameters"]:
                    if async == param["name"]:
                        # used for callback type
                        if len(param["parameters"]) > 0:
                            function["returns"] = param["parameters"][0]
                    else:
                        content.extend(format_object(param["name"], param))
                
                if len(content) > 0:
                    lines.extend(api_header("Parameters", content))

            if "returns" in function:
                content = []
                content.extend(format_object("_returns", function["returns"]))
                content.append("")
                content.append(".. _Promise: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise")
                lines.extend(api_header("Return type (`Promise`_)", content))

            lines.extend(format_permissions(function, namespace))
            
            if "hints" in function:
                lines.extend(format_hints(function))


    if "events" in namespace:
        sidebartoc.append("  * `Events`_")
        lines.append("")
        lines.extend(header_2("Events", "api-main-section"))
        for event in sorted(namespace["events"], key=lambda t: t["sort"] + t["name"] if "sort" in t else "0" + t["name"]):
            support = event.get("support")
            if support and "version_added" in support:
                if support.get("version_added") == False:
                    continue
            lines.extend(header_3(
                "%s" % (event["name"]), # , (%s)format_params(event)
                label="%s.%s" % (namespace["namespace"], event["name"]),
                info=format_addition(event)
            ))

            if "description" in event:
                lines.extend(replace_code(event["description"]).split("\n"))
                lines.append("")

            if "changed" in event:
                lines.extend(api_header("API changes", format_changes(event)))


            listener = {
                "name": "listener(%s)" % ", ".join(list(map(lambda x : x['name'], event.get("parameters", [])))),
                "description": "A function that will be called when this event occurs."
            }
            content = []
            for param in ([listener] + event.get("extraParameters", [])):
                content.extend(format_object(param["name"], param))
            extraParams = list(map(lambda x : x['name'], event.get("extraParameters", [])))
            lines.extend(api_header(
                "Parameters for %s.addListener(%s)" % (event["name"], ", ".join(["listener"] + extraParams)),
                content
            ))

            if len(event.get("parameters", [])):
                content = []
                for param in event["parameters"]:
                    content.extend(format_object(param["name"], param))
                lines.extend(api_header("Parameters passed to the listener function", content))


            
            if "returns" in event:
                lines.extend(api_header("Expected return value of the listener function", format_object("", event["returns"])))

            lines.extend(format_permissions(event, namespace))

    # loop over own type defs and additional type defs   
    for run in range(2):
        type_lines = []
        type_header = []
        type_sidebar_toc_entry = ""
        typegroup = None

        if run == 0:
            if "types" in namespace:
                typegroup = namespace['types']
                type_header.append("")
                type_header.extend(header_2("Types", "api-main-section"))
                type_sidebar_toc_entry = "  * `Types`_"
            else:
                continue
        
        if run == 1:
            typegroup = additional_type_defs
            type_header.append("")
            type_header.extend(header_2("External Types", "api-main-section"))
            type_header.append("The following types are not defined by this API, but by the underlying Mozilla WebExtension code base. They are included here, because there is no other public documentation available.")
            type_header.append("")
            type_sidebar_toc_entry = "  * `External Types`_"

        for type_ in sorted(typegroup, key=lambda t: t["id"]):
            # skip this type if it is not used
            if run == 1 and not type_['id'] in additional_type_used:
                continue
            
            type_name = type_["name"] if "name" in type_ else type_["id"]
            type_props = []

            if "description" in type_:
                type_props.extend(replace_code(type_["description"]).split("\n"))
                type_props.append("")

            if "changed" in type_:
                lines.extend(api_header("API changes", format_changes(type_)))

            if "type" in type_:
                if (type_["type"] == "object" and
                        "isInstanceOf" not in type_ and
                        ("properties" in type_ or "functions" in type_)):
                    content = []
                    if "properties" in type_:
                        items = sorted(type_["properties"].items())
                        for [key, value] in items:
                            if not value.get("optional", False):
                                content.extend(format_object(key, value))

                        for [key, value] in items:
                            if value.get("optional", False):
                                content.extend(format_object(key, value))

                    if "functions" in type_:
                        for function in sorted(type_["functions"], key=lambda t: t["sort"] + t["name"] if "sort" in t else "0" + t["name"]):
                            content.append("- ``%s(%s)``" % (function["name"], format_params(function)))
                            description = function.get("description", "")
                            if description:
                                content[-1] += " %s" % replace_code(description)

                    type_props.extend(api_header("object", content))
                else:
                    type_props.extend(api_header(get_type(type_, type_["id"]), format_object(None, type_, print_enum_only=True)))
                    
            elif "choices" in type_:
                first = True
                for choice in type_["choices"]:
                    if first:
                        first = False
                    else:
                        type_props.extend(["", "OR", ""])
                    type_props.extend(api_header(get_type(choice, type_["id"]), format_object(None, choice, print_description_only=True, enumChanges=type_.get("enumChanges"))))
                        
            if len(type_props) > 0:
                type_lines.extend(header_3(
                    type_["name"] if "name" in type_ else type_["id"],
                    label="%s.%s" % (namespace["namespace"], type_["id"]),
                    info=format_addition(type_)
                ))
                type_lines.extend(type_props)
                type_lines.append("")
            
        if len(type_lines) > 0:
            lines.extend(type_header)
            lines.extend(type_lines)
            sidebartoc.append(type_sidebar_toc_entry)

    if "properties" in namespace:
        sidebartoc.append("  * `Properties`_")
        lines.append("")
        lines.extend(header_2("Properties", "api-main-section"))

        for key in sorted(namespace["properties"].iterkeys()):
            support = namespace["properties"][key].get("support")
            if support and "version_added" in support:
                if support.get("version_added") == False:
                    continue
            lines.extend(header_3(key, label="%s.%s" % (namespace["namespace"], key)))
            lines.extend(replace_code(namespace["properties"][key].get("description")).split("\n"))
            lines.append("")

    index = 0
    previous = ""
    while index < len(lines):
        if lines[index] == "" and previous == "":
            del lines[index]
        else:
            previous = lines[index]
            index += 1

    if lines[-1] != "":
        lines.append("")

    examples = os.path.join(EXAMPLE_DIR, namespace["namespace"] + ".rst")
    if os.path.exists(examples):
        lines.append("")
        lines.extend(header_2("Examples","examples"))
        lines.append("")
        lines.append(".. include:: /examples/%s.rst" % (namespace["namespace"]))
        sidebartoc.append("  * `Examples`_")

    # Prepend sidebar toc.
    sidebartoc.extend([
        "",
        "  .. include:: /overlay/developer-resources.rst",
        ""
    ])

    return "\n".join(sidebartoc + lines).encode("utf-8")

def map_permission_to_key(permission):
    mapping = {
        "accountsRead": "accountsRead",
        "messagesMove": "messagesMove",
    }
    if permission in mapping:
        return mapping[permission]
    return permission

def format_manifest_namespace(manifest, namespace, sidebartoc):
    global unique_id
    #unique_id = 1

    if "types" not in manifest:
        return

    lines = []
    property_lines = []
    permission_lines = []

    permission_strings = {}
    for permissions_file in permissions_files:
        with codecs.open(permissions_file, encoding="utf-8") as pf:
            for line in pf:
                if line.startswith("webext-perms-description"):
                    parts = line.split("=", 2)
                    permission_name = parts[0][25:].replace("-", "." ).strip()
                    # Simple approach to get around updated locale strings, which
                    # usually have numbers added to them.
                    permission_name = re.sub(r'[0-9]', '', permission_name)
                    permission_strings[permission_name] = parts[1].strip()

    for type_ in manifest["types"]:
        if type_.get("$extend", None) == "WebExtensionManifest":
            for [name, value] in sorted(type_["properties"].items(), key=lambda t: t[0] if "sort" not in t[1] else t[1]["sort"]):
                property_lines.extend(format_object(name, value))
        if type_.get("$extend", None) in [
            "OptionalPermission",
            "OptionalPermissionNoPrompt",
            "Permission",
            "PermissionNoPrompt"
        ]:
            for choice in type_["choices"]:
                if "enum" in choice:
                    for value in choice["enum"]:
                        if "ignore_permissions" not in namespace or value not in namespace["ignore_permissions"]:
                            description = None
                            if "permissions" in manifest and value in manifest["permissions"] and "description" in manifest["permissions"][value]:
                                description = [manifest["permissions"][value]["description"]]
                            elif map_permission_to_key(value) in permission_strings:
                                description = [permission_strings[map_permission_to_key(value)]]
                            permission_lines.extend(api_member(name=":permission:`" + value + "`", description=description))

    if len(permission_lines) > 0:
        permission_lines.append("")

    if len(property_lines) > 0:
        lines = header_2("Manifest file properties", "api-main-section") + property_lines
        sidebartoc.append("  * `Manifest file properties`_")

    if len(permission_lines) > 0:
        lines.extend(header_2("Permissions", "api-main-section"))
        lines.extend(permission_lines)
        sidebartoc.append("  * `Permissions`_")

    return lines


if __name__ == "__main__":
    global src_dir, permissions_files

    parser = argparse.ArgumentParser(
        description="Create WebExtensions documentation from schema files"
    )
    parser.add_argument("schema", help="""Path to schema files""")
    parser.add_argument("path", help="""Path to mozilla-central""")
    parser.add_argument("file", nargs="*",
                        help="""The name of an API to document, which corresponds
                        to a .json file in the schemas directory""")
    args = parser.parse_args()

    src_dir = args.schema
    permissions_files = [
        os.path.join(args.path, "toolkit/locales/en-US/toolkit/global/extensionPermissions.ftl"),
        os.path.join(args.path, "comm/mail/locales/en-US/messenger/extensionPermissions.ftl")
    ]

    # read additional type defs
    additional_type_defs_file = os.path.join(OVERLAY_DIR, "additional_type_defs.json")
    with open(additional_type_defs_file) as fp_input:
        content = fp_input.read()
        content = re.sub(r"(^|\n)//.*", "", content)
        additional_type_defs = json.loads(content)

    files = []
    if len(args.file) == 0:
        # Do all files.
        for filename in glob.glob(os.path.join(src_dir, "*.json")):
            filename = os.path.basename(filename)[:-5]
            if not filename.endswith("_child"):
                files.append(filename)
    else:
        for filename in args.file:
            if os.path.exists(os.path.join(src_dir, filename + ".json")):
                files.append(filename)

    if len(files) == 0:
        print "No files found"

    for filename in sorted(files):
        # Is there a child implementation?
        child = None
        if os.path.exists(os.path.join(src_dir, filename + "_child.json")):
            with open(os.path.join(src_dir, filename + "_child.json")) as fp_child:
                print("Reading file: " + filename + "_child.json")
                childString = fp_child.read()
                childString = re.sub(r"(^|\n)//.*", "", childString)
                child = json.loads(childString)

        with open(os.path.join(src_dir, filename + ".json")) as fp_parent:
            print("Reading file: " + filename + ".json")
            parentString = fp_parent.read()
            parentString = re.sub(r"(^|\n)//.*", "", parentString)
            parent = json.loads(parentString)
            if child is not None:
                document = child
                merge_objects(parent, document)
            else:
                document = parent

        overlays = {}
        if os.path.exists(os.path.join(OVERLAY_DIR, filename + ".json")):
            with open(os.path.join(OVERLAY_DIR, filename + ".json")) as fp_overlay:
                overlay = json.load(fp_overlay)
                for namespace in overlay:
                    overlays[namespace["namespace"]] = namespace 

        # get all namespaces defined in the current file
        namespaces = {}
        for namespace in document:
            namespaces[namespace["namespace"]] = namespace 

        # Loop thru all namespaces/pages and evaluate the manifest for EACH namespace
        # in case we need to put global types (without external doc) used in the
        # manifest local in this page 
        for namespace in document:
            if namespace["namespace"] == "manifest":
                # overlay manifest
                if "manifest" in overlays:
                    merge_objects(overlays["manifest"], namespace)
                
                continue

            all_namespaces.append(namespace["namespace"])
            additional_type_used = []
            current_namespace = namespace.copy()
            manifest = namespaces.get("manifest", None)

            # overlay namespace
            if current_namespace["namespace"] in overlays:
                for entry in overlays[current_namespace["namespace"]]:
                    if entry in ["types", "events", "functions"]:
                        merge_objects(overlays[current_namespace["namespace"]][entry], current_namespace[entry])
                    elif entry in ["permissions", "ignore_permissions"]:
                        current_namespace[entry] = list(overlays[current_namespace["namespace"]][entry])

            # Import selected manifest types into the namespace (used in theme)
            # Decided to select which types should be imported, as sometimes
            # importing all types is not desired.
            if manifest:
                manifestTypes = manifest.get("types", None);
                namespaceTypes = current_namespace.get("types", None)
                if manifestTypes and namespaceTypes:
                    manifestTypeDict = {}
                    for manifestType in manifestTypes:
                            if "id" in manifestType:
                                manifestTypeDict[manifestType["id"]] = manifestType
                    
                    for index, item in enumerate(namespaceTypes):
                        if "$import_from_manifest" in item:
                            namespaceTypes[index] = manifestTypeDict[item["$import_from_manifest"]]

            with open(os.path.join(DEST_DIR, current_namespace["namespace"] + ".rst"), "w") as fp_output:
                fp_output.write(format_namespace(manifest, current_namespace))

    print ""
    print "Found namespaces"
    for x in range(len(all_namespaces)):
        print all_namespaces[x]