docgen

#!/usr/bin/env python
# 
# Chocolate Doom self-documentation tool.  This works similar to javadoc
# or doxygen, but documents command line parameters and configuration
# file values, generating documentation in Unix manpage, wikitext and
# plain text forms.
#
# Comments are read from the source code in the following form:
#
#   //!
#   // @arg <extra arguments>
#   // @category Category
#   // @platform <some platform that the parameter is specific to>
#   //
#   // Long description of the parameter
#   //
#
#   something_involving = M_CheckParm("-param");
#
# For configuration file values:
#
#   //! @begin_config_file myconfig
#
#   //!
#   // Description of the configuration file value.
#   //
#
#   CONFIG_VARIABLE_INT(my_variable,       c_variable),
#

import io
import sys
import os
import re
import glob
import getopt

TEXT_WRAP_WIDTH = 78
INCLUDE_STATEMENT_RE = re.compile("@include\s+(\S+)")

# Use appropriate stdout function for Python 2 or 3

def stdout(buf):
    if sys.version_info.major < 3:
        sys.stdout.write(buf)
    else:
        sys.stdout.buffer.write(buf)

# Find the maximum width of a list of parameters (for plain text output)

def parameter_list_width(params):
    w = 0

    for p in params:
        pw = len(p.name) + 5

        if p.args:
            pw += len(p.args)

        if pw > w:
            w = pw

    return w

class ConfigFile:
    def __init__(self, filename):
        self.filename = filename
        self.variables = []

    def add_variable(self, variable):
        self.variables.append(variable)

    def manpage_output(self):
        result = ".SH CONFIGURATION VARIABLES\n"

        for v in self.variables:
            result += ".TP\n"
            result += v.manpage_output()

        return result

    def plaintext_output(self):
        result = ""

        w = parameter_list_width(self.variables)

        for p in self.variables:
            result += p.plaintext_output(w)

        result = result.rstrip() + "\n"

        return result

class Category:
    def __init__(self, description):
        self.description = description
        self.params = []

    def add_param(self, param):
        self.params.append(param)

    # Plain text output

    def plaintext_output(self):
        result = "=== %s ===\n\n" % self.description

        self.params.sort()

        w = parameter_list_width(self.params)

        for p in self.params:
            if p.should_show():
                result += p.plaintext_output(w)

        result = result.rstrip() + "\n"

        return result

    def markdown_output(self):
        result = "## %s\n\n| Parameter | Description |\n| - | - |\n" % self.description

        self.params.sort()

        for p in self.params:
            if p.should_show():
                result += p.markdown_output()

        result = result.rstrip() + "\n"

        return result

    def completion_output(self):
        result = ""

        self.params.sort()

        for p in self.params:
            if p.should_show():
                result += p.completion_output(0)

        result = result.rstrip()

        return result

    def manpage_output(self):
        result = ".SH " + self.description.upper() + "\n"

        self.params.sort()

        for p in self.params:
            if p.should_show():
                result += ".TP\n"
                result += p.manpage_output()

        return result

    def wiki_output(self):
        result = "=== %s ===\n" % self.description

        self.params.sort()

        for p in self.params:
            if p.should_show():
                result += "; " + p.wiki_output() + "\n"

        # Escape special HTML characters

        result = result.replace("&", "&amp;")
        result = result.replace("<", "&lt;")
        result = result.replace(">", "&gt;")

        return result

categories = (
    (None,      Category("General options")),
    ("game",    Category("Game start options")),
    ("video",   Category("Display options")),
    ("net",     Category("Networking options")),
    ("mod",     Category("Dehacked and WAD merging")),
    ("demo",    Category("Demo options")),
    ("compat",  Category("Compatibility")),
    ("obscure", Category("Obscure and less-used options")),
)

wikipages = []
config_files = {}

# Show options that are in Vanilla Doom? Or only new options?

show_vanilla_options = True

class Parameter:
    def __lt__(self, other):
        return self.name < other.name

    def __init__(self):
        self.text = ""
        self.name = ""
        self.args = None
        self.platform = None
        self.category = None
        self.vanilla_option = False
        self.games = None

    def should_show(self):
        return not self.vanilla_option or show_vanilla_options

    def add_text(self, text):
        if len(text) <= 0:
            pass
        elif text[0] == "@":
            match = re.match('@(\S+)\s*(.*)', text)

            if not match:
                raise "Malformed option line: %s" % text

            option_type = match.group(1)
            data = match.group(2)
            
            if option_type == "arg":
                self.args = data
            elif option_type == "platform":
                self.platform = data
            elif option_type == "category":
                self.category = data
            elif option_type == "vanilla":
                self.vanilla_option = True
            elif option_type == "game":
                self.games = re.split(r'\s+', data.strip())
            else:
                raise "Unknown option type '%s'" % option_type

        else:
            self.text += text + " "

    def _games_only_text(self, pattern="(%s only)"):
        if not match_game and self.games:
            games_list = ", ".join(map(str.capitalize, self.games))
            return " " + (pattern % games_list)
        else:
            return ""

    def manpage_output(self):
        result = self.name

        if self.args:
            result += " " + self.args

        result = '\\fB' + result + '\\fR'

        result += "\n"

        if self.platform:
            result += "[%s only] " % self.platform

        escaped = re.sub('\\\\', '\\\\\\\\', self.text)

        result += escaped + self._games_only_text() + "\n"

        return result

    def wiki_output(self):
        result = self.name

        if self.args:
            result += " " + self.args

        result += ": "

        result += add_wiki_links(self.text)

        if self.platform:
            result += "'''(%s only)'''" % self.platform
        result += self._games_only_text("'''(%s only)'''")

        return result

    def markdown_output(self):
        if self.args:
            name = "%s %s" % (self.name, self.args)
        else:
            name = "%s" % self.name

        name = name.replace("|", "\\|")

        text = self.text
        if self.platform:
            text += " (%s only)" % self.platform

        text = text.replace("|", "\\|")

        result = "| %s | %s |\n" % (name, text)

        # html escape
        result = result.replace("<", "&lt;")
        result = result.replace(">", "&gt;")

        return result

    def plaintext_output(self, indent):
        # Build the first line, with the argument on
        start = "  " + self.name
        if self.args:
            start += " " + self.args

        # pad up to the plaintext width
        start += " " * (indent - len(start))

        # Build the description text
        description = self.text
        if self.platform:
            description += " (%s only)" % self.platform
        description += self._games_only_text()

        # Build the complete text for the argument
        # Split the description into words and add a word at a time
        result = ""
        words = [word for word in re.split('\s+', description) if word]
        maxlen = TEXT_WRAP_WIDTH - indent
        outlines = [[]]
        for word in words:
            linelen = sum(len(w) + 1 for w in outlines[-1])
            if linelen + len(word) > maxlen:
                outlines.append([])
            outlines[-1].append(word)

        linesep = "\n" + " " * indent

        return (start +
                linesep.join(" ".join(line) for line in outlines) +
                "\n\n")

    def completion_output(self, w):

        result = self.name + " "

        return result

# Read list of wiki pages

def read_wikipages():
    f = io.open("wikipages", encoding='UTF-8')

    try:
        for line in f:
            line = line.rstrip()

            line = re.sub('\#.*$', '', line)

            if not re.match(r'^\s*$', line):
                wikipages.append(line)
    finally:
        f.close()

# Add wiki page links

def add_wiki_links(text):
    def replace_name(m):
        linktext = m.group(1)
        if linktext == pagename:
            return '[[%s]]' % pagename
        else:
            return '[[%s|%s]]' % (pagename, linktext)

    for pagename in wikipages:
        text = re.sub(r'\b(%s)\b' % re.escape(pagename), replace_name,
                      text, count=0, flags=re.IGNORECASE)

    return text

def add_parameter(param, line, config_file):

    # If we're only targeting a particular game, check this is one of
    # the ones we're targeting.

    if match_game and param.games and match_game not in param.games:
        return

    # Is this documenting a command line parameter?

    match = re.search('(M_CheckParm(WithArgs)|M_ParmExists)?\s*\(\s*"(.*?)"',
                      line)

    if match:
        param.name = match.group(3)
        category = dict(categories)[param.category]
        category.add_param(param)
        return

    # Documenting a configuration file variable?

    match = re.search('CONFIG_VARIABLE_\S+\s*\(\s*(\S+?)\),', line)

    if match:
        param.name = match.group(1)
        config_file.add_variable(param)
        return

    raise Exception(param.text)

def process_file(filename):

    current_config_file = None

    f = io.open(filename, encoding='UTF-8')

    try:
        param = None
        waiting_for_checkparm = False

        for line in f:
            line = line.rstrip()

            # Ignore empty lines

            if re.match('\s*$', line):
                continue

            # Currently reading a doc comment?

            if param:
                # End of doc comment

                if not re.match('\s*//', line):
                    waiting_for_checkparm = True

                # The first non-empty line after the documentation comment
                # ends must contain the thing being documented.

                if waiting_for_checkparm:
                    add_parameter(param, line, current_config_file)
                    param = None
                else:
                    # More documentation text

                    munged_line = re.sub('\s*\/\/\s*', '', line, 1)
                    munged_line = re.sub('\s*$', '', munged_line)
                    param.add_text(munged_line)

            # Check for start of a doc comment

            if re.search("//!", line):
                match = re.search("@begin_config_file\s*(\S+)", line)

                if match:
                    # Beginning a configuration file
                    tagname = match.group(1)
                    current_config_file = ConfigFile(tagname)
                    config_files[tagname] = current_config_file
                else:
                    # Start of a normal comment
                    param = Parameter()
                    waiting_for_checkparm = False
    finally:
        f.close()

def process_files(path):
    # Process all C source files.

    if os.path.isdir(path):
        files = glob.glob(path + "/*.c")

        for filename in files:
            process_file(filename)
    else:
        # Special case to allow a single file to be specified as a target

        process_file(path)

def print_template(template_file, substs, content):
    f = io.open(template_file, encoding='UTF-8')

    try:
        for line in f:
            match = INCLUDE_STATEMENT_RE.search(line)
            if match:
                filename = match.group(1)
                filename = os.path.join(os.path.dirname(template_file),
                                        filename)
                print_template(filename, substs, content)
            else:
                line = line.replace("@content", content)
                for k,v in substs.items():
                    line = line.replace(k,v)
                stdout(line.rstrip().encode('UTF-8') + b'\n')
    finally:
        f.close()

def manpage_output(targets, substs, template_file):

    content = ""

    for t in targets:
        content += t.manpage_output() + "\n"

    content = content.replace("-", "\\-")

    print_template(template_file, substs, content)

def wiki_output(targets, _, template):
    read_wikipages()

    for t in targets:
        stdout(t.wiki_output().encode('UTF-8') + b'\n')

def markdown_output(targets, substs, template_file):
    content = ""

    for t in targets:
        content += t.markdown_output() + "\n"

    print_template(template_file, substs, content)

def plaintext_output(targets, substs, template_file):

    content = ""

    for t in targets:
        content += t.plaintext_output() + "\n"

    print_template(template_file, substs, content)

def completion_output(targets, substs, template_file):

    content = ""

    for t in targets:
        content += t.completion_output() + "\n"

    print_template(template_file, substs, content)

def usage():
    print("Usage: %s [-V] [-c tag] [-g game] -n program_name -s package_name [ -z shortname ] ( -M | -m | -w | -p ) <dir>..." \
            % sys.argv[0])
    print("   -c :  Provide documentation for the specified configuration file")
    print("         (matches the given tag name in the source file)")
    print("   -s :  Package name, e.g. Chocolate Doom (for substitution)")
    print("   -z :  Package short-name, e.g. Chocolate (for substitution)")
    print("   -n :  Program name, e.g. chocolate (for substitution)")
    print("   -M :  Markdown output")
    print("   -m :  Manpage output")
    print("   -w :  Wikitext output")
    print("   -p :  Plaintext output")
    print("   -b :  Bash-Completion output")
    print("   -V :  Don't show Vanilla Doom options")
    print("   -g :  Only document options for specified game.")
    sys.exit(0)

# Parse command line

opts, args = getopt.getopt(sys.argv[1:], "n:s:z:M:m:wp:b:c:g:V")

output_function = None
template = None
doc_config_file = None
match_game = None
substs = {}

for opt in opts:
    if opt[0] == "-n":
        substs["@PROGRAM_SPREFIX@"] = opt[1]
    if opt[0] == "-s":
        substs["@PACKAGE_NAME@"] = opt[1]
    if opt[0] == "-z":
        substs["@PACKAGE_SHORTNAME@"] = opt[1]
    if opt[0] == "-m":
        output_function = manpage_output
        template = opt[1]
    elif opt[0] == "-M":
        output_function = markdown_output
        template = opt[1]
    elif opt[0] == "-w":
        output_function = wiki_output
    elif opt[0] == "-p":
        output_function = plaintext_output
        template = opt[1]
    elif opt[0] == "-b":
        output_function = completion_output
        template = opt[1]
    elif opt[0] == "-V":
        show_vanilla_options = False
    elif opt[0] == "-c":
        doc_config_file = opt[1]
    elif opt[0] == "-g":
        match_game = opt[1]
        substs["@GAME@"] = opt[1]
        substs["@GAME_UPPER@"] = opt[1].title()
        if "doom" == opt[1]:
            substs["@CFGFILE@"] = "default.cfg"
        else:
            substs["@CFGFILE@"] = opt[1] + ".cfg"

if output_function == None or len(args) < 1:
    usage()
else:
    # Process specified files

    for path in args:
        process_files(path)

    # Build a list of things to document
    if doc_config_file:
        documentation_targets = [config_files[doc_config_file]]
    else:
        documentation_targets = [c for _, c in categories]

    # Generate the output
    output_function(documentation_targets, substs, template)