protodoc/api_proto_plugin: generic API protoc plugin framework.

Split out the generic plugin and FileDescriptorProto traversal bits from
protodoc. This is in aid of the work in envoyproxy#8082 ad envoyproxy#8083, where additional
protoc plugins will be responsible for v2 -> v3alpha API migrations and
translation code generation.

This is only the start really of the api_proto_plugin framework. I
anticipate additional bits of protodoc will move here later, including
field type analysis and oneof handling.

In some respects, this is a re-implementation of some of
https://github.com/lyft/protoc-gen-star in Python. The advantage is that
this is super lightweight, has few dependencies and can be easily
hacked. We also embed various bits of API business logic, e.g.
annotations, in the framework (for now).

Risk level: Low
Testing: diff -ru against previous protodoc.py RST output, identical modulo some
  trivial whitespace that doesn't appear in generated HTML. There are no
  real tests yet, I anticipate adding some golden proto style tests.

Signed-off-by: Harvey Tuch <htuch@google.com>

tools/api_proto_plugin/BUILD

@@ -0,0 +1,17 @@
+licenses(["notice"])  # Apache 2
+
+py_library(
+    name = "api_proto_plugin",
+    srcs = [
+        "annotations.py",
+        "plugin.py",
+        "traverse.py",
+        "type_context.py",
+        "visitor.py",
+    ],
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_google_protobuf//:protobuf_python",
+    ],
+)

tools/api_proto_plugin/annotations.py

@@ -0,0 +1,81 @@
+"""Envoy API annotations."""
+
+from collections import namedtuple
+
+import re
+
+# Key-value annotation regex.
+ANNOTATION_REGEX = re.compile('\[#([\w-]+?):(.*?)\]\s?', re.DOTALL)
+
+# Page/section titles with special prefixes in the proto comments
+DOC_TITLE_ANNOTATION = 'protodoc-title'
+
+# Not implemented yet annotation on leading comments, leading to insertion of
+# warning on field.
+NOT_IMPLEMENTED_WARN_ANNOTATION = 'not-implemented-warn'
+
+# Not implemented yet annotation on leading comments, leading to hiding of
+# field.
+NOT_IMPLEMENTED_HIDE_ANNOTATION = 'not-implemented-hide'
+
+# Comment that allows for easy searching for things that need cleaning up in the next major
+# API version.
+NEXT_MAJOR_VERSION_ANNOTATION = 'next-major-version'
+
+# Comment. Just used for adding text that will not go into the docs at all.
+COMMENT_ANNOTATION = 'comment'
+
+# proto compatibility status.
+PROTO_STATUS_ANNOTATION = 'proto-status'
+
+# Where v2 differs from v1..
+V2_API_DIFF_ANNOTATION = 'v2-api-diff'
+
+VALID_ANNOTATIONS = set([
+    DOC_TITLE_ANNOTATION,
+    NOT_IMPLEMENTED_WARN_ANNOTATION,
+    NOT_IMPLEMENTED_HIDE_ANNOTATION,
+    V2_API_DIFF_ANNOTATION,
+    NEXT_MAJOR_VERSION_ANNOTATION,
+    COMMENT_ANNOTATION,
+    PROTO_STATUS_ANNOTATION,
+])
+
+# These can propagate from file scope to message/enum scope (and be overridden).
+INHERITED_ANNOTATIONS = set([
+    PROTO_STATUS_ANNOTATION,
+])
+
+
+class AnnotationError(Exception):
+  """Base error class for the annotations module."""
+
+
+def ExtractAnnotations(s, inherited_annotations=None):
+  """Extract annotations map from a given comment string.
+
+  Args:
+    s: string that may contains annotations.
+    inherited_annotations: annotation map from file-level inherited annotations
+      (or None) if this is a file-level comment.
+
+  Returns:
+    Annotation map.
+  """
+  annotations = {
+      k: v
+      for k, v in (inherited_annotations or {}).items()
+      if k in INHERITED_ANNOTATIONS
+  }
+  # Extract annotations.
+  groups = re.findall(ANNOTATION_REGEX, s)
+  for group in groups:
+    annotation = group[0]
+    if annotation not in VALID_ANNOTATIONS:
+      raise AnnotationError('Unknown annotation: %s' % annotation)
+    annotations[group[0]] = group[1].lstrip()
+  return annotations
+
+
+def WithoutAnnotations(s):
+  return re.sub(ANNOTATION_REGEX, '', s)

tools/api_proto_plugin/plugin.py

@@ -0,0 +1,60 @@
+"""Python protoc plugin for Envoy APIs."""
+
+import cProfile
+import io
+import os
+import pstats
+import sys
+
+from tools.api_proto_plugin import traverse
+
+from google.protobuf.compiler import plugin_pb2
+
+
+def Plugin(output_suffix, visitor):
+  """Protoc plugin entry point.
+
+  This defines protoc plugin and manages the stdin -> stdout flow. An
+  api_proto_plugin is defined by the provided visitor.
+
+  See
+  http://www.expobrain.net/2015/09/13/create-a-plugin-for-google-protocol-buffer/
+  for further details on protoc plugin basics.
+
+  Args:
+    output_suffix: output files are generated alongside their corresponding
+      input .proto, with this filename suffix.
+    visitor: visitor.Visitor defining the business logic of the plugin.
+  """
+  request = plugin_pb2.CodeGeneratorRequest()
+  request.ParseFromString(sys.stdin.buffer.read())
+  response = plugin_pb2.CodeGeneratorResponse()
+  cprofile_enabled = os.getenv('CPROFILE_ENABLED')
+
+  # We use file_to_generate rather than file_proto here since we are invoked
+  # inside a Bazel aspect, each node in the DAG will be visited once by the
+  # aspect and we only want to generate docs for the current node.
+  for file_to_generate in request.file_to_generate:
+    # Find the FileDescriptorProto for the file we actually are generating.
+    file_proto = [
+        pf for pf in request.proto_file if pf.name == file_to_generate
+    ][0]
+    f = response.file.add()
+    f.name = file_proto.name + output_suffix
+    if cprofile_enabled:
+      pr = cProfile.Profile()
+      pr.enable()
+    # We don't actually generate any RST right now, we just string dump the
+    # input proto file descriptor into the output file.
+    f.content = traverse.TraverseFile(file_proto, visitor)
+    if cprofile_enabled:
+      pr.disable()
+      stats_stream = io.StringIO()
+      ps = pstats.Stats(
+          pr, stream=stats_stream).sort_stats(
+              os.getenv('CPROFILE_SORTBY', 'cumulative'))
+      stats_file = response.file.add()
+      stats_file.name = file_proto.name + output_suffix + '.profile'
+      ps.print_stats()
+      stats_file.content = stats_stream.getvalue()
+  sys.stdout.buffer.write(response.SerializeToString())

tools/api_proto_plugin/traverse.py

@@ -0,0 +1,80 @@
+"""FileDescriptorProto traversal for api_proto_plugin framework."""
+
+from tools.api_proto_plugin import type_context
+
+
+def TraverseEnum(type_context, enum_proto, visitor):
+  """Traverse an enum definition.
+
+  Args:
+    type_context: type_context.TypeContext for enum type.
+    enum_proto: EnumDescriptorProto for enum.
+    visitor: visitor.Visitor defining the business logic of the plugin.
+
+  Returns:
+    Plugin specific output.
+  """
+  return visitor.VisitEnum(enum_proto, type_context)
+
+
+def TraverseMessage(type_context, msg_proto, visitor):
+  """Traverse a message definition.
+
+  Args:
+    type_context: type_context.TypeContext for message type.
+    msg_proto: DescriptorProto for message.
+    visitor: visitor.Visitor defining the business logic of the plugin.
+
+  Returns:
+    Plugin specific output.
+  """
+  # Skip messages synthesized to represent map types.
+  if msg_proto.options.map_entry:
+    return ''
+  # We need to do some extra work to recover the map type annotation from the
+  # synthesized messages.
+  type_context.map_typenames = {
+      '%s.%s' % (type_context.name, nested_msg.name):
+      (nested_msg.field[0], nested_msg.field[1])
+      for nested_msg in msg_proto.nested_type
+      if nested_msg.options.map_entry
+  }
+  nested_msgs = [
+      TraverseMessage(
+          type_context.ExtendNestedMessage(index, nested_msg.name), nested_msg,
+          visitor) for index, nested_msg in enumerate(msg_proto.nested_type)
+  ]
+  nested_enums = [
+      TraverseEnum(
+          type_context.ExtendNestedEnum(index, nested_enum.name), nested_enum,
+          visitor) for index, nested_enum in enumerate(msg_proto.enum_type)
+  ]
+  return visitor.VisitMessage(msg_proto, type_context, nested_msgs,
+                              nested_enums)
+
+
+def TraverseFile(file_proto, visitor):
+  """Traverse a proto file definition.
+
+  Args:
+    file_proto: FileDescriptorProto for file.
+    visitor: visitor.Visitor defining the business logic of the plugin.
+
+  Returns:
+    Plugin specific output.
+  """
+  source_code_info = type_context.SourceCodeInfo(file_proto.name,
+                                                 file_proto.source_code_info)
+  package_type_context = type_context.TypeContext(source_code_info,
+                                                  file_proto.package)
+  msgs = [
+      TraverseMessage(
+          package_type_context.ExtendMessage(index, msg.name), msg, visitor)
+      for index, msg in enumerate(file_proto.message_type)
+  ]
+  enums = [
+      TraverseEnum(
+          package_type_context.ExtendEnum(index, enum.name), enum, visitor)
+      for index, enum in enumerate(file_proto.enum_type)
+  ]
+  return visitor.VisitFile(file_proto, package_type_context, msgs, enums)