build: undo some protobuf hacks, put some new ones in.

* Use protobuf from HEAD to ensure we have the
  protocolbuffers/protobuf#3327 fix for CI and local
  builds. Users can opt to use a specific release tag with the
  ENVOY_PROTOBUF_COMMIT env var.

* Remove the workaround for protocolbuffers/protobuf#3327
  in do_ci.sh.

* Remove the multiple protobuf versions that existed because
  protocolbuffers/protobuf#2508 wasn't merged.

* Add some evil symlink stuff to get @protobuf_bzl inferred from
  wherever WORKSPACE points the envoy_dependencies path at.

As a bonus, enabled more verbose build of external deps so we don't sit
around for minutes on initial checkout with no activity indicator. This
can be done safely now as everyone should be at Bazel 0.5.2.

bazel/repositories.bzl

@@ -19,21 +19,20 @@ def _repository_impl(ctxt):
     result = ctxt.execute(
         ["./repositories.sh"] + ctxt.attr.recipes,
         environment = environment,
-        # Ideally we would print progress, but instead this hangs on "INFO: Loading
-        # complete.  Analyzing.." today, see
-        # https://github.com/bazelbuild/bazel/issues/1289. We could set quiet=False
-        # as well to indicate progress, but that isn't supported in versions folks
-        # are using right now (0.4.5).
-        # TODO(htuch): Revisit this when everyone is on newer Bazel versions.
-        #
-        # quiet = False,
+        quiet = False,
     )
     print("External dep build exited with return code: %d" % result.return_code)
     print(result.stdout)
     print(result.stderr)
     if result.return_code != 0:
         fail("External dep build failed")
 
+def _protobuf_repository_impl(ctxt):
+    ctxt.symlink(Label(ctxt.attr.envoy_deps_path + "thirdparty/protobuf:protobuf.bzl"), "protobuf.bzl")
+    ctxt.symlink(Label(ctxt.attr.envoy_deps_path + "thirdparty/protobuf:BUILD"), "BUILD")
+    ctxt.symlink(ctxt.path(Label(ctxt.attr.envoy_deps_path + "thirdparty/protobuf:BUILD")).dirname.get_child("src"),
+                 "src")
+
 def py_jinja2_dep():
     BUILD = """
 py_library(
@@ -126,17 +125,6 @@ def envoy_api_deps(skip_targets):
     )
 
 def envoy_dependencies(path = "@envoy_deps//", skip_protobuf_bzl = False, skip_targets = []):
-    if not skip_protobuf_bzl:
-        native.git_repository(
-            name = "protobuf_bzl",
-            # Using a non-canonical repository/branch here. This is a workaround to the lack of
-            # merge on https://github.com/google/protobuf/pull/2508, which is needed for supporting
-            # arbitrary CC compiler locations from the environment. The branch is
-            # https://github.com/htuch/protobuf/tree/v3.2.0-default-shell-env, which is the 3.2.0
-            # release with the above mentioned PR cherry picked.
-            commit = "d490587268931da78c942a6372ef57bb53db80da",
-            remote = REPO_LOCATIONS["protobuf"],
-        )
     native.bind(
         name = "cc_wkt_protos",
         actual = "@protobuf_bzl//:cc_wkt_protos",
@@ -173,6 +161,22 @@ def envoy_dependencies(path = "@envoy_deps//", skip_protobuf_bzl = False, skip_t
         recipes = recipes.to_list(),
     )
 
+    protobuf_repository = repository_rule(
+        implementation = _protobuf_repository_impl,
+        attrs = {
+            "envoy_deps_path": attr.string(),
+        },
+    )
+
+    # If the WORKSPACE hasn't already supplied @protobuf_bzl and told us to skip it, we need to map in the
+    # full repo into @protobuf_bzl so that we can depend on this in envoy_build_system.bzl and for things
+    # like @protobuf_bzl//:cc_wkt_protos in envoy-api. We do this by some evil symlink stuff.
+    if not skip_protobuf_bzl:
+        protobuf_repository(
+            name = "protobuf_bzl",
+            envoy_deps_path = path,
+        )
+
     for t in TARGET_RECIPES:
         if t not in skip_targets:
             native.bind(

ci/WORKSPACE

@@ -5,12 +5,6 @@ load("//bazel:cc_configure.bzl", "cc_configure")
 
 envoy_dependencies(
     path = "//ci/prebuilt",
-    skip_protobuf_bzl = True,
-)
-
-local_repository(
-    name = "protobuf_bzl",
-    path = "/thirdparty/protobuf",
 )
 
 cc_configure()

ci/WORKSPACE.filter.example

@@ -10,12 +10,6 @@ load("//bazel:cc_configure.bzl", "cc_configure")
 
 envoy_dependencies(
     path = "@envoy//ci/prebuilt",
-    skip_protobuf_bzl = True,
-)
-
-local_repository(
-    name = "protobuf_bzl",
-    path = "/thirdparty/protobuf",
 )
 
 cc_configure()

ci/build_container/build_recipes/protobuf.sh

@@ -2,12 +2,16 @@
 
 set -e
 
-VERSION=3.3.0
+# Unless overriden with an explicit release tag, e.g. v3.2.0rc2, we use a pinned
+# HEAD commit. This is only until we get a release with
+# https://github.com/google/protobuf/pull/3327, i.e. v3.4.0.
+[ -z "$ENVOY_PROTOBUF_COMMIT" ] && ENVOY_PROTOBUF_COMMIT=062df3d0724d9ae5e3c65d481dc1d3aca811152e  # 2017-07-20
 
-wget -O protobuf-$VERSION.tar.gz https://github.com/google/protobuf/releases/download/v$VERSION/protobuf-cpp-$VERSION.tar.gz
-tar xf protobuf-$VERSION.tar.gz
-rsync -av protobuf-$VERSION/* $THIRDPARTY_SRC/protobuf
-cd protobuf-$VERSION
+git clone https://github.com/google/protobuf.git
+rsync -av protobuf/* $THIRDPARTY_SRC/protobuf
+cd protobuf
+git reset --hard "$ENVOY_PROTOBUF_COMMIT"
+./autogen.sh
 ./configure --prefix=$THIRDPARTY_BUILD --enable-shared=no
 make V=1 install
 

ci/do_ci.sh

@@ -9,7 +9,6 @@ echo "building using ${NUM_CPUS} CPUs"
 
 function bazel_release_binary_build() {
   echo "Building..."
-  BUILD_TYPE=opt protobuf_3322_workaround
   cd "${ENVOY_CI_DIR}"
   bazel --batch build ${BAZEL_BUILD_OPTIONS} -c opt //source/exe:envoy-static.stamped
   # Copy the envoy-static binary somewhere that we can access outside of the
@@ -21,7 +20,6 @@ function bazel_release_binary_build() {
 
 function bazel_debug_binary_build() {
   echo "Building..."
-  BUILD_TYPE=dbg protobuf_3322_workaround
   cd "${ENVOY_CI_DIR}"
   bazel --batch build ${BAZEL_BUILD_OPTIONS} -c dbg //source/exe:envoy-static.stamped
   # Copy the envoy-static binary somewhere that we can access outside of the
@@ -31,11 +29,6 @@ function bazel_debug_binary_build() {
     "${ENVOY_DELIVERY_DIR}"/envoy-debug
 }
 
-# See https://github.com/google/protobuf/issues/3322
-function protobuf_3322_workaround() {
-  ln -sf /thirdparty_build_"${BUILD_TYPE}" "${ENVOY_SRCDIR}"/ci/prebuilt/thirdparty_build
-}
-
 if [[ "$1" == "bazel.release" ]]; then
   setup_gcc_toolchain
   echo "bazel release build with tests..."
@@ -62,7 +55,6 @@ elif [[ "$1" == "bazel.debug.server_only" ]]; then
   exit 0
 elif [[ "$1" == "bazel.asan" ]]; then
   setup_clang_toolchain
-  BUILD_TYPE=dbg protobuf_3322_workaround
   echo "bazel ASAN/UBSAN debug build with tests..."
   cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}"
   echo "Building and testing..."
@@ -71,7 +63,6 @@ elif [[ "$1" == "bazel.asan" ]]; then
   exit 0
 elif [[ "$1" == "bazel.tsan" ]]; then
   setup_clang_toolchain
-  BUILD_TYPE=dbg protobuf_3322_workaround
   echo "bazel TSAN debug build with tests..."
   cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}"
   echo "Building and testing..."
@@ -80,7 +71,6 @@ elif [[ "$1" == "bazel.tsan" ]]; then
   exit 0
 elif [[ "$1" == "bazel.dev" ]]; then
   setup_clang_toolchain
-  BUILD_TYPE=dbg protobuf_3322_workaround
   # This doesn't go into CI but is available for developer convenience.
   echo "bazel fastbuild build with tests..."
   cd "${ENVOY_CI_DIR}"
@@ -96,10 +86,6 @@ elif [[ "$1" == "bazel.dev" ]]; then
   exit 0
 elif [[ "$1" == "bazel.coverage" ]]; then
   setup_gcc_toolchain
-  # Technically test/run_envoy_bazel_coverage.sh is doing a -c dbg build, but it
-  # also sets -DNDEBUG, so for protobuf #3322 purposes, we need to setup for
-  # opt.
-  BUILD_TYPE=opt protobuf_3322_workaround
   echo "bazel coverage build with tests..."
   export GCOVR="/thirdparty/gcovr/scripts/gcovr"
   export GCOVR_DIR="${ENVOY_BUILD_DIR}/bazel-envoy"