Merge pull request grpc#19433 from nicolasnoble/moiz-upb

Building ubp in Bazel properly.
sheenaqotj · Jul 25, 2019 · 1769169 · 1769169
2 parents 06285b3 + a4608cb
commit 1769169
Show file tree

Hide file tree

Showing 66 changed files with 3,659 additions and 1,312 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -42,8 +42,8 @@
 	path = third_party/libcxx
 	url = https://github.com/llvm-mirror/libcxx.git
 	branch = release_60
-[submodule "third_party/data-plane-api"]
-	path = third_party/data-plane-api
+[submodule "third_party/envoy-api"]
+	path = third_party/envoy-api
 	url = https://github.com/envoyproxy/data-plane-api.git
 [submodule "third_party/googleapis"]
 	path = third_party/googleapis
@@ -53,4 +53,4 @@
 	url = https://github.com/envoyproxy/protoc-gen-validate.git
 [submodule "third_party/upb"]
 	path = third_party/upb
-	url = https://github.com/google/upb.git
+	url = https://github.com/protocolbuffers/upb.git
diff --git a/BUILD b/BUILD
@@ -30,6 +30,7 @@ load(
     "//bazel:grpc_build_system.bzl",
     "grpc_cc_library",
     "grpc_generate_one_off_targets",
+    "grpc_upb_proto_library",
 )
 
 config_setting(
@@ -2275,165 +2276,44 @@ grpc_cc_library(
     ],
 )
 
-#TODO: Get this into build.yaml once we start using it.
-grpc_cc_library(
-    name = "envoy_lrs_upb",
-    srcs = [
-        "src/core/ext/upb-generated/envoy/api/v2/endpoint/load_report.upb.c",
-        "src/core/ext/upb-generated/envoy/service/load_stats/v2/lrs.upb.c",
-    ],
-    hdrs = [
-        "src/core/ext/upb-generated/envoy/api/v2/endpoint/load_report.upb.h",
-        "src/core/ext/upb-generated/envoy/service/load_stats/v2/lrs.upb.h",
-    ],
-    language = "c++",
-    external_deps = [
-        "upb_lib",
-    ],
-    deps = [
-        ":envoy_core_upb",
-        ":google_api_upb",
-        ":proto_gen_validate_upb",
-    ],
-    tags = ["no_windows"],
+grpc_upb_proto_library(
+    name = "upb_load_report",
+    deps = ["@envoy_api//envoy/api/v2/endpoint:load_report_export"]
 )
 
-grpc_cc_library(
-    name = "envoy_ads_upb",
-    srcs = [
-        "src/core/ext/upb-generated/envoy/api/v2/auth/cert.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/cds.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/cluster/circuit_breaker.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/cluster/outlier_detection.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/discovery.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/eds.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/endpoint/endpoint.upb.c",
-        "src/core/ext/upb-generated/envoy/service/discovery/v2/ads.upb.c",
-    ],
-    hdrs = [
-        "src/core/ext/upb-generated/envoy/api/v2/auth/cert.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/cds.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/cluster/circuit_breaker.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/cluster/outlier_detection.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/discovery.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/eds.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/endpoint/endpoint.upb.h",
-        "src/core/ext/upb-generated/envoy/service/discovery/v2/ads.upb.h",
-    ],
-    external_deps = [
-        "upb_lib",
-    ],
-    language = "c++",
-    deps = [
-        ":envoy_core_upb",
-        ":envoy_type_upb",
-        ":google_api_upb",
-        ":proto_gen_validate_upb",
-    ],
-    tags = ["no_windows"],
+grpc_upb_proto_library(
+    name = "upb_lrs",
+    deps = ["@envoy_api//envoy/service/load_stats/v2:lrs_export"]
 )
 
-grpc_cc_library(
-    name = "envoy_core_upb",
-    srcs = [
-        "src/core/ext/upb-generated/envoy/api/v2/core/address.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/core/base.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/core/config_source.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/core/grpc_service.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/core/health_check.upb.c",
-        "src/core/ext/upb-generated/envoy/api/v2/core/protocol.upb.c",
-    ],
-    hdrs = [
-        "src/core/ext/upb-generated/envoy/api/v2/core/address.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/core/base.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/core/config_source.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/core/grpc_service.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/core/health_check.upb.h",
-        "src/core/ext/upb-generated/envoy/api/v2/core/protocol.upb.h",
-    ],
-    external_deps = [
-        "upb_lib",
-    ],
-    language = "c++",
-    tags = ["no_windows"],
-    deps = [
-        ":envoy_type_upb",
-        ":google_api_upb",
-        ":proto_gen_validate_upb",
-    ],
+grpc_upb_proto_library(
+    name = "upb_cds",
+    deps = ["@envoy_api//envoy/api/v2:cds_export"]
 )
 
+#TODO: Get this into build.yaml once we start using it.
 grpc_cc_library(
-    name = "envoy_type_upb",
-    srcs = [
-        "src/core/ext/upb-generated/envoy/type/percent.upb.c",
-        "src/core/ext/upb-generated/envoy/type/range.upb.c",
-    ],
-    hdrs = [
-        "src/core/ext/upb-generated/envoy/type/percent.upb.h",
-        "src/core/ext/upb-generated/envoy/type/range.upb.h",
-    ],
+    name = "envoy_lrs_upb",
+    language = "c++",
     external_deps = [
         "upb_lib",
     ],
-    language = "c++",
-    tags = ["no_windows"],
     deps = [
-        ":google_api_upb",
-        ":proto_gen_validate_upb",
+        ":upb_load_report",
+        ":upb_lrs"
     ],
+    tags = ["no_windows"],
 )
 
 grpc_cc_library(
-    name = "proto_gen_validate_upb",
-    srcs = [
-        "src/core/ext/upb-generated/gogoproto/gogo.upb.c",
-        "src/core/ext/upb-generated/validate/validate.upb.c",
-    ],
-    hdrs = [
-        "src/core/ext/upb-generated/gogoproto/gogo.upb.h",
-        "src/core/ext/upb-generated/validate/validate.upb.h",
-    ],
+    name = "envoy_ads_upb",
     external_deps = [
         "upb_lib",
     ],
     language = "c++",
-    tags = ["no_windows"],
     deps = [
-        ":google_api_upb",
-    ],
-)
-
-grpc_cc_library(
-    name = "google_api_upb",
-    srcs = [
-        "src/core/ext/upb-generated/google/api/annotations.upb.c",
-        "src/core/ext/upb-generated/google/api/http.upb.c",
-        "src/core/ext/upb-generated/google/protobuf/any.upb.c",
-        "src/core/ext/upb-generated/google/protobuf/descriptor.upb.c",
-        "src/core/ext/upb-generated/google/protobuf/duration.upb.c",
-        "src/core/ext/upb-generated/google/protobuf/empty.upb.c",
-        "src/core/ext/upb-generated/google/protobuf/struct.upb.c",
-        "src/core/ext/upb-generated/google/protobuf/timestamp.upb.c",
-        "src/core/ext/upb-generated/google/protobuf/wrappers.upb.c",
-        "src/core/ext/upb-generated/google/rpc/status.upb.c",
-    ],
-    hdrs = [
-        "src/core/ext/upb-generated/google/api/annotations.upb.h",
-        "src/core/ext/upb-generated/google/api/http.upb.h",
-        "src/core/ext/upb-generated/google/protobuf/any.upb.h",
-        "src/core/ext/upb-generated/google/protobuf/descriptor.upb.h",
-        "src/core/ext/upb-generated/google/protobuf/duration.upb.h",
-        "src/core/ext/upb-generated/google/protobuf/empty.upb.h",
-        "src/core/ext/upb-generated/google/protobuf/struct.upb.h",
-        "src/core/ext/upb-generated/google/protobuf/timestamp.upb.h",
-        "src/core/ext/upb-generated/google/protobuf/wrappers.upb.h",
-        "src/core/ext/upb-generated/google/rpc/status.upb.h",
+        ":upb_cds",
     ],
-    external_deps = [
-        "upb_lib",
-    ],
-    language = "c++",
     tags = ["no_windows"],
 )
 

diff --git a/Makefile b/Makefile
@@ -370,7 +370,7 @@ CPPFLAGS += -fPIC
 LDFLAGS += -fPIC
 endif
 
-INCLUDES = . include $(GENDIR)
+INCLUDES = . include $(GENDIR) third_party/upb third_party/upb/generated_for_cmake
 LDFLAGS += -Llibs/$(CONFIG)
 
 ifeq ($(SYSTEM),Darwin)
@@ -7890,7 +7890,7 @@ endif
 
 
 LIBUPB_SRC = \
-    third_party/upb/google/protobuf/descriptor.upb.c \
+    third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.c \
     third_party/upb/upb/decode.c \
     third_party/upb/upb/def.c \
     third_party/upb/upb/encode.c \
@@ -7905,7 +7905,7 @@ PUBLIC_HEADERS_C += \
 
 LIBUPB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUPB_SRC))))
 
-$(LIBUPB_OBJS): CFLAGS += -Ithird_party/upb -Wno-sign-conversion -Wno-shadow -Wno-conversion -Wno-implicit-fallthrough -Wno-sign-compare -Wno-missing-field-initializers
+$(LIBUPB_OBJS): CFLAGS += -Wno-sign-conversion -Wno-shadow -Wno-conversion -Wno-implicit-fallthrough -Wno-sign-compare -Wno-missing-field-initializers
 
 $(LIBDIR)/$(CONFIG)/libupb.a: $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP)  $(LIBUPB_OBJS) 
 	$(E) "[AR]      Creating $@"

diff --git a/WORKSPACE b/WORKSPACE
@@ -15,7 +15,7 @@ register_execution_platforms(
 )
 
 register_toolchains(
-    "//third_party/toolchains/bazel_0.23.2_rbe_windows:cc-toolchain-x64_windows",
+    "//third_party/toolchains/bazel_0.26.0_rbe_windows:cc-toolchain-x64_windows",
 )
 
 git_repository(
@@ -65,3 +65,14 @@ rbe_autoconfig(
         },
     ),
 )
+
+
+load("@upb//bazel:workspace_deps.bzl", "upb_deps")
+upb_deps()
+
+load("@envoy_api//bazel:repositories.bzl", "api_dependencies")
+api_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")
+go_rules_dependencies()
+go_register_toolchains()
diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
@@ -24,6 +24,7 @@
 #
 
 load("//bazel:cc_grpc_library.bzl", "cc_grpc_library")
+load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library")
 
 # The set of pollers to test against if a test exercises polling
 POLLERS = ["epollex", "epoll1", "poll"]
@@ -238,3 +239,7 @@ def grpc_package(name, visibility = "private", features = []):
             default_visibility = visibility,
             features = features,
         )
+
+def grpc_upb_proto_library(name, deps):
+    upb_proto_library(name = name, deps = deps)
+
diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl
@@ -172,9 +172,9 @@ def grpc_deps():
     if "com_google_absl" not in native.existing_rules():
         http_archive(
             name = "com_google_absl",
-            sha256 = "7ddf863ddced6fa5bf7304103f9c7aa619c20a2fcf84475512c8d3834b9d14fa",
-            strip_prefix = "abseil-cpp-61c9bf3e3e1c28a4aa6d7f1be4b37fd473bb5529",
-            url = "https://github.com/abseil/abseil-cpp/archive/61c9bf3e3e1c28a4aa6d7f1be4b37fd473bb5529.tar.gz",
+            sha256 = "fd4edc10767c28b23bf9f41114c6bcd9625c165a31baa0e6939f01058029a912",
+            strip_prefix = "abseil-cpp-74d91756c11bc22f9b0108b94da9326f7f9e376f",
+            url = "https://github.com/abseil/abseil-cpp/archive/74d91756c11bc22f9b0108b94da9326f7f9e376f.tar.gz",
         )
 
     if "bazel_toolchains" not in native.existing_rules():
@@ -203,15 +203,27 @@ def grpc_deps():
             strip_prefix = "opencensus-cpp-c9a4da319bc669a772928ffc55af4a61be1a1176",
             url = "https://github.com/census-instrumentation/opencensus-cpp/archive/c9a4da319bc669a772928ffc55af4a61be1a1176.tar.gz",
         )
-
     if "upb" not in native.existing_rules():
         http_archive(
             name = "upb",
-            sha256 = "0e749a8973968397f849a3b42e28ee9c85dc418c2477954c2a6a4495f323241d",
-            strip_prefix = "upb-ed9faae0993704b033c594b072d65e1bf19207fa",
-            url = "https://github.com/google/upb/archive/ed9faae0993704b033c594b072d65e1bf19207fa.tar.gz",
+            sha256 = "1107ca1ea9aef3880b60ffb2179d06970394f1d6b62d0565e72b544b4924468d",
+            strip_prefix = "upb-1ee1e362565f681fdd46f16bf9f70f51c905ddd4",
+            url = "https://github.com/protocolbuffers/upb/archive/1ee1e362565f681fdd46f16bf9f70f51c905ddd4.tar.gz",
+        )
+    if "envoy_api" not in native.existing_rules():
+        http_archive(
+            name = "envoy_api",
+            sha256 = "a2c6e854fa9653b0ed6510e31ec7c51eac43d578b54cd75c0bc1898f7515c60d",
+            strip_prefix = "data-plane-api-a83394157ad97f4dadbc8ed81f56ad5b3a72e542",
+            url = "https://github.com/envoyproxy/data-plane-api/archive/a83394157ad97f4dadbc8ed81f56ad5b3a72e542.tar.gz",
         )
 
+    if "io_bazel_rules_go" not in native.existing_rules():
+        http_archive(
+            name = "io_bazel_rules_go",
+            urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.18.5/rules_go-0.18.5.tar.gz"],
+            sha256 = "a82a352bffae6bee4e95f68a8d80a70e87f42c4741e6a448bec11998fcc82329",
+        )
 # TODO: move some dependencies from "grpc_deps" here?
 def grpc_test_only_deps():
     """Internal, not intended for use by packages that are consuming grpc.

diff --git a/build.yaml b/build.yaml
@@ -6127,7 +6127,7 @@ defaults:
     CXXFLAGS: -Wnon-virtual-dtor
     LDFLAGS: -g
   upb:
-    CFLAGS: -Ithird_party/upb -Wno-sign-conversion -Wno-shadow -Wno-conversion -Wno-implicit-fallthrough
+    CFLAGS: -Wno-sign-conversion -Wno-shadow -Wno-conversion -Wno-implicit-fallthrough
       -Wno-sign-compare -Wno-missing-field-initializers
   zlib:
     CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration

diff --git a/grpc.gyp b/grpc.gyp
@@ -2160,7 +2160,7 @@
       'dependencies': [
       ],
       'sources': [
-        'third_party/upb/google/protobuf/descriptor.upb.c',
+        'third_party/upb/generated_for_cmake/google/protobuf/descriptor.upb.c',
         'third_party/upb/upb/decode.c',
         'third_party/upb/upb/def.c',
         'third_party/upb/upb/encode.c',

diff --git a/src/core/ext/upb-generated/envoy/api/v2/auth/cert.upb.c b/src/core/ext/upb-generated/envoy/api/v2/auth/cert.upb.c
@@ -124,20 +124,22 @@ const upb_msglayout envoy_api_v2_auth_CommonTlsContext_CombinedCertificateValida
   UPB_SIZE(8, 16), 2, false,
 };
 
-static const upb_msglayout *const envoy_api_v2_auth_UpstreamTlsContext_submsgs[1] = {
+static const upb_msglayout *const envoy_api_v2_auth_UpstreamTlsContext_submsgs[2] = {
   &envoy_api_v2_auth_CommonTlsContext_msginit,
+  &google_protobuf_UInt32Value_msginit,
 };
 
-static const upb_msglayout_field envoy_api_v2_auth_UpstreamTlsContext__fields[3] = {
+static const upb_msglayout_field envoy_api_v2_auth_UpstreamTlsContext__fields[4] = {
   {1, UPB_SIZE(12, 24), 0, 0, 11, 1},
   {2, UPB_SIZE(4, 8), 0, 0, 9, 1},
   {3, UPB_SIZE(0, 0), 0, 0, 8, 1},
+  {4, UPB_SIZE(16, 32), 0, 1, 11, 1},
 };
 
 const upb_msglayout envoy_api_v2_auth_UpstreamTlsContext_msginit = {
   &envoy_api_v2_auth_UpstreamTlsContext_submsgs[0],
   &envoy_api_v2_auth_UpstreamTlsContext__fields[0],
-  UPB_SIZE(16, 32), 3, false,
+  UPB_SIZE(24, 48), 4, false,
 };
 
 static const upb_msglayout *const envoy_api_v2_auth_DownstreamTlsContext_submsgs[5] = {