Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

protobuf: refactor proto visitor pattern. #9807

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions source/common/config/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,7 @@ envoy_cc_library(
deps = [
":api_type_oracle_lib",
"//source/common/protobuf",
"//source/common/protobuf:visitor_lib",
"//source/common/protobuf:well_known_lib",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
],
Expand Down
53 changes: 7 additions & 46 deletions source/common/config/version_converter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "common/common/assert.h"
#include "common/config/api_type_oracle.h"
#include "common/protobuf/visitor.h"
#include "common/protobuf/well_known.h"

#include "absl/strings/match.h"
Expand All @@ -13,46 +14,6 @@ namespace {

const char DeprecatedFieldShadowPrefix[] = "hidden_envoy_deprecated_";

class ProtoVisitor {
public:
virtual ~ProtoVisitor() = default;

// Invoked when a field is visited, with the message, field descriptor and
// context. Returns a new context for use when traversing the sub-message in a
// field.
virtual const void* onField(Protobuf::Message&, const Protobuf::FieldDescriptor&,
const void* ctxt) {
return ctxt;
}

// Invoked when a message is visited, with the message and a context.
virtual void onMessage(Protobuf::Message&, const void*){};
};

// TODO(htuch): refactor these message visitor patterns into utility.cc and share with
// MessageUtil::checkForUnexpectedFields.
void traverseMutableMessage(ProtoVisitor& visitor, Protobuf::Message& message, const void* ctxt) {
visitor.onMessage(message, ctxt);
const Protobuf::Descriptor* descriptor = message.GetDescriptor();
const Protobuf::Reflection* reflection = message.GetReflection();
for (int i = 0; i < descriptor->field_count(); ++i) {
const Protobuf::FieldDescriptor* field = descriptor->field(i);
const void* field_ctxt = visitor.onField(message, *field, ctxt);
// If this is a message, recurse to scrub deprecated fields in the sub-message.
if (field->cpp_type() == Protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (field->is_repeated()) {
const int size = reflection->FieldSize(message, field);
for (int j = 0; j < size; ++j) {
traverseMutableMessage(visitor, *reflection->MutableRepeatedMessage(&message, field, j),
field_ctxt);
}
} else if (reflection->HasField(message, field)) {
traverseMutableMessage(visitor, *reflection->MutableMessage(&message, field), field_ctxt);
}
}
}
}

// Reinterpret a Protobuf message as another Protobuf message by converting to
// wire format and back. This only works for messages that can be effectively
// duck typed this way, e.g. with a subtype relationship modulo field name.
Expand Down Expand Up @@ -86,7 +47,7 @@ DynamicMessagePtr createForDescriptorWithCast(const Protobuf::Message& message,
// internally, we later want to recover their original types.
void annotateWithOriginalType(const Protobuf::Descriptor& prev_descriptor,
Protobuf::Message& next_message) {
class TypeAnnotatingProtoVisitor : public ProtoVisitor {
class TypeAnnotatingProtoVisitor : public ProtobufMessage::ProtoVisitor {
public:
void onMessage(Protobuf::Message& message, const void* ctxt) override {
const Protobuf::Descriptor* descriptor = message.GetDescriptor();
Expand Down Expand Up @@ -125,7 +86,7 @@ void annotateWithOriginalType(const Protobuf::Descriptor& prev_descriptor,
}
};
TypeAnnotatingProtoVisitor proto_visitor;
traverseMutableMessage(proto_visitor, next_message, &prev_descriptor);
ProtobufMessage::traverseMutableMessage(proto_visitor, next_message, &prev_descriptor);
}

} // namespace
Expand All @@ -138,7 +99,7 @@ void VersionConverter::upgrade(const Protobuf::Message& prev_message,
}

void VersionConverter::eraseOriginalTypeInformation(Protobuf::Message& message) {
class TypeErasingProtoVisitor : public ProtoVisitor {
class TypeErasingProtoVisitor : public ProtobufMessage::ProtoVisitor {
public:
void onMessage(Protobuf::Message& message, const void*) override {
const Protobuf::Reflection* reflection = message.GetReflection();
Expand All @@ -147,7 +108,7 @@ void VersionConverter::eraseOriginalTypeInformation(Protobuf::Message& message)
}
};
TypeErasingProtoVisitor proto_visitor;
traverseMutableMessage(proto_visitor, message, nullptr);
ProtobufMessage::traverseMutableMessage(proto_visitor, message, nullptr);
}

DynamicMessagePtr VersionConverter::recoverOriginal(const Protobuf::Message& upgraded_message) {
Expand Down Expand Up @@ -224,7 +185,7 @@ void VersionConverter::prepareMessageForGrpcWire(Protobuf::Message& message,
}

void VersionUtil::scrubHiddenEnvoyDeprecated(Protobuf::Message& message) {
class HiddenFieldScrubbingProtoVisitor : public ProtoVisitor {
class HiddenFieldScrubbingProtoVisitor : public ProtobufMessage::ProtoVisitor {
public:
const void* onField(Protobuf::Message& message, const Protobuf::FieldDescriptor& field,
const void*) override {
Expand All @@ -236,7 +197,7 @@ void VersionUtil::scrubHiddenEnvoyDeprecated(Protobuf::Message& message) {
}
};
HiddenFieldScrubbingProtoVisitor proto_visitor;
traverseMutableMessage(proto_visitor, message, nullptr);
ProtobufMessage::traverseMutableMessage(proto_visitor, message, nullptr);
}

} // namespace Config
Expand Down
8 changes: 8 additions & 0 deletions source/common/protobuf/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,20 @@ envoy_cc_library(
"//source/common/common:utility_lib",
"//source/common/config:api_type_oracle_lib",
"//source/common/config:version_converter_lib",
"//source/common/protobuf:visitor_lib",
"@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto",
"@envoy_api//envoy/annotations:pkg_cc_proto",
"@envoy_api//envoy/type/v3:pkg_cc_proto",
],
)

envoy_cc_library(
name = "visitor_lib",
srcs = ["visitor.cc"],
hdrs = ["visitor.h"],
deps = [":protobuf"],
)

envoy_cc_library(
name = "well_known_lib",
hdrs = ["well_known.h"],
Expand Down
94 changes: 46 additions & 48 deletions source/common/protobuf/utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "common/config/version_converter.h"
#include "common/protobuf/message_validator_impl.h"
#include "common/protobuf/protobuf.h"
#include "common/protobuf/visitor.h"
#include "common/protobuf/well_known.h"

#include "absl/strings/match.h"
Expand Down Expand Up @@ -375,42 +376,25 @@ void checkForDeprecatedNonRepeatedEnumValue(const Protobuf::Message& message,
}
}

void checkForUnexpectedFields(const Protobuf::Message& message,
ProtobufMessage::ValidationVisitor& validation_visitor,
Runtime::Loader* runtime) {
// Reject unknown fields.
const auto& unknown_fields = message.GetReflection()->GetUnknownFields(message);
if (!unknown_fields.empty()) {
std::string error_msg;
for (int n = 0; n < unknown_fields.field_count(); ++n) {
if (unknown_fields.field(n).number() == ProtobufWellKnown::OriginalTypeFieldNumber) {
continue;
}
error_msg += absl::StrCat(n > 0 ? ", " : "", unknown_fields.field(n).number());
}
// We use the validation visitor but have hard coded behavior below for deprecated fields.
// TODO(htuch): Unify the deprecated and unknown visitor handling behind the validation
// visitor pattern. https://github.com/envoyproxy/envoy/issues/8092.
if (!error_msg.empty()) {
validation_visitor.onUnknownField("type " + message.GetTypeName() +
" with unknown field set {" + error_msg + "}");
}
}
class UnexpectedFieldProtoVisitor : public ProtobufMessage::ConstProtoVisitor {
public:
UnexpectedFieldProtoVisitor(ProtobufMessage::ValidationVisitor& validation_visitor,
Runtime::Loader* runtime)
: validation_visitor_(validation_visitor), runtime_(runtime) {}

const Protobuf::Descriptor* descriptor = message.GetDescriptor();
const Protobuf::Reflection* reflection = message.GetReflection();
for (int i = 0; i < descriptor->field_count(); ++i) {
const Protobuf::FieldDescriptor* field = descriptor->field(i);
absl::string_view filename = filenameFromPath(field->file()->name());
const void* onField(const Protobuf::Message& message, const Protobuf::FieldDescriptor& field,
const void*) override {
const Protobuf::Reflection* reflection = message.GetReflection();
absl::string_view filename = filenameFromPath(field.file()->name());

// Before we check to see if the field is in use, see if there's a
// deprecated default enum value.
checkForDeprecatedNonRepeatedEnumValue(message, filename, field, reflection, runtime);
checkForDeprecatedNonRepeatedEnumValue(message, filename, &field, reflection, runtime_);

// If this field is not in use, continue.
if ((field->is_repeated() && reflection->FieldSize(message, field) == 0) ||
(!field->is_repeated() && !reflection->HasField(message, field))) {
continue;
if ((field.is_repeated() && reflection->FieldSize(message, &field) == 0) ||
(!field.is_repeated() && !reflection->HasField(message, &field))) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

optional, but it seems like this might be useful enough to make part of the onField call. I'd think folks would either want to be traversing structure (not care) or looking at valid fields (care) and if they care it's easy enough to get this wrong.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Surprisingly tricky, since what is counted as used/unused is subtle for scalar fields. This can be seen in this code, since it first performs the scalar enum check before conditioning on whether the field is set in some sense.

return nullptr;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we ever return something which is used, or plan to?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, in TypeAnnotatingProtoVisitor::onField().

}

#ifdef ENVOY_DISABLE_DEPRECATED_FEATURES
Expand All @@ -421,22 +405,22 @@ void checkForUnexpectedFields(const Protobuf::Message& message,
// Allow runtime to be null both to not crash if this is called before server initialization,
// and so proto validation works in context where runtime singleton is not set up (e.g.
// standalone config validation utilities)
if (runtime && field->options().deprecated()) {
if (runtime_ && field.options().deprecated()) {
// This is set here, rather than above, so that in the absence of a
// registry (i.e. test) the default for if a feature is allowed or not is
// based on ENVOY_DISABLE_DEPRECATED_FEATURES.
warn_only &= !field->options().GetExtension(envoy::annotations::disallowed_by_default);
warn_only = runtime->snapshot().deprecatedFeatureEnabled(
absl::StrCat("envoy.deprecated_features:", field->full_name()), warn_only);
warn_only &= !field.options().GetExtension(envoy::annotations::disallowed_by_default);
warn_only = runtime_->snapshot().deprecatedFeatureEnabled(
absl::StrCat("envoy.deprecated_features:", field.full_name()), warn_only);
}

// If this field is deprecated, warn or throw an error.
if (field->options().deprecated()) {
if (field.options().deprecated()) {
std::string err = fmt::format(
"Using deprecated option '{}' from file {}. This configuration will be removed from "
"Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/intro/deprecated "
"for details.",
field->full_name(), filename);
field.full_name(), filename);
if (warn_only) {
ENVOY_LOG_MISC(warn, "{}", err);
} else {
Expand All @@ -448,29 +432,43 @@ void checkForUnexpectedFields(const Protobuf::Message& message,
throw ProtoValidationException(err + fatal_error, message);
}
}
return nullptr;
}

// If this is a message, recurse to check for deprecated fields in the sub-message.
if (field->cpp_type() == Protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (field->is_repeated()) {
const int size = reflection->FieldSize(message, field);
for (int j = 0; j < size; ++j) {
checkForUnexpectedFields(reflection->GetRepeatedMessage(message, field, j),
validation_visitor, runtime);
void onMessage(const Protobuf::Message& message, const void*) override {
// Reject unknown fields.
const auto& unknown_fields = message.GetReflection()->GetUnknownFields(message);
if (!unknown_fields.empty()) {
std::string error_msg;
for (int n = 0; n < unknown_fields.field_count(); ++n) {
if (unknown_fields.field(n).number() == ProtobufWellKnown::OriginalTypeFieldNumber) {
continue;
}
} else {
checkForUnexpectedFields(reflection->GetMessage(message, field), validation_visitor,
runtime);
error_msg += absl::StrCat(n > 0 ? ", " : "", unknown_fields.field(n).number());
}
// We use the validation visitor but have hard coded behavior below for deprecated fields.
// TODO(htuch): Unify the deprecated and unknown visitor handling behind the validation
// visitor pattern. https://github.com/envoyproxy/envoy/issues/8092.
if (!error_msg.empty()) {
validation_visitor_.onUnknownField("type " + message.GetTypeName() +
" with unknown field set {" + error_msg + "}");
}
}
}
}

private:
ProtobufMessage::ValidationVisitor& validation_visitor_;
Runtime::Loader* runtime_;
};

} // namespace

void MessageUtil::checkForUnexpectedFields(const Protobuf::Message& message,
ProtobufMessage::ValidationVisitor& validation_visitor,
Runtime::Loader* runtime) {
::Envoy::checkForUnexpectedFields(API_RECOVER_ORIGINAL(message), validation_visitor, runtime);
UnexpectedFieldProtoVisitor unexpected_field_visitor(validation_visitor, runtime);
ProtobufMessage::traverseMessage(unexpected_field_visitor, API_RECOVER_ORIGINAL(message),
nullptr);
}

std::string MessageUtil::getYamlStringFromMessage(const Protobuf::Message& message,
Expand Down
50 changes: 50 additions & 0 deletions source/common/protobuf/visitor.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#include "common/protobuf/visitor.h"

namespace Envoy {
namespace ProtobufMessage {

void traverseMutableMessage(ProtoVisitor& visitor, Protobuf::Message& message, const void* ctxt) {
visitor.onMessage(message, ctxt);
const Protobuf::Descriptor* descriptor = message.GetDescriptor();
const Protobuf::Reflection* reflection = message.GetReflection();
for (int i = 0; i < descriptor->field_count(); ++i) {
const Protobuf::FieldDescriptor* field = descriptor->field(i);
const void* field_ctxt = visitor.onField(message, *field, ctxt);
// If this is a message, recurse to scrub deprecated fields in the sub-message.
if (field->cpp_type() == Protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (field->is_repeated()) {
const int size = reflection->FieldSize(message, field);
for (int j = 0; j < size; ++j) {
traverseMutableMessage(visitor, *reflection->MutableRepeatedMessage(&message, field, j),
field_ctxt);
}
} else if (reflection->HasField(message, field)) {
traverseMutableMessage(visitor, *reflection->MutableMessage(&message, field), field_ctxt);
}
}
}
}
void traverseMessage(ConstProtoVisitor& visitor, const Protobuf::Message& message,
const void* ctxt) {
visitor.onMessage(message, ctxt);
const Protobuf::Descriptor* descriptor = message.GetDescriptor();
const Protobuf::Reflection* reflection = message.GetReflection();
for (int i = 0; i < descriptor->field_count(); ++i) {
const Protobuf::FieldDescriptor* field = descriptor->field(i);
const void* field_ctxt = visitor.onField(message, *field, ctxt);
// If this is a message, recurse to scrub deprecated fields in the sub-message.
htuch marked this conversation as resolved.
Show resolved Hide resolved
if (field->cpp_type() == Protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (field->is_repeated()) {
const int size = reflection->FieldSize(message, field);
for (int j = 0; j < size; ++j) {
traverseMessage(visitor, reflection->GetRepeatedMessage(message, field, j), field_ctxt);
}
} else if (reflection->HasField(message, field)) {
traverseMessage(visitor, reflection->GetMessage(message, field), field_ctxt);
}
}
}
}

} // namespace ProtobufMessage
} // namespace Envoy
43 changes: 43 additions & 0 deletions source/common/protobuf/visitor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#pragma once

#include "common/protobuf/protobuf.h"

namespace Envoy {
namespace ProtobufMessage {

class ProtoVisitor {
public:
virtual ~ProtoVisitor() = default;

// Invoked when a field is visited, with the message, field descriptor and context. Returns a new
// context for use when traversing the sub-message in a field.
virtual const void* onField(Protobuf::Message&, const Protobuf::FieldDescriptor&,
const void* ctxt) {
return ctxt;
}

// Invoked when a message is visited, with the message and a context.
virtual void onMessage(Protobuf::Message&, const void*){};
};

class ConstProtoVisitor {
public:
virtual ~ConstProtoVisitor() = default;

// Invoked when a field is visited, with the message, field descriptor and context. Returns a new
// context for use when traversing the sub-message in a field.
virtual const void* onField(const Protobuf::Message&, const Protobuf::FieldDescriptor&,
const void* ctxt) {
return ctxt;
}

// Invoked when a message is visited, with the message and a context.
virtual void onMessage(const Protobuf::Message&, const void*){};
};

void traverseMutableMessage(ProtoVisitor& visitor, Protobuf::Message& message, const void* ctxt);
void traverseMessage(ConstProtoVisitor& visitor, const Protobuf::Message& message,
const void* ctxt);

} // namespace ProtobufMessage
} // namespace Envoy