forked from chromium/chromium
-
Notifications
You must be signed in to change notification settings - Fork 0
/
PRESUBMIT.py
359 lines (317 loc) · 13.9 KB
/
PRESUBMIT.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
# Copyright (c) 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Presubmit script for sync
This checks that ModelTypeInfo entries in model_type.cc follow conventions.
See CheckModelTypeInfoMap or model_type.cc for more detail on the rules.
"""
import os
# Some definitions don't follow all the conventions we want to enforce.
# It's either difficult or impossible to fix this, so we ignore the problem(s).
GRANDFATHERED_MODEL_TYPES = [
'UNSPECIFIED', # Doesn't have a root tag or notification type.
'TOP_LEVEL_FOLDER', # Doesn't have a root tag or notification type.
'AUTOFILL_WALLET_DATA', # Root tag and model type string lack DATA suffix.
'APP_SETTINGS', # Model type string has inconsistent capitalization.
'EXTENSION_SETTINGS', # Model type string has inconsistent capitalization.
'SUPERVISED_USER_SETTINGS', # Root tag and model type string replace
# 'Supervised' with 'Managed'
'SUPERVISED_USERS', # See previous.
'SUPERVISED_USER_WHITELISTS', # See previous.
'SUPERVISED_USER_SHARED_SETTINGS', # See previous.
'PROXY_TABS', # Doesn't have a root tag or notification type.
'NIGORI'] # Model type string is 'encryption keys'.
# Number of distinct fields in a map entry; used to create
# sets that check for uniqueness.
MAP_ENTRY_FIELD_COUNT = 6
# String that precedes the ModelType when referencing the
# proto field number enum e.g.
# sync_pb::EntitySpecifics::kManagedUserFieldNumber.
# Used to map from enum references to the ModelType.
FIELD_NUMBER_PREFIX = 'sync_pb::EntitySpecifics::k'
# Start and end regexes for finding the EntitySpecifics definition in
# sync.proto.
PROTO_DEFINITION_START_PATTERN = '^message EntitySpecifics'
PROTO_DEFINITION_END_PATTERN = '^\}'
# Start and end regexes for finding the ModelTypeInfoMap definition
# in model_type.cc.
MODEL_TYPE_START_PATTERN = '^const ModelTypeInfo kModelTypeInfoMap'
MODEL_TYPE_END_PATTERN = '^\};'
# Strings relating to files we'll need to read.
# model_type.cc is where the ModelTypeInfoMap is
# sync.proto is where the proto definitions for ModelTypes are.
PROTO_FILE_PATH = './protocol/sync.proto'
MODEL_TYPE_FILE_NAME = 'model_type.cc'
def CheckChangeOnUpload(input_api, output_api):
"""Preupload check function required by presubmit convention.
See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
"""
for f in input_api.AffectedFiles():
if(f.LocalPath().endswith(MODEL_TYPE_FILE_NAME)):
return CheckModelTypeInfoMap(input_api, output_api, f)
return []
def CheckChangeOnCommit(input_api, output_api):
"""Precommit check function required by presubmit convention.
See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
"""
for f in input_api.AffectedFiles():
if f.LocalPath().endswith(MODEL_TYPE_FILE_NAME):
return CheckModelTypeInfoMap(input_api, output_api, f)
return []
def CheckModelTypeInfoMap(input_api, output_api, model_type_file):
"""Checks the kModelTypeInfoMap in model_type.cc follows conventions.
Checks that the kModelTypeInfoMap follows the below rules:
1) The model type string should match the model type name, but with
only the first letter capitalized and spaces instead of underscores.
2) The root tag should be the same as the model type but all lowercase.
3) The notification type should match the proto message name.
4) No duplicate data across model types.
Args:
input_api: presubmit_support InputApi instance
output_api: presubmit_support OutputApi instance
model_type_file: AffectedFile object where the ModelTypeInfoMap is
Returns:
A (potentially empty) list PresubmitError objects corresponding to
violations of the above rules.
"""
accumulated_problems = []
map_entries = ParseModelTypeEntries(
input_api, model_type_file.AbsoluteLocalPath())
# If any line of the map changed, we check the whole thing since
# definitions span multiple lines and there are rules that apply across
# all definitions e.g. no duplicated field values.
check_map = False
for line_num, _ in model_type_file.ChangedContents():
for map_entry in map_entries:
if line_num in map_entry.affected_lines:
check_map = True
break
if not check_map:
return []
proto_field_definitions = ParseSyncProtoFieldIdentifiers(
input_api, os.path.abspath(PROTO_FILE_PATH))
accumulated_problems.extend(
CheckNoDuplicatedFieldValues(output_api, map_entries))
for map_entry in map_entries:
entry_problems = []
entry_problems.extend(
CheckNotificationTypeMatchesProtoMessageName(
output_api, map_entry, proto_field_definitions))
if map_entry.model_type not in GRANDFATHERED_MODEL_TYPES:
entry_problems.extend(
CheckModelTypeStringMatchesModelType(output_api, map_entry))
entry_problems.extend(
CheckRootTagMatchesModelType(output_api, map_entry))
if len(entry_problems) > 0:
accumulated_problems.extend(entry_problems)
return accumulated_problems
class ModelTypeEnumEntry(object):
"""Class that encapsulates a ModelTypeInfo definition in model_type.cc.
Allows access to each of the named fields in the definition and also
which lines the definition spans.
Attributes:
model_type: entry's ModelType enum value
notification_type: model type's notification string
root_tag: model type's root tag
model_type_string: string corresponding to the ModelType
field_number: proto field number
histogram_val: value identifying ModelType in histogram
affected_lines: lines in model_type.cc that the definition spans
"""
def __init__(self, entry_strings, affected_lines):
(model_type, notification_type, root_tag, model_type_string,
field_number, histogram_val) = entry_strings
self.model_type = model_type
self.notification_type = notification_type
self.root_tag = root_tag
self.model_type_string = model_type_string
self.field_number = field_number
self.histogram_val = histogram_val
self.affected_lines = affected_lines
def ParseModelTypeEntries(input_api, model_type_cc_path):
"""Parses model_type_cc_path for ModelTypeEnumEntries
Args:
input_api: presubmit_support InputAPI instance
model_type_cc_path: path to file containing the ModelTypeInfo entries
Returns:
A list of ModelTypeEnumEntry objects read from model_type.cc.
e.g. ('AUTOFILL_WALLET_METADATA', 'WALLET_METADATA',
'autofill_wallet_metadata', 'Autofill Wallet Metadata',
'sync_pb::EntitySpecifics::kWalletMetadataFieldNumber', '35',
[63, 64, 65])
"""
file_contents = input_api.ReadFile(model_type_cc_path)
start_pattern = input_api.re.compile(MODEL_TYPE_START_PATTERN)
end_pattern = input_api.re.compile(MODEL_TYPE_END_PATTERN)
results, definition_strings, definition_lines = [], [], []
inside_enum = False
current_line_number = 1
for line in file_contents.splitlines():
if start_pattern.match(line):
inside_enum = True
continue
if inside_enum:
if end_pattern.match(line):
break
line_entries = line.strip().strip('{},').split(',')
definition_strings.extend([entry.strip('" ') for entry in line_entries])
definition_lines.append(current_line_number)
if line.endswith('},'):
results.append(ModelTypeEnumEntry(definition_strings, definition_lines))
definition_strings = []
definition_lines = []
current_line_number += 1
return results
def ParseSyncProtoFieldIdentifiers(input_api, sync_proto_path):
"""Parses proto field identifiers from the EntitySpecifics definition.
Args:
input_api: presubmit_support InputAPI instance
proto_path: path to the file containing the proto field definitions
Returns:
A dictionary of the format {'SyncDataType': 'field_identifier'}
e.g. {'AutofillSpecifics': 'autofill'}
"""
proto_field_definitions = {}
proto_file_contents = input_api.ReadFile(sync_proto_path).splitlines()
start_pattern = input_api.re.compile(PROTO_DEFINITION_START_PATTERN)
end_pattern = input_api.re.compile(PROTO_DEFINITION_END_PATTERN)
in_proto_def = False
for line in proto_file_contents:
if start_pattern.match(line):
in_proto_def = True
continue
if in_proto_def:
if end_pattern.match(line):
break
line = line.strip()
split_proto_line = line.split(' ')
# ignore comments and lines that don't contain definitions.
if '//' in line or len(split_proto_line) < 3:
continue
field_typename = split_proto_line[1]
field_identifier = split_proto_line[2]
proto_field_definitions[field_typename] = field_identifier
return proto_field_definitions
def StripTrailingS(string):
return string.rstrip('sS')
def IsTitleCased(string):
return reduce(lambda bool1, bool2: bool1 and bool2,
[s[0].isupper() for s in string.split(' ')])
def FormatPresubmitError(output_api, message, affected_lines):
""" Outputs a formatted error message with filename and line number(s).
"""
if len(affected_lines) > 1:
message_including_lines = 'Error at lines %d-%d in model_type.cc: %s' %(
affected_lines[0], affected_lines[-1], message)
else:
message_including_lines = 'Error at line %d in model_type.cc: %s' %(
affected_lines[0], message)
return output_api.PresubmitError(message_including_lines)
def CheckNotificationTypeMatchesProtoMessageName(
output_api, map_entry, proto_field_definitions):
"""Check that map_entry's notification type matches sync.proto.
Verifies that the notification_type matches the name of the field defined
in the sync.proto by looking it up in the proto_field_definitions map.
Args:
output_api: presubmit_support OutputApi instance
map_entry: ModelTypeEnumEntry instance
proto_field_definitions: dict of proto field types and field names
Returns:
A potentially empty list of PresubmitError objects corresponding to
violations of the above rule
"""
if map_entry.field_number == '-1':
return []
proto_message_name = proto_field_definitions[
FieldNumberToPrototypeString(map_entry.field_number)]
if map_entry.notification_type.lower() != proto_message_name:
return [
FormatPresubmitError(
output_api,'notification type "%s" does not match proto message'
' name defined in sync.proto: ' '"%s"' %
(map_entry.notification_type, proto_message_name),
map_entry.affected_lines)]
return []
def CheckNoDuplicatedFieldValues(output_api, map_entries):
"""Check that map_entries has no duplicated field values.
Verifies that every map_entry in map_entries doesn't have a field value
used elsewhere in map_entries, ignoring special values ("" and -1).
Args:
output_api: presubmit_support OutputApi instance
map_entries: list of ModelTypeEnumEntry objects to check
Returns:
A list PresubmitError objects for each duplicated field value
"""
problem_list = []
field_value_sets = [set() for i in range(MAP_ENTRY_FIELD_COUNT)]
for map_entry in map_entries:
field_values = [
map_entry.model_type, map_entry.notification_type,
map_entry.root_tag, map_entry.model_type_string,
map_entry.field_number, map_entry.histogram_val]
for i in range(MAP_ENTRY_FIELD_COUNT):
field_value = field_values[i]
field_value_set = field_value_sets[i]
if field_value in field_value_set:
problem_list.append(
FormatPresubmitError(
output_api, 'Duplicated field value "%s"' % field_value,
map_entry.affected_lines))
elif len(field_value) > 0 and field_value != '-1':
field_value_set.add(field_value)
return problem_list
def CheckModelTypeStringMatchesModelType(output_api, map_entry):
"""Check that map_entry's model_type_string matches ModelType.
Args:
output_api: presubmit_support OutputApi instance
map_entry: ModelTypeEnumEntry object to check
Returns:
A list of PresubmitError objects for each violation
"""
problem_list = []
expected_model_type_string = map_entry.model_type.lower().replace('_', ' ')
if (StripTrailingS(expected_model_type_string) !=
StripTrailingS(map_entry.model_type_string.lower())):
problem_list.append(
FormatPresubmitError(
output_api,'model type string "%s" does not match model type.'
' It should be "%s"' % (
map_entry.model_type_string, expected_model_type_string.title()),
map_entry.affected_lines))
if not IsTitleCased(map_entry.model_type_string):
problem_list.append(
FormatPresubmitError(
output_api,'model type string "%s" should be title cased' %
(map_entry.model_type_string), map_entry.affected_lines))
return problem_list
def CheckRootTagMatchesModelType(output_api, map_entry):
"""Check that map_entry's root tag matches ModelType.
Args:
output_api: presubmit_support OutputAPI instance
map_entry: ModelTypeEnumEntry object to check
Returns:
A list of PresubmitError objects for each violation
"""
expected_root_tag = map_entry.model_type.lower()
if (StripTrailingS(expected_root_tag) !=
StripTrailingS(map_entry.root_tag)):
return [
FormatPresubmitError(
output_api,'root tag "%s" does not match model type. It should'
'be "%s"' % (map_entry.root_tag, expected_root_tag),
map_entry.affected_lines)]
return []
def FieldNumberToPrototypeString(field_number):
"""Converts a field number enum reference to an EntitySpecifics string.
Converts a reference to the field number enum to the corresponding
proto data type string.
Args:
field_number: string representation of a field number enum reference
Returns:
A string that is the corresponding proto field data type. e.g.
FieldNumberToPrototypeString('EntitySpecifics::kAppFieldNumber')
=> 'AppSpecifics'
"""
return field_number.replace(FIELD_NUMBER_PREFIX, '').replace(
'FieldNumber', 'Specifics').replace(
'AppNotificationSpecifics', 'AppNotification')