Add gn args to metadata and metadata diffing

When diffing, should help confirm we're comparing apples to apples.
When just analyzing. Validating that the correct gn args were used as
actually quite important, so they are worth recording.

NOTRY=true
BUG=681694

Review-Url: https://codereview.chromium.org/2802893002
Cr-Commit-Position: refs/heads/master@{#462559}

tools/binary_size/describe.py

@@ -39,22 +39,23 @@ def _DescribeSectionSizes(self, section_sizes):
                            if k in relevant_names or k.startswith('.data'))
     total_bytes = sum(v for k, v in section_sizes.iteritems()
                       if k in section_names and k != '.bss')
+    yield ''
     yield 'Section Sizes (Total={:,} bytes):'.format(total_bytes)
     for name in section_names:
       size = section_sizes[name]
       if name == '.bss':
-        yield '{}: {:,} bytes (not included in totals)'.format(name, size)
+        yield '    {}: {:,} bytes (not included in totals)'.format(name, size)
       else:
         percent = float(size) / total_bytes if total_bytes else 0
-        yield '{}: {:,} bytes ({:.1%})'.format(name, size, percent)
+        yield '    {}: {:,} bytes ({:.1%})'.format(name, size, percent)
 
     if self.verbose:
       yield ''
       yield 'Other section sizes:'
       section_names = sorted(k for k in section_sizes.iterkeys()
                              if k not in section_names)
       for name in section_names:
-        yield '{}: {:,} bytes'.format(name, section_sizes[name])
+        yield '    {}: {:,} bytes'.format(name, section_sizes[name])
 
   def _DescribeSymbol(self, sym):
     # SymbolGroups are passed here when we don't want to expand them.
@@ -138,22 +139,43 @@ def prefix_func(sym):
     group_desc = self._DescribeSymbolGroup(diff, prefix_func=prefix_func)
     return itertools.chain((header_str,), group_desc)
 
+  def _DescribeSizeInfoDiff(self, diff):
+    common_metadata = {k: v for k, v in diff.old_metadata.iteritems()
+                       if diff.new_metadata[k] == v}
+    old_metadata = {k: v for k, v in diff.old_metadata.iteritems()
+                    if k not in common_metadata}
+    new_metadata = {k: v for k, v in diff.new_metadata.iteritems()
+                    if k not in common_metadata}
+    metadata_desc = itertools.chain(
+        ('Common Metadata:',),
+        ('    %s' % line for line in DescribeMetadata(common_metadata)),
+        ('Old Metadata:',),
+        ('    %s' % line for line in DescribeMetadata(old_metadata)),
+        ('New Metadata:',),
+        ('    %s' % line for line in DescribeMetadata(new_metadata)))
+    section_desc = self._DescribeSectionSizes(diff.section_sizes)
+    group_desc = self.GenerateLines(diff.symbols)
+    return itertools.chain(metadata_desc, section_desc, ('',), group_desc)
+
+  def _DescribeSizeInfo(self, size_info):
+    metadata_desc = itertools.chain(
+        ('Metadata:',),
+        ('    %s' % line for line in DescribeMetadata(size_info.metadata)))
+    section_desc = self._DescribeSectionSizes(size_info.section_sizes)
+    group_desc = self.GenerateLines(size_info.symbols)
+    return itertools.chain(metadata_desc, section_desc, ('',), group_desc)
+
   def GenerateLines(self, obj):
+    if isinstance(obj, models.SizeInfoDiff):
+      return self._DescribeSizeInfoDiff(obj)
     if isinstance(obj, models.SizeInfo):
-      metadata_desc = 'Metadata: %s' % DescribeSizeInfoMetadata(obj)
-      section_desc = self._DescribeSectionSizes(obj.section_sizes)
-      group_desc = self.GenerateLines(obj.symbols)
-      return itertools.chain((metadata_desc,), section_desc, ('',), group_desc)
-
+      return self._DescribeSizeInfo(obj)
     if isinstance(obj, models.SymbolDiff):
       return self._DescribeSymbolDiff(obj)
-
     if isinstance(obj, models.SymbolGroup):
       return self._DescribeSymbolGroup(obj)
-
     if isinstance(obj, models.Symbol):
       return self._DescribeSymbol(obj)
-
     return (repr(obj),)
 
 
@@ -199,14 +221,17 @@ def _UtcToLocal(utc):
   return utc + offset
 
 
-def DescribeSizeInfoMetadata(size_info):
-  display_dict = size_info.metadata.copy()
+def DescribeMetadata(metadata):
+  display_dict = metadata.copy()
   timestamp = display_dict.get(models.METADATA_ELF_MTIME)
   if timestamp:
     timestamp_obj = datetime.datetime.utcfromtimestamp(timestamp)
     display_dict[models.METADATA_ELF_MTIME] = (
         _UtcToLocal(timestamp_obj).strftime('%Y-%m-%d %H:%M:%S'))
-  return ' '.join(sorted('%s=%s' % t for t in display_dict.iteritems()))
+  gn_args = display_dict.get(models.METADATA_GN_ARGS)
+  if gn_args:
+    display_dict[models.METADATA_GN_ARGS] = '; '.join(gn_args)
+  return sorted('%s=%s' % t for t in display_dict.iteritems())
 
 
 def GenerateLines(obj, verbose=False):

tools/binary_size/integration_test.py

@@ -87,6 +87,8 @@ def test_ConsoleNullDiff(self):
   def test_ActualDiff(self):
     map1 = self._CloneSizeInfo()
     map2 = self._CloneSizeInfo()
+    map1.metadata = {"foo": 1, "bar": [1,2,3], "baz": "yes"}
+    map2.metadata = {"foo": 1, "bar": [1,3], "baz": "yes"}
     map1.symbols -= map1.symbols[0]
     map2.symbols -= map2.symbols[-1]
     map1.symbols[1].size -= 10

tools/binary_size/map2size.py

@@ -288,6 +288,19 @@ def _SectionSizesFromElf(elf_path, tool_prefix):
   return section_sizes
 
 
+def _ParseGnArgs(args_path):
+  """Returns a list of normalized "key=value" strings."""
+  args = {}
+  with open(args_path) as f:
+    for l in f:
+      # Strips #s even if within string literal. Not a problem in practice.
+      parts = l.split('#')[0].split('=')
+      if len(parts) != 2:
+        continue
+      args[parts[0].strip()] = parts[1].strip()
+  return ["%s=%s" % x for x in sorted(args.iteritems())]
+
+
 def main(argv):
   parser = argparse.ArgumentParser(argv)
   parser.add_argument('elf_file', help='Path to input ELF file.')
@@ -322,6 +335,7 @@ def main(argv):
     timestamp_obj = datetime.datetime.utcfromtimestamp(os.path.getmtime(
         args.elf_file))
     timestamp = calendar.timegm(timestamp_obj.timetuple())
+    gn_args = _ParseGnArgs(os.path.join(lazy_paths.output_directory, 'args.gn'))
 
     def relative_to_out(path):
       return os.path.relpath(path, lazy_paths.VerifyOutputDirectory())
@@ -332,6 +346,7 @@ def relative_to_out(path):
         models.METADATA_ELF_FILENAME: relative_to_out(args.elf_file),
         models.METADATA_ELF_MTIME: timestamp,
         models.METADATA_ELF_BUILD_ID: build_id,
+        models.METADATA_GN_ARGS: gn_args,
     }
 
   size_info = Analyze(map_file_path, lazy_paths)
@@ -346,8 +361,8 @@ def relative_to_out(path):
 
     size_info.metadata = metadata
 
-  logging.info('Recording metadata: %s',
-               describe.DescribeSizeInfoMetadata(size_info))
+  logging.info('Recording metadata: \n  %s',
+               '\n  '.join(describe.DescribeMetadata(size_info.metadata)))
   logging.info('Saving result to %s', args.output_file)
   file_format.SaveSizeInfo(size_info, args.output_file)
   logging.info('Done')

tools/binary_size/models.py

@@ -31,10 +31,12 @@
 
 
 METADATA_GIT_REVISION = 'git_revision'
-METADATA_MAP_FILENAME = 'map_file_name'
-METADATA_ELF_FILENAME = 'elf_file_name'
+METADATA_MAP_FILENAME = 'map_file_name'  # Path relative to output_directory.
+METADATA_ELF_FILENAME = 'elf_file_name'  # Path relative to output_directory.
 METADATA_ELF_MTIME = 'elf_mtime'  # int timestamp in utc.
 METADATA_ELF_BUILD_ID = 'elf_build_id'
+METADATA_GN_ARGS = 'gn_args'
+
 
 SECTION_TO_SECTION_NAME = {
     'b': '.bss',
@@ -49,7 +51,7 @@ class SizeInfo(object):
 
   Fields:
     section_sizes: A dict of section_name -> size.
-    symbols: A SymbolGroup (or SymbolDiff) with all symbols in it.
+    symbols: A SymbolGroup with all symbols in it.
     metadata: A dict.
   """
   __slots__ = (
@@ -65,6 +67,29 @@ def __init__(self, section_sizes, symbols, metadata=None):
     self.metadata = metadata or {}
 
 
+class SizeInfoDiff(object):
+  """What you get when you Diff() two SizeInfo objects.
+
+  Fields:
+    section_sizes: A dict of section_name -> size delta.
+    symbols: A SymbolDiff with all symbols in it.
+    old_metadata: metadata of the "old" SizeInfo.
+    new_metadata: metadata of the "new" SizeInfo.
+  """
+  __slots__ = (
+      'section_sizes',
+      'symbols',
+      'old_metadata',
+      'new_metadata',
+  )
+
+  def __init__(self, section_sizes, symbols, old_metadata, new_metadata):
+    self.section_sizes = section_sizes
+    self.symbols = symbols
+    self.old_metadata = old_metadata
+    self.new_metadata = new_metadata
+
+
 class BaseSymbol(object):
   """Base class for Symbol and SymbolGroup.
 
@@ -547,9 +572,7 @@ def WhereNotUnchanged(self):
 def Diff(new, old):
   """Diffs two SizeInfo or SymbolGroup objects.
 
-  When diffing SizeInfos, ret.section_sizes are the result of |new| - |old|, and
-  ret.symbols will be a SymbolDiff.
-
+  When diffing SizeInfos, a SizeInfoDiff is returned.
   When diffing SymbolGroups, a SymbolDiff is returned.
 
   Returns:
@@ -561,7 +584,7 @@ def Diff(new, old):
     section_sizes = {
         k:new.section_sizes[k] - v for k, v in old.section_sizes.iteritems()}
     symbol_diff = Diff(new.symbols, old.symbols)
-    return SizeInfo(section_sizes, symbol_diff)
+    return SizeInfoDiff(section_sizes, symbol_diff, old.metadata, new.metadata)
 
   assert isinstance(new, SymbolGroup) and isinstance(old, SymbolGroup)
   symbols_by_key = collections.defaultdict(list)

tools/binary_size/testdata/ActualDiff.golden

@@ -1,46 +1,53 @@
-Metadata: 
+Common Metadata:
+    baz=yes
+    foo=1
+Old Metadata:
+    bar=[1, 3]
+New Metadata:
+    bar=[1, 2, 3]
+
 Section Sizes (Total=0 bytes):
-.bss: 0 bytes (not included in totals)
-.data: 0 bytes (0.0%)
-.data.rel.ro: 0 bytes (0.0%)
-.data.rel.ro.local: 0 bytes (0.0%)
-.rodata: 0 bytes (0.0%)
-.text: 0 bytes (0.0%)
+    .bss: 0 bytes (not included in totals)
+    .data: 0 bytes (0.0%)
+    .data.rel.ro: 0 bytes (0.0%)
+    .data.rel.ro.local: 0 bytes (0.0%)
+    .rodata: 0 bytes (0.0%)
+    .text: 0 bytes (0.0%)
 
 Other section sizes:
-.ARM.attributes: 0 bytes
-.ARM.exidx: 0 bytes
-.ARM.extab: 0 bytes
-.comment: 0 bytes
-.debug_abbrev: 0 bytes
-.debug_aranges: 0 bytes
-.debug_frame: 0 bytes
-.debug_info: 0 bytes
-.debug_line: 0 bytes
-.debug_loc: 0 bytes
-.debug_pubnames: 0 bytes
-.debug_pubtypes: 0 bytes
-.debug_ranges: 0 bytes
-.debug_str: 0 bytes
-.dynamic: 0 bytes
-.dynstr: 0 bytes
-.dynsym: 0 bytes
-.fini_array: 0 bytes
-.gnu.version: 0 bytes
-.gnu.version_d: 0 bytes
-.gnu.version_r: 0 bytes
-.got: 0 bytes
-.hash: 0 bytes
-.init_array: 0 bytes
-.interp: 0 bytes
-.note.gnu.build-id: 0 bytes
-.note.gnu.gold-version: 0 bytes
-.plt: 0 bytes
-.rel.dyn: 0 bytes
-.rel.plt: 0 bytes
-.shstrtab: 0 bytes
-.strtab: 0 bytes
-.symtab: 0 bytes
+    .ARM.attributes: 0 bytes
+    .ARM.exidx: 0 bytes
+    .ARM.extab: 0 bytes
+    .comment: 0 bytes
+    .debug_abbrev: 0 bytes
+    .debug_aranges: 0 bytes
+    .debug_frame: 0 bytes
+    .debug_info: 0 bytes
+    .debug_line: 0 bytes
+    .debug_loc: 0 bytes
+    .debug_pubnames: 0 bytes
+    .debug_pubtypes: 0 bytes
+    .debug_ranges: 0 bytes
+    .debug_str: 0 bytes
+    .dynamic: 0 bytes
+    .dynstr: 0 bytes
+    .dynsym: 0 bytes
+    .fini_array: 0 bytes
+    .gnu.version: 0 bytes
+    .gnu.version_d: 0 bytes
+    .gnu.version_r: 0 bytes
+    .got: 0 bytes
+    .hash: 0 bytes
+    .init_array: 0 bytes
+    .interp: 0 bytes
+    .note.gnu.build-id: 0 bytes
+    .note.gnu.gold-version: 0 bytes
+    .plt: 0 bytes
+    .rel.dyn: 0 bytes
+    .rel.plt: 0 bytes
+    .shstrtab: 0 bytes
+    .strtab: 0 bytes
+    .symtab: 0 bytes
 
 1 symbols added (+), 1 changed (~), 1 removed (-), 42 unchanged (=)
 Showing 45 symbols with total size: 0 bytes

tools/binary_size/testdata/ConsoleNullDiff.golden

@@ -1,11 +1,14 @@
-Metadata: 
+Common Metadata:
+Old Metadata:
+New Metadata:
+
 Section Sizes (Total=0 bytes):
-.bss: 0 bytes (not included in totals)
-.data: 0 bytes (0.0%)
-.data.rel.ro: 0 bytes (0.0%)
-.data.rel.ro.local: 0 bytes (0.0%)
-.rodata: 0 bytes (0.0%)
-.text: 0 bytes (0.0%)
+    .bss: 0 bytes (not included in totals)
+    .data: 0 bytes (0.0%)
+    .data.rel.ro: 0 bytes (0.0%)
+    .data.rel.ro.local: 0 bytes (0.0%)
+    .rodata: 0 bytes (0.0%)
+    .text: 0 bytes (0.0%)
 
 0 symbols added (+), 0 changed (~), 0 removed (-), 45 unchanged (not shown)
 Showing 0 symbols with total size: 0 bytes