Skip to content

Commit

Permalink
Android: Create size-info files for bundles
Browse files Browse the repository at this point in the history
These files are needed by SuperSize to do binary size analysis.

Refactors logic for creating these apkbuilder->separated action
so that it can be used by both apks and aabs.

Tested that .info files produces by the modern .aab and .apk are
about the same size.

Bug: 873714
Change-Id: Icbe8aee1f05da29d787c92a9697f2b1b719f7f42
Reviewed-on: https://chromium-review.googlesource.com/c/1461349
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Peter Wen <wnwen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#630803}
  • Loading branch information
agrieve authored and Commit Bot committed Feb 11, 2019
1 parent 357a88a commit b838d83
Show file tree
Hide file tree
Showing 11 changed files with 290 additions and 246 deletions.
2 changes: 1 addition & 1 deletion PRESUBMIT.py
Original file line number Diff line number Diff line change
Expand Up @@ -664,6 +664,7 @@
'build/android/gyp/create_app_bundle.pydeps',
'build/android/gyp/create_apk_operations_script.pydeps',
'build/android/gyp/create_java_binary_script.pydeps',
'build/android/gyp/create_size_info_files.pydeps',
'build/android/gyp/create_stack_script.pydeps',
'build/android/gyp/create_test_runner_script.pydeps',
'build/android/gyp/create_tool_wrapper.pydeps',
Expand All @@ -681,7 +682,6 @@
'build/android/gyp/jinja_template.pydeps',
'build/android/gyp/lint.pydeps',
'build/android/gyp/main_dex_list.pydeps',
'build/android/gyp/merge_jar_info_files.pydeps',
'build/android/gyp/merge_manifest.pydeps',
'build/android/gyp/prepare_resources.pydeps',
'build/android/gyp/proguard.pydeps',
Expand Down
65 changes: 16 additions & 49 deletions build/android/gyp/apkbuilder.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,18 @@
def _ParseArgs(args):
parser = argparse.ArgumentParser()
build_utils.AddDepfileOption(parser)
parser.add_argument('--assets',
help='GYP-list of files to add as assets in the form '
'"srcPath:zipPath", where ":zipPath" is optional.',
default='[]')
parser.add_argument('--java-resources',
help='GYP-list of java_resources JARs to include.',
default='[]')
parser.add_argument(
'--assets',
help='GYP-list of files to add as assets in the form '
'"srcPath:zipPath", where ":zipPath" is optional.')
parser.add_argument(
'--java-resources', help='GYP-list of java_resources JARs to include.')
parser.add_argument('--write-asset-list',
action='store_true',
help='Whether to create an assets/assets_list file.')
parser.add_argument('--uncompressed-assets',
help='Same as --assets, except disables compression.',
default='[]')
parser.add_argument(
'--uncompressed-assets',
help='Same as --assets, except disables compression.')
parser.add_argument('--resource-apk',
help='An .ap_ file built using aapt',
required=True)
Expand All @@ -51,10 +50,6 @@ def _ParseArgs(args):
required=True)
parser.add_argument('--format', choices=['apk', 'bundle-module'],
default='apk', help='Specify output format.')
parser.add_argument('--apk-pak-info-path',
help='Path to the *.apk.pak.info file')
parser.add_argument('--apk-res-info-path',
help='Path to the *.apk.res.info file')
parser.add_argument('--dex-file',
help='Path to the classes.dex to use')
parser.add_argument('--uncompress-dex', action='store_true',
Expand All @@ -74,13 +69,13 @@ def _ParseArgs(args):
parser.add_argument('--secondary-android-abi',
help='The secondary Android architecture to use for'
'secondary native libraries')
parser.add_argument('--native-lib-placeholders',
help='GYP-list of native library placeholders to add.',
default='[]')
parser.add_argument('--secondary-native-lib-placeholders',
help='GYP-list of native library placeholders to add '
'for the secondary ABI',
default='[]')
parser.add_argument(
'--native-lib-placeholders',
help='GYP-list of native library placeholders to add.')
parser.add_argument(
'--secondary-native-lib-placeholders',
help='GYP-list of native library placeholders to add '
'for the secondary ABI')
parser.add_argument('--uncompress-shared-libraries', default='False',
choices=['true', 'True', 'false', 'False'],
help='Whether to uncompress native shared libraries. Argument must be '
Expand Down Expand Up @@ -226,24 +221,6 @@ def _AddNativeLibraries(out_apk, native_libs, android_abi, uncompress):
compress=compress)


def _MergeResInfoFiles(res_info_path, resource_apk):
resource_apk_info_path = resource_apk + '.info'
shutil.copy(resource_apk_info_path, res_info_path)


def _FilterPakInfoPaths(assets):
return [f.split(':')[0] + '.info' for f in assets if f.endswith('.pak')]


def _MergePakInfoFiles(merged_path, pak_infos):
info_lines = set()
for pak_info_path in pak_infos:
with open(pak_info_path, 'r') as src_info_file:
info_lines.update(src_info_file.readlines())
with open(merged_path, 'w') as merged_info_file:
merged_info_file.writelines(sorted(info_lines))


def main(args):
args = build_utils.ExpandFileArgs(args)
options = _ParseArgs(args)
Expand All @@ -266,11 +243,6 @@ def main(args):
assets = _ExpandPaths(options.assets)
uncompressed_assets = _ExpandPaths(options.uncompressed_assets)

if options.apk_pak_info_path:
pak_infos = _FilterPakInfoPaths(
options.assets + options.uncompressed_assets)
depfile_deps.extend(pak_infos)

# Included via .build_config, so need to write it to depfile.
depfile_deps.extend(x[0] for x in assets)
depfile_deps.extend(x[0] for x in uncompressed_assets)
Expand Down Expand Up @@ -388,11 +360,6 @@ def copy_resource(zipinfo, out_dir=''):
apk_root_dir + apk_path,
data=java_resource_jar.read(apk_path))

if options.apk_pak_info_path:
_MergePakInfoFiles(options.apk_pak_info_path, pak_infos)
if options.apk_res_info_path:
_MergeResInfoFiles(options.apk_res_info_path, options.resource_apk)

if options.format == 'apk':
finalize_apk.FinalizeApk(options.apksigner_path, options.zipalign_path,
f.name, f.name, options.key_path,
Expand Down
2 changes: 1 addition & 1 deletion build/android/gyp/compile_resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,7 @@ def _CreateResourceInfoFile(
lines.update(zip_info_file.readlines())
for dest, source in renamed_paths.iteritems():
lines.add('Rename:{},{}\n'.format(dest, source))
with open(apk_info_path, 'w') as info_file:
with build_utils.AtomicOutput(apk_info_path) as info_file:
info_file.writelines(sorted(lines))


Expand Down
167 changes: 167 additions & 0 deletions build/android/gyp/create_size_info_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
#!/usr/bin/env python

# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Creates size-info/*.info files used by SuperSize."""

import argparse
import os
import sys
import zipfile

from util import build_utils
from util import jar_info_utils
from util import md5_check


def _MergeResInfoFiles(res_info_path, info_paths):
# Concatenate them all.
# only_if_changed=False since no build rules depend on this as an input.
with build_utils.AtomicOutput(res_info_path, only_if_changed=False) as dst:
for p in info_paths:
with open(p) as src:
dst.write(src.read())


def _PakInfoPathsForAssets(assets):
return [f.split(':')[0] + '.info' for f in assets if f.endswith('.pak')]


def _MergePakInfoFiles(merged_path, pak_infos):
info_lines = set()
for pak_info_path in pak_infos:
with open(pak_info_path, 'r') as src_info_file:
info_lines.update(src_info_file.readlines())
# only_if_changed=False since no build rules depend on this as an input.
with build_utils.AtomicOutput(merged_path, only_if_changed=False) as f:
f.writelines(sorted(info_lines))


def _FullJavaNameFromClassFilePath(path):
# Input: base/android/java/src/org/chromium/Foo.class
# Output: base.android.java.src.org.chromium.Foo
if not path.endswith('.class'):
return ''
path = os.path.splitext(path)[0]
parts = []
while path:
# Use split to be platform independent.
head, tail = os.path.split(path)
path = head
parts.append(tail)
parts.reverse() # Package comes first
return '.'.join(parts)


def _MergeJarInfoFiles(output, inputs):
"""Merge several .jar.info files to generate an .apk.jar.info.
Args:
output: output file path.
inputs: List of .info.jar or .jar files.
"""
info_data = dict()
for path in inputs:
# android_java_prebuilt adds jar files in the src directory (relative to
# the output directory, usually ../../third_party/example.jar).
# android_aar_prebuilt collects jar files in the aar file and uses the
# java_prebuilt rule to generate gen/example/classes.jar files.
# We scan these prebuilt jars to parse each class path for the FQN. This
# allows us to later map these classes back to their respective src
# directories.
# TODO(agrieve): This should probably also check that the mtime of the .info
# is newer than that of the .jar, or change prebuilts to always output
# .info files so that they always exist (and change the depfile to
# depend directly on them).
if path.endswith('.info'):
info_data.update(jar_info_utils.ParseJarInfoFile(path))
else:
with zipfile.ZipFile(path) as zip_info:
for name in zip_info.namelist():
fully_qualified_name = _FullJavaNameFromClassFilePath(name)
if fully_qualified_name:
info_data[fully_qualified_name] = '{}/{}'.format(path, name)

# only_if_changed=False since no build rules depend on this as an input.
with build_utils.AtomicOutput(output, only_if_changed=False) as f:
jar_info_utils.WriteJarInfoFile(f, info_data)


def _FindJarInputs(jar_paths):
ret = []
for jar_path in jar_paths:
jar_info_path = jar_path + '.info'
if os.path.exists(jar_info_path):
ret.append(jar_info_path)
else:
ret.append(jar_path)
return ret


def main(args):
args = build_utils.ExpandFileArgs(args)
parser = argparse.ArgumentParser(description=__doc__)
build_utils.AddDepfileOption(parser)
parser.add_argument(
'--jar-info-path', required=True, help='Output .jar.info file')
parser.add_argument(
'--pak-info-path', required=True, help='Output .pak.info file')
parser.add_argument(
'--res-info-path', required=True, help='Output .res.info file')
parser.add_argument(
'--jar-files',
required=True,
action='append',
help='GN-list of .jar file paths')
parser.add_argument(
'--assets',
required=True,
action='append',
help='GN-list of files to add as assets in the form '
'"srcPath:zipPath", where ":zipPath" is optional.')
parser.add_argument(
'--uncompressed-assets',
required=True,
action='append',
help='Same as --assets, except disables compression.')
parser.add_argument(
'--resource-apk',
dest='resource_apks',
required=True,
action='append',
help='An .ap_ file built using aapt')

options = parser.parse_args(args)

options.jar_files = build_utils.ParseGnList(options.jar_files)
options.assets = build_utils.ParseGnList(options.assets)
options.uncompressed_assets = build_utils.ParseGnList(
options.uncompressed_assets)

jar_inputs = _FindJarInputs(set(options.jar_files))
pak_inputs = _PakInfoPathsForAssets(options.assets +
options.uncompressed_assets)
res_inputs = [p + '.info' for p in options.resource_apks]

# Don't bother re-running if no .info files have changed (saves ~250ms).
md5_check.CallAndRecordIfStale(
lambda: _MergeJarInfoFiles(options.jar_info_path, jar_inputs),
input_paths=jar_inputs,
output_paths=[options.jar_info_path])

# Always recreate these (just as fast as md5 checking them).
_MergePakInfoFiles(options.pak_info_path, pak_inputs)
_MergeResInfoFiles(options.res_info_path, res_inputs)

all_inputs = jar_inputs + pak_inputs + res_inputs
build_utils.WriteDepfile(
options.depfile,
options.jar_info_path,
inputs=all_inputs,
add_pydeps=False)


if __name__ == '__main__':
main(sys.argv[1:])
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Generated by running:
# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/merge_jar_info_files.pydeps build/android/gyp/merge_jar_info_files.py
# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_size_info_files.pydeps build/android/gyp/create_size_info_files.py
../../gn_helpers.py
merge_jar_info_files.py
create_size_info_files.py
util/__init__.py
util/build_utils.py
util/jar_info_utils.py
Expand Down
19 changes: 6 additions & 13 deletions build/android/gyp/javac.py
Original file line number Diff line number Diff line change
Expand Up @@ -469,13 +469,6 @@ def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
logging.info('Completed all steps in _OnStaleMd5')


def _ParseAndFlattenGnLists(gn_lists):
ret = []
for arg in gn_lists:
ret.extend(build_utils.ParseGnList(arg))
return ret


def _ParseOptions(argv):
parser = optparse.OptionParser()
build_utils.AddDepfileOption(parser)
Expand Down Expand Up @@ -559,13 +552,13 @@ def _ParseOptions(argv):
options, args = parser.parse_args(argv)
build_utils.CheckOptions(options, parser, required=('jar_path',))

options.bootclasspath = _ParseAndFlattenGnLists(options.bootclasspath)
options.full_classpath = _ParseAndFlattenGnLists(options.full_classpath)
options.interface_classpath = _ParseAndFlattenGnLists(
options.bootclasspath = build_utils.ParseGnList(options.bootclasspath)
options.full_classpath = build_utils.ParseGnList(options.full_classpath)
options.interface_classpath = build_utils.ParseGnList(
options.interface_classpath)
options.processorpath = _ParseAndFlattenGnLists(options.processorpath)
options.processors = _ParseAndFlattenGnLists(options.processors)
options.java_srcjars = _ParseAndFlattenGnLists(options.java_srcjars)
options.processorpath = build_utils.ParseGnList(options.processorpath)
options.processors = build_utils.ParseGnList(options.processors)
options.java_srcjars = build_utils.ParseGnList(options.java_srcjars)

if options.java_version == '1.8' and options.bootclasspath:
# Android's boot jar doesn't contain all java 8 classes.
Expand Down
Loading

0 comments on commit b838d83

Please sign in to comment.