NaCl: Update revision in DEPS, r13018 -> r13078.

Chromium NaCl scripts now uses package_version to extract.

Now that we have split up the toolchain into multiple tars, we should
use the package versioning script to manage which tars to extract.
Unfortunately, currently the SDK uses different directory names compared
to what the NaCl toolchain directories normally use, so we also must map
and move the toolchain directories to the SDK ones.

BUG=365817
TEST= trybots

Review URL: https://codereview.chromium.org/249153003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@266284 0039d316-1c4b-4281-b951-d872f2087c98

DEPS

@@ -28,7 +28,7 @@ vars = {
   "chromiumos_git": "https://chromium.googlesource.com/chromiumos",
   "skia_git": "https://skia.googlesource.com",
   "swig_revision": "230490",
-  "nacl_revision": "13018",
+  "nacl_revision": "13078",
   # After changing nacl_revision, run 'glient sync' and check native_client/DEPS
   # to update other nacl_*_revision's.
   "nacl_tools_revision": "12970",  # native_client/DEPS: tools_rev
@@ -588,8 +588,7 @@ hooks = [
     "pattern": ".",
     "action": [
         "python", "src/build/download_nacl_toolchains.py",
-         "--no-arm-trusted",
-         "--keep",
+        "--exclude", "arm_trusted",
     ],
   },
   {

build/download_nacl_toolchains.py

@@ -17,16 +17,16 @@ def Main(args):
   src_dir = os.path.dirname(script_dir)
   nacl_dir = os.path.join(src_dir, 'native_client')
   nacl_build_dir = os.path.join(nacl_dir, 'build')
-  download_script = os.path.join(nacl_build_dir, 'download_toolchains.py')
-  if not os.path.exists(download_script):
-    print "Can't find '%s'" % download_script
+  package_version_dir = os.path.join(nacl_build_dir, 'package_version')
+  package_version = os.path.join(package_version_dir, 'package_version.py')
+  if not os.path.exists(package_version):
+    print "Can't find '%s'" % package_version
     print 'Presumably you are intentionally building without NativeClient.'
     print 'Skipping NativeClient toolchain download.'
     sys.exit(0)
-  sys.path.insert(0, nacl_build_dir)
-  import download_toolchains
+  sys.path.insert(0, package_version_dir)
+  import package_version
 
-  # TODO (robertm): Finish getting PNaCl ready for prime time.
   # BUG:
   # We remove this --optional-pnacl argument, and instead replace it with
   # --no-pnacl for most cases.  However, if the bot name is an sdk
@@ -41,21 +41,17 @@ def Main(args):
     if use_pnacl:
       print '\n*** DOWNLOADING PNACL TOOLCHAIN ***\n'
     else:
-      args.append('--no-pnacl')
+      args.extend(['--exclude', 'pnacl_newlib'])
 
   # Only download the ARM gcc toolchain if we are building for ARM
   # TODO(olonho): we need to invent more reliable way to get build
   # configuration info, to know if we're building for ARM.
-  if 'target_arch=arm' in os.environ.get('GYP_DEFINES', ''):
-      args.append('--arm-untrusted')
+  if 'target_arch=arm' not in os.environ.get('GYP_DEFINES', ''):
+      args.extend(['--exclude', 'nacl_arm_newlib'])
 
-  # Append the name of the file to use as a version and hash source.
-  # NOTE:  While not recommended, it is possible to redirect this file to
-  # a chrome location to avoid branching NaCl if just a toolchain needs
-  # to be bumped.
-  args.append(os.path.join(nacl_dir, 'TOOL_REVISIONS'))
-
-  download_toolchains.main(args)
+  args.append('sync')
+  args.append('--extract')
+  package_version.main(args)
   return 0
 
 

chrome/test/nacl_test_injection/buildbot_chrome_nacl_stage.py

@@ -225,9 +225,12 @@ def BuildAndTest(options):
                options.json_build_results_output_file)
 
   # Download the toolchain(s).
-  RunCommand([python,
-              os.path.join(nacl_dir, 'build', 'download_toolchains.py'),
-              '--keep', '--no-arm-trusted', '--no-pnacl', 'TOOL_REVISIONS'],
+  pkg_ver_dir = os.path.join(nacl_dir, 'build', 'package_version')
+  RunCommand([python, os.path.join(pkg_ver_dir, 'package_version.py'),
+              '--exclude', 'arm_trusted',
+              '--exclude', 'pnacl_newlib',
+              '--exclude', 'nacl_arm_newlib',
+              'sync', '--extract'],
              nacl_dir, os.environ)
 
   CleanTempDir()

native_client_sdk/src/build_tools/build_sdk.py

@@ -49,7 +49,12 @@
 import getos
 import oshelpers
 
-CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')
+BUILD_DIR = os.path.join(NACL_DIR, 'build')
+NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
+NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
+
+CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
+PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
 
 NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src'
 NACLPORTS_REV = 1226
@@ -58,29 +63,14 @@
 
 options = None
 
-
-def GetGlibcToolchain():
-  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
-  tcname = 'toolchain_%s_x86.tar.bz2' % getos.GetPlatform()
-  return os.path.join(tcdir, tcname)
-
-
-def GetNewlibToolchain():
-  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
-  tcname = 'naclsdk_%s_x86.tgz' % getos.GetPlatform()
-  return os.path.join(tcdir, tcname)
-
-
-def GetBionicToolchain():
-  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
-  tcname = 'naclsdk_%s_arm_bionic.tgz' % getos.GetPlatform()
-  return os.path.join(tcdir, tcname)
-
-
-def GetPNaClToolchain():
-  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
-  tcname = 'naclsdk_pnacl_%s_x86.tgz' % getos.GetPlatform()
-  return os.path.join(tcdir, tcname)
+  # Map of: ToolchainName: (PackageName, SDKDir).
+TOOLCHAIN_PACKAGE_MAP = {
+    'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
+    'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
+    'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
+    'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
+    'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
+    }
 
 
 def GetToolchainNaClInclude(tcname, tcpath, arch):
@@ -131,8 +121,14 @@ def GetToolchainDirName(tcname, xarch):
 
 
 def GetGypToolchainLib(tcname, xarch):
-  tcpath = os.path.join(GetGypGenDir(xarch), 'sdk', 'toolchain',
-                        GetToolchainDirName(tcname, xarch))
+  if xarch == 'arm':
+    toolchain = xarch
+  else:
+    toolchain = tcname
+
+  tcpath = os.path.join(GetGypGenDir(xarch), 'sdk',
+                        '%s_x86' % getos.GetPlatform(),
+                        TOOLCHAIN_PACKAGE_MAP[toolchain][0])
   return GetToolchainNaClLib(tcname, tcpath, xarch)
 
 
@@ -149,12 +145,12 @@ def GetPNaClNativeLib(tcpath, arch):
 
 
 def BuildStepDownloadToolchains(toolchains):
-  buildbot_common.BuildStep('Running download_toolchains.py')
-  download_script = os.path.join('build', 'download_toolchains.py')
-  args = [sys.executable, download_script, '--no-arm-trusted',
-          '--arm-untrusted', '--keep']
+  buildbot_common.BuildStep('Running package_version.py')
+  args = [sys.executable, PKGVER, '--exclude', 'arm_trusted']
   if 'bionic' in toolchains:
-    args.append('--allow-bionic')
+    build_platform = '%s_x86' % getos.GetPlatform()
+    args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')])
+  args.append('sync')
   buildbot_common.Run(args, cwd=NACL_DIR)
 
 
@@ -199,66 +195,39 @@ def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
 def BuildStepUntarToolchains(pepperdir, toolchains):
   buildbot_common.BuildStep('Untar Toolchains')
   platform = getos.GetPlatform()
+  build_platform = '%s_x86' % platform
   tmpdir = os.path.join(OUT_DIR, 'tc_temp')
   buildbot_common.RemoveDir(tmpdir)
   buildbot_common.MakeDir(tmpdir)
 
-  if 'newlib' in toolchains:
-    # Untar the newlib toolchains
-    tarfile = GetNewlibToolchain()
-    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
-                        cwd=NACL_DIR)
-
-    # Then rename/move it to the pepper toolchain directory
-    srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk')
-    tcname = platform + '_x86_newlib'
-    newlibdir = os.path.join(pepperdir, 'toolchain', tcname)
-    buildbot_common.Move(srcdir, newlibdir)
-
-  if 'bionic' in toolchains:
-    # Untar the bionic toolchains
-    tarfile = GetBionicToolchain()
-    tcname = platform + '_arm_bionic'
-    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
-                        cwd=NACL_DIR)
-    srcdir = os.path.join(tmpdir, tcname)
-    bionicdir = os.path.join(pepperdir, 'toolchain', tcname)
-    buildbot_common.Move(srcdir, bionicdir)
-
-  if 'arm' in toolchains:
-    # Copy the existing arm toolchain from native_client tree
-    tcname = platform + '_arm_newlib'
-    arm_toolchain = os.path.join(NACL_DIR, 'toolchain', '%s_x86' % platform,
-                                 'nacl_arm_newlib' )
-    arm_toolchain_sdk = os.path.join(pepperdir, 'toolchain', tcname)
-    buildbot_common.CopyDir(arm_toolchain, arm_toolchain_sdk)
-
-  if 'glibc' in toolchains:
-    # Untar the glibc toolchains
-    tarfile = GetGlibcToolchain()
-    tcname = platform + '_x86_glibc'
-    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
-                        cwd=NACL_DIR)
-
-    # Then rename/move it to the pepper toolchain directory
-    srcdir = os.path.join(tmpdir, 'toolchain', platform + '_x86')
-    glibcdir = os.path.join(pepperdir, 'toolchain', tcname)
-    buildbot_common.Move(srcdir, glibcdir)
-
-  # Untar the pnacl toolchains
-  if 'pnacl' in toolchains:
-    tmpdir = os.path.join(tmpdir, 'pnacl')
-    buildbot_common.RemoveDir(tmpdir)
-    buildbot_common.MakeDir(tmpdir)
-    tarfile = GetPNaClToolchain()
-    tcname = platform + '_pnacl'
-    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
-                        cwd=NACL_DIR)
-
-    # Then rename/move it to the pepper toolchain directory
-    pnacldir = os.path.join(pepperdir, 'toolchain', tcname)
-    buildbot_common.Move(tmpdir, pnacldir)
-
+  # Create a list of extract packages tuples, the first part should be
+  # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
+  # directory relative to pepperdir/toolchain.
+  extract_packages = []
+  for toolchain in toolchains:
+    toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None)
+    if toolchain_map:
+      package_name, tcname = toolchain_map
+      package_tuple = (os.path.join(build_platform, package_name),
+                       tcname % {'platform': platform})
+      extract_packages.append(package_tuple)
+
+  if extract_packages:
+    # Extract all of the packages into the temp directory.
+    package_names = [package_tuple[0] for package_tuple in extract_packages]
+    buildbot_common.Run([sys.executable, PKGVER,
+                           '--packages', ','.join(package_names),
+                           '--tar-dir', NACL_TOOLCHAINTARS_DIR,
+                           '--dest-dir', tmpdir,
+                           'extract'])
+
+    # Move all the packages we extracted to the correct destination.
+    for package_name, dest_dir in extract_packages:
+      full_src_dir = os.path.join(tmpdir, package_name)
+      full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir)
+      buildbot_common.Move(full_src_dir, full_dst_dir)
+
+  # Cleanup the temporary directory we are no longer using.
   buildbot_common.RemoveDir(tmpdir)
 
 

ppapi/native_client/src/untrusted/pnacl_support_extension/pnacl_support_extension.gyp

@@ -16,19 +16,17 @@
       'action_name': 'Untar pnacl_translator',
       'description': 'Untar pnacl_translator',
       'inputs': [
-        '<(DEPTH)/native_client/build/cygtar.py',
-        '<(DEPTH)/native_client/toolchain/.tars/naclsdk_pnacl_translator.tgz',
+        '<(DEPTH)/native_client/build/package_version/package_version.py',
+        '<(DEPTH)/native_client/toolchain/.tars/<(OS)_x86/pnacl_translator.json',
       ],
-      'outputs': ['<(SHARED_INTERMEDIATE_DIR)/pnacl_translator/stamp.untar'],
+      'outputs': ['<(SHARED_INTERMEDIATE_DIR)/<(OS)_x86/pnacl_translator/pnacl_translator.json'],
       'action': [
         'python',
-        '<(DEPTH)/native_client/build/untar_toolchain.py',
-        '--tool', 'pnacl',
-        '--tmp', '<(SHARED_INTERMEDIATE_DIR)/temp/untar',
-        '--sdk', '<(SHARED_INTERMEDIATE_DIR)/temp/sdk',
-        '--os', '<(OS)',
-        '--fin', '<(SHARED_INTERMEDIATE_DIR)/pnacl_translator',
-        '<(DEPTH)/native_client/toolchain/.tars/naclsdk_pnacl_translator.tgz',
+        '<(DEPTH)/native_client/build/package_version/package_version.py',
+        '--packages', 'pnacl_translator',
+        '--tar-dir', '<(DEPTH)/native_client/toolchain/.tars',
+        '--dest-dir', '<(SHARED_INTERMEDIATE_DIR)',
+        'extract',
       ],
     }],
   },
@@ -53,7 +51,7 @@
             'inputs': [
               'pnacl_component_crx_gen.py',
               # A stamp file representing the contents of pnacl_translator.
-              '<(SHARED_INTERMEDIATE_DIR)/pnacl_translator/stamp.untar',
+              '<(SHARED_INTERMEDIATE_DIR)/<(OS)_x86/pnacl_translator/pnacl_translator.json',
               '<(DEPTH)/native_client/pnacl/driver/pnacl_info_template.json',
               '<(DEPTH)/native_client/TOOL_REVISIONS',
             ],
@@ -185,7 +183,7 @@
               '<@(lib_overrides)',
               '--target_arch=<(target_arch)',
               '--info_template_path=<(DEPTH)/native_client/pnacl/driver/pnacl_info_template.json',
-              '--pnacl_translator_path=<(SHARED_INTERMEDIATE_DIR)/pnacl_translator',
+              '--pnacl_translator_path=<(SHARED_INTERMEDIATE_DIR)/<(OS)_x86/pnacl_translator',
               '--tool_revisions_path=<(DEPTH)/native_client/TOOL_REVISIONS',
               # ABI Version Number.
               '1',