Add Google Cloud Storage support to the documentation AppEngine serve…

…r using Cloud Storage official Python client library.

With this CL, URL paths can be directly mapped to Cloud Storage buckets (http://developer.chrome.com/path/... -> gs://bucket/...) using content_storage.json configuration file.

open-source-thrid-party-reviews@ team: this CL adds a third-party library to Chromium repo. The library will be used only in the documentation server and it is the official Google library to access Google Cloud Storage from a Google AppEngine application (docs server).

BUG=338007

Review URL: https://codereview.chromium.org/139303023

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@250663 0039d316-1c4b-4281-b951-d872f2087c98

.gitignore

@@ -220,6 +220,7 @@ v8.log
 /third_party/gles2_conform
 /third_party/gnu_binutils/
 /third_party/gold
+/third_party/google_appengine_cloudstorage
 /third_party/google_toolbox_for_mac/src
 /third_party/googlemac
 /third_party/gperf

chrome/common/extensions/docs/server2/.gitignore

@@ -1 +1,2 @@
 third_party/
+local_debug/

chrome/common/extensions/docs/server2/README

@@ -21,6 +21,33 @@ be sufficient. If for some reason you want to test against the app engine SDK:
   3. View docs at http://localhost:8080/(apps|extensions)/<doc_name>
 
 
+--------------------------------------------
+Using Google Cloud Storage content providers
+
+With preview.py:
+
+  1. create a directory "[...]/server2/local_debug/gcs/<bucketname>" for every
+     gcs bucket referenced in content_providers.json
+
+  2. copy files to the respective local bucket directories. Preview.py has
+     no access to the real Google Cloud Storage.
+
+With start_dev_server.py:
+
+    1. Install gsutils from https://developers.google.com/storage/docs/gsutil
+
+    2. Set gsutil accordingly to the official instructions.
+
+    3. Make sure you have permission to the GCS buckets specified in
+       content_providers.json by running "gsutil ls gs://bucketname"
+
+    4. Get an oauth token (see instructions at the comment of
+       gcs_file_system_provider.py) and save it to the file
+       "[...]/server2/local_debug/gcs_debug.conf"
+
+Remember that the step 4 needs to be repeated every 10 minutes or so,
+because the oauth access token expires quickly.
+
 --------------------
 Deploying the Server
 

chrome/common/extensions/docs/server2/app.yaml

@@ -1,5 +1,5 @@
 application: chrome-apps-doc
-version: 3-6-0
+version: 3-7-0
 runtime: python27
 api_version: 1
 threadsafe: false

chrome/common/extensions/docs/server2/app_engine_handler.py

@@ -43,4 +43,4 @@ def get(self):
         self.response.status = response.status
       else:
         self.response.out.write('Internal server error')
-        self.response.status = 500
+        self.response.status = 500

chrome/common/extensions/docs/server2/build_server.py

@@ -75,6 +75,9 @@ def main():
                  make_init=False)
   MakeInit(LOCAL_THIRD_PARTY_DIR)
 
+  CopyThirdParty(os.path.join(THIRD_PARTY_DIR, 'google_appengine_cloudstorage',
+                 'cloudstorage'), 'cloudstorage')
+
   # To be able to use the Handlebar class we need this import in __init__.py.
   with open(os.path.join(LOCAL_THIRD_PARTY_DIR,
                          'handlebar',

chrome/common/extensions/docs/server2/content_providers.py

@@ -3,12 +3,15 @@
 # found in the LICENSE file.
 
 import logging
+import os
 import traceback
 
 from chroot_file_system import ChrootFileSystem
 from content_provider import ContentProvider
-from extensions_paths import CONTENT_PROVIDERS
+import environment
+from extensions_paths import CONTENT_PROVIDERS, LOCAL_DEBUG_DIR
 from future import Gettable, Future
+from local_file_system import LocalFileSystem
 from third_party.json_schema_compiler.memoize import memoize
 
 
@@ -39,11 +42,32 @@ class ContentProviders(object):
   def __init__(self,
                compiled_fs_factory,
                host_file_system,
-               github_file_system_provider):
+               github_file_system_provider,
+               gcs_file_system_provider):
     self._compiled_fs_factory = compiled_fs_factory
     self._host_file_system = host_file_system
     self._github_file_system_provider = github_file_system_provider
-    self._cache = compiled_fs_factory.ForJson(host_file_system)
+    self._gcs_file_system_provider = gcs_file_system_provider
+    self._cache = None
+
+    # If running the devserver and there is a LOCAL_DEBUG_DIR, we
+    # will read the content_provider configuration from there instead
+    # of fetching it from SVN trunk or patch.
+    if environment.IsDevServer() and os.path.exists(LOCAL_DEBUG_DIR):
+      local_fs = LocalFileSystem(LOCAL_DEBUG_DIR)
+      conf_stat = None
+      try:
+        conf_stat = local_fs.Stat(CONTENT_PROVIDERS)
+      except:
+        pass
+
+      if conf_stat:
+        logging.warn(("Using local debug folder (%s) for "
+                      "content_provider.json configuration") % LOCAL_DEBUG_DIR)
+        self._cache = compiled_fs_factory.ForJson(local_fs)
+
+    if not self._cache:
+      self._cache = compiled_fs_factory.ForJson(host_file_system)
 
   @memoize
   def GetByName(self, name):
@@ -94,6 +118,20 @@ def _CreateContentProvider(self, name, config):
         return None
       file_system = ChrootFileSystem(self._host_file_system,
                                      chromium_config['dir'])
+    elif 'gcs' in config:
+      gcs_config = config['gcs']
+      if 'bucket' not in gcs_config:
+        logging.error('%s: "gcs" must have a "bucket" property' % name)
+        return None
+      bucket = gcs_config['bucket']
+      if not bucket.startswith('gs://'):
+        logging.error('%s: bucket %s should start with gs://' % (name, bucket))
+        return None
+      bucket = bucket[len('gs://'):]
+      file_system = self._gcs_file_system_provider.Create(bucket)
+      if 'dir' in gcs_config:
+        file_system = ChrootFileSystem(file_system, gcs_config['dir'])
+
     elif 'github' in config:
       github_config = config['github']
       if 'owner' not in github_config or 'repo' not in github_config:
@@ -103,9 +141,9 @@ def _CreateContentProvider(self, name, config):
           github_config['owner'], github_config['repo'])
       if 'dir' in github_config:
         file_system = ChrootFileSystem(file_system, github_config['dir'])
+
     else:
-      logging.error(
-          '%s: content provider type "%s" not supported' % (name, type_))
+      logging.error('%s: content provider type not supported' % name)
       return None
 
     return ContentProvider(name,

chrome/common/extensions/docs/server2/content_providers_test.py

@@ -9,6 +9,7 @@
 from compiled_file_system import CompiledFileSystem
 from content_providers import ContentProviders
 from extensions_paths import EXTENSIONS
+from gcs_file_system_provider import CloudStorageFileSystemProvider
 from object_store_creator import ObjectStoreCreator
 from test_file_system import TestFileSystem
 from test_util import DisableLogging
@@ -102,10 +103,14 @@ class ContentProvidersTest(unittest.TestCase):
   def setUp(self):
     test_file_system = TestFileSystem(_FILE_SYSTEM_DATA, relative_to=EXTENSIONS)
     self._github_fs_provider = _MockGithubFileSystemProvider(test_file_system)
+    object_store_creator = ObjectStoreCreator.ForTest()
+    # TODO(mangini): create tests for GCS
+    self._gcs_fs_provider = CloudStorageFileSystemProvider(object_store_creator)
     self._content_providers = ContentProviders(
-        CompiledFileSystem.Factory(ObjectStoreCreator.ForTest()),
+        CompiledFileSystem.Factory(object_store_creator),
         test_file_system,
-        self._github_fs_provider)
+        self._github_fs_provider,
+        self._gcs_fs_provider)
 
   def testSimpleRootPath(self):
     provider = self._content_providers.GetByName('apples')

chrome/common/extensions/docs/server2/cron.yaml

@@ -2,4 +2,4 @@ cron:
 - description: Repopulates all cached data.
   url: /_cron
   schedule: every 5 minutes
-  target: 3-6-0
+  target: 3-7-0

chrome/common/extensions/docs/server2/cron_servlet.py

@@ -15,6 +15,7 @@
 from extensions_paths import EXAMPLES, PUBLIC_TEMPLATES, STATIC_DOCS
 from file_system_util import CreateURLsFromPaths
 from future import Gettable, Future
+from gcs_file_system_provider import CloudStorageFileSystemProvider
 from github_file_system_provider import GithubFileSystemProvider
 from host_file_system_provider import HostFileSystemProvider
 from object_store_creator import ObjectStoreCreator
@@ -101,6 +102,9 @@ def CreateHostFileSystemProvider(self,
     def CreateGithubFileSystemProvider(self, object_store_creator):
       return GithubFileSystemProvider(object_store_creator)
 
+    def CreateGCSFileSystemProvider(self, object_store_creator):
+      return CloudStorageFileSystemProvider(object_store_creator)
+
     def GetAppVersion(self):
       return GetAppVersion()
 
@@ -287,8 +291,11 @@ def _CreateServerInstance(self, revision):
         object_store_creator, max_trunk_revision=revision)
     github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
         object_store_creator)
+    gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider(
+        object_store_creator)
     return ServerInstance(object_store_creator,
                           CompiledFileSystem.Factory(object_store_creator),
                           branch_utility,
                           host_file_system_provider,
-                          github_file_system_provider)
+                          github_file_system_provider,
+                          gcs_file_system_provider)

chrome/common/extensions/docs/server2/cron_servlet_test.py

@@ -13,6 +13,7 @@
 from extensions_paths import (
     APP_YAML, CONTENT_PROVIDERS, EXTENSIONS, PUBLIC_TEMPLATES, SERVER2,
     STATIC_DOCS)
+from gcs_file_system_provider import CloudStorageFileSystemProvider
 from github_file_system_provider import GithubFileSystemProvider
 from host_file_system_provider import HostFileSystemProvider
 from local_file_system import LocalFileSystem
@@ -50,6 +51,9 @@ def constructor(branch=None, revision=None):
   def CreateGithubFileSystemProvider(self, object_store_creator):
     return GithubFileSystemProvider.ForEmpty()
 
+  def CreateGCSFileSystemProvider(self, object_store_creator):
+    return CloudStorageFileSystemProvider.ForEmpty()
+
   def GetAppVersion(self):
     return self._app_version
 

chrome/common/extensions/docs/server2/extensions_paths.py

@@ -30,3 +30,7 @@
 PUBLIC_TEMPLATES = join(TEMPLATES, 'public/')
 
 CONTENT_PROVIDERS = join(JSON_TEMPLATES, 'content_providers.json')
+
+LOCAL_DEBUG_DIR = join(SERVER2, 'local_debug/')
+LOCAL_GCS_DIR = join(LOCAL_DEBUG_DIR, 'gcs/')
+LOCAL_GCS_DEBUG_CONF = join(LOCAL_DEBUG_DIR, 'gcs_debug.conf')

chrome/common/extensions/docs/server2/gcs_file_system.py

@@ -0,0 +1,120 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from third_party.cloudstorage import cloudstorage_api
+from third_party.cloudstorage import common
+from third_party.cloudstorage import errors
+
+from docs_server_utils import StringIdentity
+from file_system import FileSystem, FileNotFoundError, StatInfo
+from future import Gettable, Future
+
+import logging
+import traceback
+
+'''See gcs_file_system_provider.py for documentation on using Google Cloud
+Storage as a filesystem.
+'''
+def _ReadFile(filename):
+  try:
+    with cloudstorage_api.open(filename, 'r') as f:
+      return f.read()
+  except errors.Error:
+    raise FileNotFoundError('Read failed for %s: %s' % (filename,
+        traceback.format_exc()))
+
+def _ListDir(dir_name):
+  try:
+    files = cloudstorage_api.listbucket(dir_name)
+    return [os_path.filename for os_path in files]
+  except errors.Error:
+    raise FileNotFoundError('cloudstorage.listbucket failed for %s: %s' %
+                            (dir_name, traceback.format_exc()))
+
+def _CreateStatInfo(bucket, path):
+  bucket = '/%s' % bucket
+  full_path = '/'.join( (bucket, path.lstrip('/')) )
+  try:
+    if full_path.endswith('/'):
+      child_versions = dict()
+      version = 0
+      # Fetching stats for all files under full_path, recursively. The
+      # listbucket method uses a prefix approach to simulate hierarchy,
+      # but calling it without the "delimiter" argument searches for prefix,
+      # which means, for directories, everything beneath it.
+      for _file in cloudstorage_api.listbucket(full_path):
+        if not _file.is_dir:
+          # GCS doesn't have metadata for dirs
+          child_stat = cloudstorage_api.stat('%s' % _file.filename).st_ctime
+          filename = _file.filename[len(bucket)+1:]
+          child_versions[filename] = child_stat
+          version = max(version, child_stat)
+    else:
+      child_versions = None
+      version = cloudstorage_api.stat(full_path).st_ctime
+    return StatInfo(version, child_versions)
+  except (TypeError, errors.Error):
+    raise FileNotFoundError('cloudstorage.stat failed for %s: %s' % (path,
+                            traceback.format_exc()))
+
+
+class CloudStorageFileSystem(FileSystem):
+  '''FileSystem implementation which fetches resources from Google Cloud
+  Storage.
+  '''
+  def __init__(self, bucket, debug_access_token=None, debug_bucket_prefix=None):
+    self._bucket = bucket
+    if debug_access_token:
+      logging.debug('gcs: using debug access token: %s' % debug_access_token)
+      common.set_access_token(debug_access_token)
+    if debug_bucket_prefix:
+      logging.debug('gcs: prefixing all bucket names with %s' %
+                    debug_bucket_prefix)
+      self._bucket = debug_bucket_prefix + self._bucket
+
+  def Read(self, paths):
+    def resolve():
+      try:
+        result = {}
+        for path in paths:
+          full_path = '/%s/%s' % (self._bucket, path.lstrip('/'))
+          logging.debug('gcs: requested path %s, reading %s' %
+                        (path, full_path))
+          if path == '' or path.endswith('/'):
+            result[path] = _ListDir(full_path)
+          else:
+            result[path] = _ReadFile(full_path)
+        return result
+      except errors.AuthorizationError:
+        self._warnAboutAuthError()
+        raise
+
+    return Future(delegate=Gettable(resolve))
+
+  def Refresh(self):
+    return Future(value=())
+
+  def Stat(self, path):
+    try:
+      return _CreateStatInfo(self._bucket, path)
+    except errors.AuthorizationError:
+      self._warnAboutAuthError()
+      raise
+
+  def GetIdentity(self):
+    return '@'.join((self.__class__.__name__, StringIdentity(self._bucket)))
+
+  def __repr__(self):
+    return 'LocalFileSystem(%s)' % self._bucket
+
+  def _warnAboutAuthError(self):
+    logging.warn(('Authentication error on Cloud Storage. Check if your'
+                  ' appengine project has permissions to Read the GCS'
+                  ' buckets. If you are running a local appengine server,'
+                  ' you need to set an access_token in'
+                  ' local_debug/gcs_debug.conf.'
+                  ' Remember that this token expires in less than 10'
+                  ' minutes, so keep it updated. See'
+                  ' gcs_file_system_provider.py for instructions.'));
+    logging.debug(traceback.format_exc())