Merge pull request tensorflow#5054 from drpngx/branch_136515806

Branch 136515806

tensorflow/BUILD

@@ -122,10 +122,6 @@ filegroup(
         "//tensorflow/contrib/metrics/kernels:all_files",
         "//tensorflow/contrib/ndlstm:all_files",
         "//tensorflow/contrib/opt:all_files",
-        "//tensorflow/contrib/quantization:all_files",
-        "//tensorflow/contrib/quantization/kernels:all_files",
-        "//tensorflow/contrib/quantization/kernels/hexagon:all_files",
-        "//tensorflow/contrib/quantization/tools:all_files",
         "//tensorflow/contrib/rnn:all_files",
         "//tensorflow/contrib/seq2seq:all_files",
         "//tensorflow/contrib/session_bundle:all_files",
@@ -148,6 +144,7 @@ filegroup(
         "//tensorflow/core/distributed_runtime:all_files",
         "//tensorflow/core/distributed_runtime/rpc:all_files",
         "//tensorflow/core/kernels:all_files",
+        "//tensorflow/core/kernels/hexagon:all_files",
         "//tensorflow/core/ops/compat:all_files",
         "//tensorflow/core/platform/cloud:all_files",
         "//tensorflow/core/platform/default/build_config:all_files",
@@ -195,6 +192,7 @@ filegroup(
         "//tensorflow/tools/docs:all_files",
         "//tensorflow/tools/git:all_files",
         "//tensorflow/tools/proto_text:all_files",
+        "//tensorflow/tools/quantization:all_files",
         "//tensorflow/tools/test:all_files",
         "//tensorflow/user_ops:all_files",
         "//third_party/hadoop:all_files",

tensorflow/c/c_api.h

@@ -89,6 +89,7 @@ typedef enum {
   TF_UINT16 = 17,
   TF_COMPLEX128 = 18,  // Double-precision complex
   TF_HALF = 19,
+  TF_RESOURCE = 20,
 } TF_DataType;
 
 // --------------------------------------------------------------------------

tensorflow/c/checkpoint_reader.cc

@@ -14,10 +14,14 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/c/checkpoint_reader.h"
+
+#include <unordered_set>
+
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/saved_tensor_slice_util.h"
 
 namespace tensorflow {
 
@@ -98,15 +102,34 @@ void CheckpointReader::GetTensor(
 TensorSliceReader::VarToShapeMap* CheckpointReader::BuildV2VarToShapeMap() {
   CHECK(v2_reader_ != nullptr);
   CHECK(v2_reader_->status().ok());
+
+  // First pass: filters out the entries of the slices.
+  std::unordered_set<string> filtered_keys;
+  BundleEntryProto entry;
   v2_reader_->Seek(kHeaderEntryKey);
+  for (v2_reader_->Next(); v2_reader_->Valid(); v2_reader_->Next()) {
+    CHECK(entry.ParseFromArray(v2_reader_->value().data(),
+                               v2_reader_->value().size()))
+        << entry.InitializationErrorString();
+    for (int i = 0; i < entry.slices_size(); ++i) {
+      const auto& slice_proto = entry.slices(i);
+      CHECK(filtered_keys
+                .insert(EncodeTensorNameSlice(
+                    v2_reader_->key().ToString() /* full var's name */,
+                    TensorSlice(slice_proto)))
+                .second);
+    }
+  }
 
+  // Second pass: adds the entries, ignoring the filtered keys.
   TensorSliceReader::VarToShapeMap* var_to_shape_map =
       new TensorSliceReader::VarToShapeMap;
-  BundleEntryProto entry;
+  v2_reader_->Seek(kHeaderEntryKey);
   for (v2_reader_->Next(); v2_reader_->Valid(); v2_reader_->Next()) {
+    if (filtered_keys.count(v2_reader_->key().ToString()) > 0) continue;
     CHECK(entry.ParseFromArray(v2_reader_->value().data(),
-                               v2_reader_->value().size()));
-    if (entry.slices_size() > 0) continue;  // Slice of some partitioned var.
+                               v2_reader_->value().size()))
+        << entry.InitializationErrorString();
     (*var_to_shape_map)[v2_reader_->key().ToString()] =
         TensorShape(entry.shape());
   }

tensorflow/c/checkpoint_reader.h

@@ -43,6 +43,8 @@ class CheckpointReader {
   bool HasTensor(const string& name) const;
   const string DebugString() const;
 
+  // Returns a map from variable namaes to its shape.  Slices of a partitioned
+  // tensor are combined into a single entry.
   const TensorSliceReader::VarToShapeMap& GetVariableToShapeMap() const;
 
   // Attempts to look up the tensor named "name" and stores the found result in

tensorflow/cc/saved_model/BUILD

@@ -31,6 +31,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
+        "//tensorflow/core/util/tensor_bundle:naming",
     ],
 )
 

tensorflow/cc/saved_model/constants.h

@@ -36,10 +36,6 @@ constexpr char kSavedModelVariablesDirectory[] = "variables";
 // SavedModel variables filename.
 constexpr char kSavedModelVariablesFilename[] = "variables";
 
-// SavedModel sharded variables filename.
-constexpr char kSavedModelVariablesShardedFilename[] =
-    "variables-\?\?\?\?\?-of-\?\?\?\?\?";
-
 // Commonly used tags.
 constexpr char kSavedModelTagServe[] = "serve";
 constexpr char kSavedModelTagTrain[] = "train";

tensorflow/cc/saved_model/loader.cc

@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/protobuf/saved_model.pb.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/util/tensor_bundle/naming.h"
 
 namespace tensorflow {
 namespace {
@@ -87,17 +88,20 @@ Status Restore(const RunOptions& run_options, const string& export_dir,
                const StringPiece variable_filename_const_op_name,
                Session* session) {
   // Find path to variables to be restored in export directory.
-  string variables_path =
+  const string variables_directory =
       io::JoinPath(export_dir, kSavedModelVariablesDirectory);
-  const string unsharded_variables_path =
-      io::JoinPath(variables_path, kSavedModelVariablesFilename);
-  if (Env::Default()->FileExists(unsharded_variables_path)) {
-    variables_path = unsharded_variables_path;
-  } else {
-    const string sharded_variables_path =
-        io::JoinPath(variables_path, kSavedModelVariablesShardedFilename);
-    variables_path = sharded_variables_path;
+  // Check for saver checkpoints in v2 format. Models exported in the checkpoint
+  // v2 format will have a variables.index file. The corresponding
+  // variables are stored in the variables.data-?????-of-????? files.
+  const string variables_index_path = io::JoinPath(
+      variables_directory, MetaFilename(kSavedModelVariablesFilename));
+  if (!Env::Default()->FileExists(variables_index_path)) {
+    return errors::NotFound(
+        "Checkpoint index file not found in SavedModel directory.");
   }
+  const string variables_path =
+      io::JoinPath(variables_directory, kSavedModelVariablesFilename);
+
   // Add variables to the graph.
   Tensor variables_path_tensor(DT_STRING, TensorShape({}));
   variables_path_tensor.scalar<string>()() = variables_path;

tensorflow/contrib/android/jni/jni_utils.h

@@ -20,6 +20,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/types.h"
 

tensorflow/contrib/cmake/CMakeLists.txt

@@ -61,6 +61,7 @@ include(gif)
 include(png)
 include(jpeg)
 include(eigen)
+include(gemmlowp)
 include(jsoncpp)
 include(farmhash)
 include(highwayhash)
@@ -89,6 +90,7 @@ include_directories(
     ${png_INCLUDE_DIR}
     ${jpeg_INCLUDE_DIR}
     ${eigen_INCLUDE_DIRS}
+    ${gemmlowp_INCLUDE_DIR}
     ${jsoncpp_INCLUDE_DIR}
     ${farmhash_INCLUDE_DIR}
     ${highwayhash_INCLUDE_DIR}

tensorflow/contrib/cmake/external/gemmlowp.cmake

@@ -0,0 +1,15 @@
+include (ExternalProject)
+
+set(gemmlowp_URL http://github.com/google/gemmlowp/archive/c0bacf11fb509a2cbe15a97362a2df067ffd57a2.tar.gz)
+set(gemmlowp_HASH SHA256=dc64a38f9927db18748d9024987c9b102115e25bc2be4b76aa8e422b8f83d882)
+set(gemmlowp_BUILD ${CMAKE_BINARY_DIR}/gemmlowp/src/gemmlowp)
+set(gemmlowp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/gemmlowp/src/gemmlowp)
+
+ExternalProject_Add(gemmlowp
+    PREFIX gemmlowp
+    URL ${gemmlowp_URL}
+    URL_HASH ${gemmlowp_HASH}
+    DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+    BUILD_IN_SOURCE 1
+    PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/patches/gemmlowp/CMakeLists.txt ${gemmlowp_BUILD}
+    INSTALL_COMMAND "")

tensorflow/contrib/cmake/patches/gemmlowp/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 2.8.3)
+
+project(gemmlowp)

tensorflow/contrib/cmake/tf_core_framework.cmake

@@ -92,6 +92,7 @@ set(tf_proto_text_srcs
     "tensorflow/core/framework/log_memory.proto"
     "tensorflow/core/framework/node_def.proto"
     "tensorflow/core/framework/op_def.proto"
+    "tensorflow/core/framework/resource_handle.proto"
     "tensorflow/core/framework/step_stats.proto"
     "tensorflow/core/framework/summary.proto"
     "tensorflow/core/framework/tensor.proto"

tensorflow/contrib/cmake/tf_core_kernels.cmake

@@ -90,6 +90,8 @@ if(WIN32)
       "${tensorflow_source_dir}/tensorflow/core/kernels/immutable_constant_op.h"
       "${tensorflow_source_dir}/tensorflow/core/kernels/sparse_matmul_op.cc"
       "${tensorflow_source_dir}/tensorflow/core/kernels/sparse_matmul_op.h"
+      "${tensorflow_source_dir}/tensorflow/core/kernels/*quantiz*.h"
+      "${tensorflow_source_dir}/tensorflow/core/kernels/*quantiz*.cc"
   )
   list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_windows_exclude_srcs})
 endif(WIN32)

tensorflow/contrib/distributions/BUILD

@@ -122,7 +122,6 @@ cuda_py_tests(
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:platform_test",
     ],
-    tags = ["notsan"],  #http://b/31216497
 )
 
 cuda_py_tests(
@@ -349,7 +348,6 @@ cuda_py_tests(
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:platform_test",
     ],
-    tags = ["notsan"],  #http://b/31455709
 )
 
 cuda_py_tests(