Merge pull request clab#59 from allenai/syncy

sync with upstream

doc/source/code_style.rst

@@ -48,7 +48,16 @@ computation graph construction.
   or small enough (memory-wise) or in other ways cheap enough (time-wise) to
   copy.
 
-**Throwing Exceptions:** When the user does something illegal, throw an
-exception. "assert" should never be used for something that might be triggered
-by a user. (As `noted <https://github.com/clab/dynet/issues/139>`_)
-
+**Error handling:** The C++ core of DyNet provides a mechanism for error handling that
+should be used in all code. It consists of 3 macros as follows (included in ``globals.h``):
+
+* ``DYNET_INVALID_ARG(msg)``: This is used to throw an error that is triggered when
+  a user passes an invalid argument to one of the functions.
+* ``DYNET_RUNTIME_ERR(msg)``: This is used to throw an error that could be triggered
+  by a user, but is not the result of an invalid argument. For example, it could be
+  used when something is not implemented yet, or when the program dies due to lack
+  of memory, etc.
+* ``DYNET_ASSERT(expr,msg)``: This is to be used to check things that should only
+  happen due to a programming error within DyNet itself, and should never be
+  triggered by a user. ``expr`` is a condition, and ``msg`` is a message explaining
+  the exception, with ``ostream``-style formatting.

dynet/cfsm-builder.cc

@@ -1,4 +1,5 @@
 #include "dynet/cfsm-builder.h"
+#include "dynet/except.h"
 
 #include <fstream>
 #include <iostream>
@@ -91,9 +92,10 @@ void ClassFactoredSoftmaxBuilder::new_graph(ComputationGraph& cg) {
 }
 
 Expression ClassFactoredSoftmaxBuilder::neg_log_softmax(const Expression& rep, unsigned wordidx) {
-  // TODO assert that new_graph has been called
+  // TODO check that new_graph has been called
   int clusteridx = widx2cidx[wordidx];
-  assert(clusteridx >= 0);  // if this fails, wordid is missing from clusters
+  if(clusteridx < 0)
+    DYNET_INVALID_ARG("Word ID " << wordidx << " missing from clusters in ClassFactoredSoftmaxBuilder::neg_log_softmax");
   Expression cscores = affine_transform({cbias, r2c, rep});
   Expression cnlp = pickneglogsoftmax(cscores, clusteridx);
   if (singleton_cluster[clusteridx]) return cnlp;
@@ -108,7 +110,7 @@ Expression ClassFactoredSoftmaxBuilder::neg_log_softmax(const Expression& rep, u
 }
 
 unsigned ClassFactoredSoftmaxBuilder::sample(const Expression& rep) {
-  // TODO assert that new_graph has been called
+  // TODO check that new_graph has been called
   Expression cscores = affine_transform({cbias, r2c, rep});
   Expression cdist_expr = softmax(cscores);
   auto cdist = as_vector(pcg->incremental_forward(cdist_expr));
@@ -174,7 +176,8 @@ Expression ClassFactoredSoftmaxBuilder::full_log_distribution(const Expression&
 void ClassFactoredSoftmaxBuilder::read_cluster_file(const std::string& cluster_file, Dict& word_dict) {
   cerr << "Reading clusters from " << cluster_file << " ...\n";
   ifstream in(cluster_file);
-  assert(in);
+  if(!in)
+    DYNET_INVALID_ARG("Could not find cluster file " << cluster_file << " in ClassFactoredSoftmax");
   int wc = 0;
   string line;
   while(getline(in, line)) {
@@ -188,9 +191,8 @@ void ClassFactoredSoftmaxBuilder::read_cluster_file(const std::string& cluster_f
     while (is_ws(line[startw]) && startw < len) { ++startw; }
     unsigned endw = startw;
     while (not_ws(line[endw]) && endw < len) { ++endw; }
-    assert(endc > startc);
-    assert(startw > endc);
-    assert(endw > startw);
+    if(endc <= startc || startw <= endc || endw <= startw)
+      DYNET_INVALID_ARG("Invalid format in cluster file " << cluster_file << " in ClassFactoredSoftmax");
     unsigned c = cdict.convert(line.substr(startc, endc - startc));
     unsigned word = word_dict.convert(line.substr(startw, endw - startw));
     if (word >= widx2cidx.size()) {

dynet/cuda.h

@@ -3,7 +3,6 @@
 #if HAVE_CUDA
 
 #include <vector>
-#include <cassert>
 #include <utility>
 #include <stdexcept>
 #include <cuda.h>
@@ -12,7 +11,6 @@
 
 #include "dynet/except.h"
 
-
 #define MAX_GPUS 256
 
 #define CUDA_CHECK(stmt) do {                              \
@@ -43,7 +41,7 @@ struct DynetParams;
 class Device;
 
 inline std::pair<int, int> SizeToBlockThreadPair(int n) {
-  assert(n > 0);
+  DYNET_ASSERT(n > 0, "Bad thread size in GPU code " << n);
   int logn;
 #if defined(_MSC_VER)
   logn = 0;

dynet/deep-lstm.cc

@@ -1,7 +1,6 @@
 #include "dynet/deep-lstm.h"
 
 #include <string>
-#include <cassert>
 #include <vector>
 #include <iostream>
 
@@ -75,7 +74,8 @@ void DeepLSTMBuilder::start_new_sequence_impl(const vector<Expression>& hinit) {
   h.clear();
   c.clear();
   if (hinit.size() > 0) {
-    assert(layers*2 == hinit.size());
+    if(layers * 2 != hinit.size())
+      DYNET_INVALID_ARG("DeepLSTMBuilder must be initialized with 2 times as many expressions as layers (hidden state and cell for each layer). However, for " << layers << " layers, " << hinit.size() << " expressions were passed in");
     h0.resize(layers);
     c0.resize(layers);
     for (unsigned i = 0; i < layers; ++i) {

dynet/devices.cc

@@ -38,8 +38,7 @@ DeviceMempoolSizes::DeviceMempoolSizes(const std::string & descriptor) {
     used[1] = stoi(strs[1]);
     used[2] = stoi(strs[2]);
   } else {
-    ostringstream s; s << "the format of --dynet-mem is invalid:" << descriptor;
-    throw std::invalid_argument(s.str());
+    DYNET_INVALID_ARG("the format of --dynet-mem is invalid: " << descriptor);
   }
 }
 
@@ -52,17 +51,20 @@ DeviceMempoolSizes Device::mark(ComputationGraph *cg) {
 }
 
 void Device::revert(const DeviceMempoolSizes & cp) {
-  assert(cp.used[0] <= pools[0]->used);
+  if(cp.used[0] > pools[0]->used)
+    DYNET_INVALID_ARG("Saved value greater than original value in Device::revert (" << cp.used[0] << " > " << pools[0]->used << ")");
   pools[0]->used = cp.used[0];
-  assert(cp.used[1] <= pools[1]->used);
+  if(cp.used[1] > pools[1]->used)
+    DYNET_INVALID_ARG("Saved value greater than original value in Device::revert (" << cp.used[1] << " > " << pools[1]->used << ")");
   pools[1]->used = cp.used[1];
-  assert(cp.used[2] <= pools[2]->used);
+  if(cp.used[2] > pools[2]->used)
+    DYNET_INVALID_ARG("Saved value greater than original value in Device::revert (" << cp.used[2] << " > " << pools[2]->used << ")");
   pools[2]->used = cp.used[2];
 }
 
 void Device::allocate_tensor(DeviceMempool mp, Tensor & tens) {
-  assert(mp != DeviceMempool::NONE);
-  assert(pools[(int)mp] != nullptr);
+  DYNET_ASSERT(mp != DeviceMempool::NONE, "Attempt to allocate tensor for NONE DeviceMempool");
+  DYNET_ASSERT(pools[(int)mp] != nullptr, "Attempt to allocate tensor for null DeviceMempool");
   tens.v = (float*)pools[(int)mp]->allocate(tens.d.size() * sizeof(float));
   tens.mem_pool = mp;
 }

dynet/dict.h

@@ -1,14 +1,14 @@
 #ifndef DYNET_DICT_H_
 #define DYNET_DICT_H_
 
-#include <cassert>
 #include <unordered_map>
 #include <string>
 #include <vector>
 #include <iostream>
 #include <stdexcept>
 
 #include "dynet/io-macros.h"
+#include "dynet/except.h"
 
 namespace boost { namespace serialization { class access; } }
 
@@ -33,14 +33,10 @@ typedef std::unordered_map<std::string, int> Map;
     auto i = d_.find(word);
     if (i == d_.end()) {
       if (frozen) {
-        if (map_unk) {
+        if (map_unk)
           return unk_id;
-        }
-        else {
-         std::cerr << map_unk << std::endl;
-          std::cerr << "Unknown word encountered: " << word << std::endl;
-          throw std::runtime_error("Unknown word encountered in frozen dictionary: " + word);
-        }
+        else
+          DYNET_RUNTIME_ERR("Unknown word encountered in frozen dictionary: " << word);
       }
       words_.push_back(word);
       return d_[word] = words_.size() - 1;
@@ -50,15 +46,16 @@ typedef std::unordered_map<std::string, int> Map;
   }
 
   inline const std::string& convert(const int& id) const {
-    assert(id < (int)words_.size());
+    if(id >= (int)words_.size())
+      DYNET_INVALID_ARG("Out-of-bounds error in Dict::convert for word ID " << id << " (dict size: " << words_.size() << ")");
     return words_[id];
   }
 
   void set_unk(const std::string& word) {
     if (!frozen)
-      throw std::runtime_error("Please call set_unk() only after dictionary is frozen");
+      DYNET_RUNTIME_ERR("Please call set_unk() only after dictionary is frozen");
     if (map_unk)
-      throw std::runtime_error("Set UNK more than one time");
+      DYNET_RUNTIME_ERR("Set UNK more than one time");
 
     // temporarily unfrozen the dictionary to allow the add of the UNK
     frozen = false;

dynet/dim.h

@@ -8,7 +8,6 @@
 #ifndef DYNET_DIM_H
 #define DYNET_DIM_H
 
-#include <cassert>
 #include <initializer_list>
 #include <type_traits>
 #include <stdexcept>
@@ -17,6 +16,7 @@
 #include <vector>
 
 #include "dynet/io-macros.h"
+#include "dynet/except.h"
 
 /**
  * \ingroup dim
@@ -160,7 +160,13 @@ struct Dim {
    * \param i Dimension index
    * \param s Dimension size
    */
-  inline void set(unsigned int i, unsigned int s) { assert(i < nd); assert(s > 0); d[i] = s; }
+  inline void set(unsigned int i, unsigned int s) {
+    if(i >= nd)
+      DYNET_INVALID_ARG("Out of bounds exception in Dim::set("<<i<<","<<s<<") for node of size " << d);
+    if(s == 0)
+      DYNET_INVALID_ARG("Attempt to set dimension size to zero in Dim::set("<<i<<","<<s<<") for node of size " << d);
+    d[i] = s;
+  }
   /**
    * \brief Access a specific dimension as you would access an array element
    *
@@ -180,7 +186,8 @@ struct Dim {
    * \param i index of the dimension to be removed
    */
   inline void delete_dim(unsigned int i) {
-    assert(i < nd);
+    if(i >= nd)
+      DYNET_INVALID_ARG("Out of bounds exception in Dim::delete_dim("<<i<<") for node of size " << d);
     if (nd == 1) {
       d[0] = 1;
     } else {
@@ -197,7 +204,7 @@ struct Dim {
   inline Dim transpose() const {
     if (nd == 1) { return Dim({1, d[0]}, bd); }
     else if (nd == 2) { return Dim({d[1], d[0]}, bd); }
-    throw std::invalid_argument("Cannot transpose Dim object with more than 2 dimensions");
+    DYNET_INVALID_ARG("Cannot transpose Dim object with more than 2 dimensions");
   }
 
   unsigned int d[DYNET_MAX_TENSOR_DIM]; /**< Array of dimension */
@@ -207,8 +214,6 @@ struct Dim {
   DYNET_SERIALIZE_DECLARE()
 };
 
-//static_assert(std::is_trivially_copyable<Dim>::value, "Dim must be trivially copyable");
-
 /**
  * \brief Check for equality between two Dim
  * \details Two Dim struct are considered equal if their dimensions and batch size are equal

dynet/except.h

@@ -2,6 +2,7 @@
 #define DYNET_EXCEPT_H_
 
 #include <stdexcept>
+#include <sstream>
 
 namespace dynet {
 
@@ -26,4 +27,23 @@ class cuda_exception : public std::runtime_error {
 };
 } // namespace dynet
 
+#define DYNET_INVALID_ARG(msg) do {             \
+    std::ostringstream oss;                     \
+    oss << msg;                                 \
+    throw std::invalid_argument(oss.str()); }   \
+  while (0);
+
+#define DYNET_ASSERT(expr, msg) do {            \
+  if(!(expr)) {                                  \
+    std::ostringstream oss;                     \
+    oss << msg;                                 \
+    throw std::runtime_error(oss.str()); }      \
+  } while (0);
+
+#define DYNET_RUNTIME_ERR(msg) do {             \
+    std::ostringstream oss;                     \
+    oss << msg;                                 \
+    throw std::runtime_error(oss.str()); }      \
+  while (0);
+
 #endif

dynet/exec.cc

@@ -28,7 +28,7 @@ const Tensor& SimpleExecutionEngine::forward(VariableIndex i) {
 }
 
 const Tensor& SimpleExecutionEngine::get_value(VariableIndex i) {
-  assert(i < cg.nodes.size());
+  DYNET_ASSERT(i < cg.nodes.size(), "Out-of-bounds variable access in SimpleExecutionEngine::get_value()");
   if (i >= num_nodes_evaluated) {
     incremental_forward();
   }
@@ -41,7 +41,7 @@ const Tensor& SimpleExecutionEngine::incremental_forward() {
 }
 
 const Tensor& SimpleExecutionEngine::incremental_forward(VariableIndex i) {
-  assert(i < cg.nodes.size());
+  DYNET_ASSERT(i < cg.nodes.size(), "Out-of-bounds variable access in SimpleExecutionEngine::incremental_forward()");
 
   // free any old memory if this is a new CG
   if (num_nodes_evaluated == 0)
@@ -63,18 +63,18 @@ const Tensor& SimpleExecutionEngine::incremental_forward(VariableIndex i) {
       }
       nfxs[num_nodes_evaluated].d = node->dim;
       // Get the device
-      assert(node->device != nullptr);
+      DYNET_ASSERT(node->device != nullptr, "Attempt to access null device in SimpleExecutionEngine::incremental_forward");
       nfxs[num_nodes_evaluated].device = node->device;
       // Get the memory
       nfxs[num_nodes_evaluated].v = static_cast<float*>(nfxs[num_nodes_evaluated].device->pools[(int)DeviceMempool::FXS]->allocate(node->dim.size() * sizeof(float)));
       if (nfxs[num_nodes_evaluated].v == nullptr)
-        throw std::runtime_error("out of memory");
+        DYNET_RUNTIME_ERR("Ran out of memory when executing node " << num_nodes_evaluated);
       void* aux_mem = nullptr;
       size_t aux_size = node->aux_storage_size();
       if (aux_size) {
         aux_mem = nfxs[num_nodes_evaluated].device->pools[(int)DeviceMempool::FXS]->allocate(aux_size);
         if (!aux_mem)
-          throw std::runtime_error("aux out of memory");
+          DYNET_RUNTIME_ERR("Ran out of auxiliary memory when executing node " << num_nodes_evaluated);
       }
       node->aux_mem = aux_mem;
 
@@ -85,18 +85,16 @@ const Tensor& SimpleExecutionEngine::incremental_forward(VariableIndex i) {
 }
 
 void SimpleExecutionEngine::backward() {
-  assert(nfxs.size() >= cg.nodes.size());
+  DYNET_ASSERT(nfxs.size() >= cg.nodes.size(), "Mismatched array sizes in SimpleExecutionEngine::backward");
   backward((VariableIndex)(cg.nodes.size()-1));
 }
 
 // TODO what is happening with parameter nodes if from_where > param_node_id ?
 void SimpleExecutionEngine::backward(VariableIndex from_where) {
   if(!(from_where < nfxs.size()))
     incremental_forward(from_where);
-  if (nfxs[from_where].d.size() != 1) {
-    ostringstream oss; oss << "backward() can only be called on scalar nodes, but node " << from_where << " has dimension: " << nfxs[from_where].d;
-    throw std::runtime_error(oss.str());
-  }
+  if (nfxs[from_where].d.size() != 1)
+    DYNET_INVALID_ARG("backward() can only be called on scalar nodes, but node " << from_where << " has dimension: " << nfxs[from_where].d);
 
   const unsigned num_nodes = from_where+1;
   ndEdfs.resize(num_nodes);
@@ -108,7 +106,7 @@ void SimpleExecutionEngine::backward(VariableIndex from_where) {
     ndEdfs[i].device = nfxs[i].device;
     ndEdfs[i].v = static_cast<float*>(ndEdfs[i].device->pools[(int)DeviceMempool::DEDFS]->allocate(dim.size() * sizeof(float)));
     if (!ndEdfs[i].v)
-      throw std::runtime_error("out of memory while attempting to allocate space for derivatives");
+      DYNET_RUNTIME_ERR("out of memory while attempting to allocate space for derivatives of node " << i);
   }
   for(Device* device : devices)
     device->pools[(int)DeviceMempool::DEDFS]->zero_allocated_memory();

dynet/expr.cc

@@ -115,6 +115,9 @@ Expression pick(const Expression& x, const vector<unsigned> & v, unsigned d) { r
 Expression pick(const Expression& x, const unsigned* pv, unsigned d) { return Expression(x.pg, x.pg->add_function<PickElement>({x.i}, pv, d)); }
 Expression pick(const Expression& x, const vector<unsigned> * pv, unsigned d) { return Expression(x.pg, x.pg->add_function<PickElement>({x.i}, pv, d)); }
 
+Expression pick_batch(const Expression& x, unsigned v) { return Expression(x.pg, x.pg->add_function<PickBatch>({x.i}, v)); }
+Expression pick_batches(const Expression & x, const std::vector<unsigned>& v) { return Expression(x.pg, x.pg->add_function<PickBatch>({x.i}, v)); }
+
 Expression pickrange(const Expression& x, unsigned v, unsigned u) { return Expression(x.pg, x.pg->add_function<PickRange>({x.i}, v, u)); }
 
 Expression pickneglogsoftmax(const Expression& x, unsigned v) { return Expression(x.pg, x.pg->add_function<PickNegLogSoftmax>({x.i}, v)); }
@@ -126,6 +129,7 @@ Expression average_cols(const Expression& x) { return Expression(x.pg, x.pg->add
 Expression sum_dim(const Expression& x, unsigned d) { return Expression(x.pg, x.pg->add_function<SumDimension>({x.i}, d)); }
 Expression sum_rows(const Expression& x) { return Expression(x.pg, x.pg->add_function<SumDimension>({x.i}, 0)); }
 Expression sum_cols(const Expression& x) { return Expression(x.pg, x.pg->add_function<SumDimension>({x.i}, 1)); }
+Expression sum_elems(const Expression& x) { return Expression(x.pg, x.pg->add_function<SumElements>({x.i})); }
 
 Expression sum_batches(const Expression& x) { return Expression(x.pg, x.pg->add_function<SumBatches>({x.i})); }