added DropoutComponent to nnet3 for ephemeral connection.

egs/swbd/s5c/local/chain/tuning/run_tdnn_7d.sh

@@ -60,7 +60,6 @@ If you want to use GPUs (and have them), go to src/, and configure and make on a
 where "nvcc" is installed.
 EOF
 fi
-
 # The iVector-extraction and feature-dumping parts are the same as the standard
 # nnet3 setup, and you can skip them by setting "--stage 8" if you have already
 # run those things.
@@ -76,7 +75,6 @@ ali_dir=exp/tri4_ali_nodup$suffix
 treedir=exp/chain/tri5_7d_tree$suffix
 lang=data/lang_chain_2y
 
-
 # if we are using the speed-perturbed data we need to generate
 # alignments for it.
 local/nnet3/run_ivector_common.sh --stage $stage \

src/matrix/kaldi-matrix.cc

@@ -180,8 +180,8 @@ void MatrixBase<Real>::AddMatMat(const Real alpha,
 
 template<typename Real>
 void MatrixBase<Real>::AddMatMatDivMat(const MatrixBase<Real>& A,
-                                     const MatrixBase<Real>& B,
-                                   const MatrixBase<Real>& C) {
+                                       const MatrixBase<Real>& B,
+                                       const MatrixBase<Real>& C) {
   KALDI_ASSERT(A.NumRows() == B.NumRows() && A.NumCols() == B.NumCols());
   KALDI_ASSERT(A.NumRows() == C.NumRows() && A.NumCols() == C.NumCols());
   for (int32 r = 0; r < A.NumRows(); r++) { // each frame...

src/matrix/kaldi-matrix.h

@@ -580,7 +580,7 @@ class MatrixBase {
 
   /// *this = a * b / c (by element; when c = 0, *this = a)
   void AddMatMatDivMat(const MatrixBase<Real>& A,
-                        const MatrixBase<Real>& B,
+                       const MatrixBase<Real>& B,
                        const MatrixBase<Real>& C);
 
   /// A version of AddMatMat specialized for when the second argument

src/nnet3/nnet-component-itf.cc

@@ -141,6 +141,8 @@ Component* Component::NewComponentOfType(const std::string &component_type) {
     ans = new StatisticsPoolingComponent();
   } else if (component_type == "ConstantFunctionComponent") {
     ans = new ConstantFunctionComponent();
+  } else if (component_type == "DropoutComponent") {
+    ans = new DropoutComponent();
   }
   if (ans != NULL) {
     KALDI_ASSERT(component_type == ans->Type());

src/nnet3/nnet-component-itf.h

@@ -350,6 +350,16 @@ class Component {
 };
 
 
+class RandomComponent: public Component {
+ public:
+  // This function is required in testing code and in other places we need
+  // consistency in the random number generation (e.g. when optimizing
+  // validation-set performance), but check where else we call srand().  You'll
+  // need to call srand as well as making this call.
+  void ResetGenerator() { random_generator_.SeedGpu(); }
+ protected:
+  CuRand<BaseFloat> random_generator_;
+};
 
 /**
  * Class UpdatableComponent is a Component which has trainable parameters; it

src/nnet3/nnet-component-test.cc

@@ -23,7 +23,16 @@
 
 namespace kaldi {
 namespace nnet3 {
-
+// Reset seeds for test time for RandomComponent
+static void ResetSeed(int32 rand_seed, const Component &c) {
+  RandomComponent *rand_component = 
+    const_cast<RandomComponent*>(dynamic_cast<const RandomComponent*>(&c));
+
+  if (rand_component != NULL) {
+    srand(rand_seed);
+    rand_component->ResetGenerator();
+  }
+}
 // returns true if two are string are equal except for what looks like it might
 // be a difference last digit of a floating point number, e.g. accept
 // 1.234 to be the same as 1.235.  Not very rigorous.
@@ -188,6 +197,8 @@ void TestNnetComponentUpdatable(Component *c) {
 void TestSimpleComponentPropagateProperties(const Component &c) {
   int32 properties = c.Properties();
   Component *c_copy = NULL, *c_copy_scaled = NULL;
+  int32 rand_seed = Rand();
+
   if (RandInt(0, 1) == 0)
     c_copy = c.Copy();  // This will test backprop with an updatable component.
   if (RandInt(0, 1) == 0 &&
@@ -223,10 +234,14 @@ void TestSimpleComponentPropagateProperties(const Component &c) {
   if ((properties & kPropagateAdds) && (properties & kPropagateInPlace)) {
     KALDI_ERR << "kPropagateAdds and kPropagateInPlace flags are incompatible.";
   }
-
+
+  ResetSeed(rand_seed, c);
   c.Propagate(NULL, input_data, &output_data1);
+
+  ResetSeed(rand_seed, c);
   c.Propagate(NULL, input_data, &output_data2);
   if (properties & kPropagateInPlace) {
+    ResetSeed(rand_seed, c);
     c.Propagate(NULL, output_data3, &output_data3);
     if (!output_data1.ApproxEqual(output_data3)) {
       KALDI_ERR << "Test of kPropagateInPlace flag for component of type "
@@ -238,12 +253,14 @@ void TestSimpleComponentPropagateProperties(const Component &c) {
   AssertEqual(output_data1, output_data2);
 
   if (c_copy_scaled) {
+    ResetSeed(rand_seed, *c_copy_scaled);
     c_copy_scaled->Propagate(NULL, input_data, &output_data4);
     output_data4.Scale(2.0);  // we scaled the parameters by 0.5 above, and the
     // output is supposed to be linear in the parameter value.
     AssertEqual(output_data1, output_data4);
   }
   if (properties & kLinearInInput) {
+    ResetSeed(rand_seed, c);
     c.Propagate(NULL, input_data_scaled, &output_data5);
     output_data5.Scale(0.5);
     AssertEqual(output_data1, output_data5);
@@ -302,14 +319,16 @@ bool TestSimpleComponentDataDerivative(const Component &c,
 
   int32 input_dim = c.InputDim(),
       output_dim = c.OutputDim(),
-      num_rows = RandInt(1, 100);
+      num_rows = RandInt(1, 100),
+      rand_seed = Rand();
   int32 properties = c.Properties();
   CuMatrix<BaseFloat> input_data(num_rows, input_dim, kSetZero, input_stride_type),
       output_data(num_rows, output_dim, kSetZero, output_stride_type),
       output_deriv(num_rows, output_dim, kSetZero, output_stride_type);
   input_data.SetRandn();
   output_deriv.SetRandn();
-
+
+  ResetSeed(rand_seed, c);
   c.Propagate(NULL, input_data, &output_data);
 
   CuMatrix<BaseFloat> input_deriv(num_rows, input_dim, kSetZero, input_stride_type),
@@ -334,6 +353,8 @@ bool TestSimpleComponentDataDerivative(const Component &c,
     predicted_objf_change(i) = TraceMatMat(perturbed_input_data, input_deriv,
                                            kTrans);
     perturbed_input_data.AddMat(1.0, input_data);
+
+    ResetSeed(rand_seed, c);
     c.Propagate(NULL, perturbed_input_data, &perturbed_output_data);
     measured_objf_change(i) = TraceMatMat(output_deriv, perturbed_output_data,
                                           kTrans) - original_objf;
@@ -503,7 +524,7 @@ int main() {
   TestStringsApproxEqual();
   for (kaldi::int32 loop = 0; loop < 2; loop++) {
 #if HAVE_CUDA == 1
-    CuDevice::Instantiate().SetDebugStrideMode(true);
+    //CuDevice::Instantiate().SetDebugStrideMode(true);
     if (loop == 0)
       CuDevice::Instantiate().SelectGpuId("no");
     else

src/nnet3/nnet-nnet.cc

@@ -784,5 +784,15 @@ std::string Nnet::Info() const {
   return os.str();
 }
 
+void Nnet::ResetGenerators() {
+  // resets random-number generators for all random
+  // components.
+  for (int32 c = 0; c < NumComponents(); c++) {
+    RandomComponent *rc = dynamic_cast<RandomComponent*>(GetComponent(c));
+    if (rc != NULL)
+      rc->ResetGenerator();
+  }
+}
+
 } // namespace nnet3
 } // namespace kaldi

src/nnet3/nnet-nnet.h

@@ -229,6 +229,11 @@ class Nnet {
 
   // Assignment operator
   Nnet& operator =(const Nnet &nnet);
+
+  void ResetGenerators(); // resets random-number generators for all
+  // random components.  You must also set srand() for this to be
+  // effective.
+
  private:
 
   void Destroy();
@@ -306,7 +311,6 @@ class Nnet {
 };
 
 
-
 } // namespace nnet3
 } // namespace kaldi
 

src/nnet3/nnet-simple-component.cc

@@ -86,6 +86,85 @@ void PnormComponent::Write(std::ostream &os, bool binary) const {
 }
 
 
+void DropoutComponent::Init(int32 dim, BaseFloat dropout_proportion) {
+  dropout_proportion_ = dropout_proportion;
+  dim_ = dim;
+}
+
+void DropoutComponent::InitFromConfig(ConfigLine *cfl) {
+  int32 dim = 0;
+  BaseFloat dropout_proportion = 0.0;
+  bool ok = cfl->GetValue("dim", &dim) &&
+    cfl->GetValue("dropout-proportion", &dropout_proportion);
+  if (!ok || cfl->HasUnusedValues() || dim <= 0 || 
+      dropout_proportion < 0.0 || dropout_proportion > 1.0)
+    KALDI_ERR << "Invalid initializer for layer of type " 
+              << Type() << ": \"" << cfl->WholeLine() << "\"";   
+  Init(dim, dropout_proportion);
+}
+
+std::string DropoutComponent::Info() const {
+  std::ostringstream stream;
+  stream << Type() << ", dim = " << dim_ 
+         << ", dropout-proportion = " << dropout_proportion_;
+  return stream.str();
+}
+
+void DropoutComponent::Propagate(const ComponentPrecomputedIndexes *indexes,
+                                 const CuMatrixBase<BaseFloat> &in,
+                                 CuMatrixBase<BaseFloat> *out) const {
+  KALDI_ASSERT(out->NumRows() == in.NumRows() && out->NumCols() == in.NumCols()
+               && in.NumCols() == dim_);
+
+  BaseFloat dropout = dropout_proportion_;
+  KALDI_ASSERT(dropout >= 0.0 && dropout <= 1.0);
+
+  // This const_cast is only safe assuming you don't attempt  
+  // to use multi-threaded code with the GPU.
+  const_cast<CuRand<BaseFloat>&>(random_generator_).RandUniform(out); 
+
+  out->Add(-dropout); // now, a proportion "dropout" will be <0.0 
+  out->ApplyHeaviside(); // apply the function (x>0?1:0).  Now, a proportion "dropout" will 
+                         // be zero and (1 - dropout) will be 1.0.
+
+  out->MulElements(in);
+}
+
+
+void DropoutComponent::Backprop(const std::string &debug_info,
+                                const ComponentPrecomputedIndexes *indexes,
+                                const CuMatrixBase<BaseFloat> &in_value,
+                                const CuMatrixBase<BaseFloat> &out_value,
+                                const CuMatrixBase<BaseFloat> &out_deriv,
+                                Component *to_update,
+                                CuMatrixBase<BaseFloat> *in_deriv) const {
+  KALDI_ASSERT(in_value.NumRows() == out_value.NumRows() &&
+               in_value.NumCols() == out_value.NumCols());
+
+  KALDI_ASSERT(in_value.NumRows() == out_deriv.NumRows() &&
+               in_value.NumCols() == out_deriv.NumCols());
+  in_deriv->AddMatMatDivMat(out_deriv, out_value, in_value);
+}
+
+
+
+void DropoutComponent::Read(std::istream &is, bool binary) {
+  ExpectOneOrTwoTokens(is, binary, "<DropoutComponent>", "<Dim>");
+  ReadBasicType(is, binary, &dim_);
+  ExpectToken(is, binary, "<DropoutProportion>");
+  ReadBasicType(is, binary, &dropout_proportion_);
+  ExpectToken(is, binary, "</DropoutComponent>");
+}
+
+void DropoutComponent::Write(std::ostream &os, bool binary) const {
+  WriteToken(os, binary, "<DropoutComponent>");
+  WriteToken(os, binary, "<Dim>");
+  WriteBasicType(os, binary, dim_);
+  WriteToken(os, binary, "<DropoutProportion>");
+  WriteBasicType(os, binary, dropout_proportion_);
+  WriteToken(os, binary, "</DropoutComponent>");
+}
+
 void SumReduceComponent::Init(int32 input_dim, int32 output_dim)  {
   input_dim_ = input_dim;
   output_dim_ = output_dim;

src/nnet3/nnet-simple-component.h

@@ -79,6 +79,59 @@ class PnormComponent: public Component {
   int32 output_dim_;
 };
 
+// This component randomly zeros dropout_proportion of the input 
+// and the derivatives are backpropagated through the nonzero inputs.
+// Typically this component used during training but not in test time.
+// The idea is described under the name Dropout, in the paper 
+// "Dropout: A Simple Way to Prevent Neural Networks from Overfitting".
+class DropoutComponent : public RandomComponent {
+ public:
+  void Init(int32 dim, BaseFloat dropout_proportion = 0.0);
+
+  DropoutComponent(int32 dim, BaseFloat dropout = 0.0) { Init(dim, dropout); }
+
+  DropoutComponent(): dim_(0), dropout_proportion_(0.0) { }
+
+  virtual int32 Properties() const {
+    return kLinearInInput|kBackpropInPlace|kSimpleComponent|kBackpropNeedsInput|kBackpropNeedsOutput;
+  }
+  virtual std::string Type() const { return "DropoutComponent"; }
+
+  virtual void InitFromConfig(ConfigLine *cfl);
+
+  virtual int32 InputDim() const { return dim_; }
+
+  virtual int32 OutputDim() const { return dim_; }
+
+  virtual void Read(std::istream &is, bool binary);
+
+  // Write component to stream
+  virtual void Write(std::ostream &os, bool binary) const;
+
+  virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
+                         const CuMatrixBase<BaseFloat> &in,
+                         CuMatrixBase<BaseFloat> *out) const;
+  virtual void Backprop(const std::string &debug_info,
+                        const ComponentPrecomputedIndexes *indexes,
+                        const CuMatrixBase<BaseFloat> &in_value,
+                        const CuMatrixBase<BaseFloat> &out_value,
+                        const CuMatrixBase<BaseFloat> &out_deriv,
+                        Component *to_update,
+                        CuMatrixBase<BaseFloat> *in_deriv) const;
+  virtual Component* Copy() const { return new DropoutComponent(dim_,
+                                                                dropout_proportion_); }
+  virtual std::string Info() const;
+
+  void SetDropoutProportion(BaseFloat dropout_proportion) { dropout_proportion_ = dropout_proportion; }
+
+ private:
+  int32 dim_;
+  /// dropout-proportion is the proportion that is dropped out,
+  /// e.g. if 0.1, we set 10% to zero value.
+  BaseFloat dropout_proportion_;
+
+};
+
 class ElementwiseProductComponent: public Component {
  public:
   void Init(int32 input_dim, int32 output_dim);

src/nnet3/nnet-test-utils.cc

@@ -926,7 +926,7 @@ void ComputeExampleComputationRequestSimple(
 static void GenerateRandomComponentConfig(std::string *component_type,
                                           std::string *config) {
 
-  int32 n = RandInt(0, 28);
+  int32 n = RandInt(0, 29);
   BaseFloat learning_rate = 0.001 * RandInt(1, 3);
 
   std::ostringstream os;
@@ -1218,7 +1218,13 @@ static void GenerateRandomComponentConfig(std::string *component_type,
       if (RandInt(0, 1) == 1)
         os << " self-repair-target=" << RandUniform();
       break;
-    } 
+    }
+    case 29: {
+      *component_type = "DropoutComponent";
+      os << "dim=" << RandInt(1, 200)
+         << " dropout-proportion=" << RandUniform();
+      break;
+    }
     default:
       KALDI_ERR << "Error generating random component";
   }

src/nnet3/nnet-utils.cc

@@ -494,6 +494,15 @@ std::string NnetInfo(const Nnet &nnet) {
   return ostr.str();
 }
 
+void SetDropoutProportion(BaseFloat dropout_proportion,
+                          Nnet *nnet) {
+  for (int32 c = 0; c < nnet->NumComponents(); c++) {
+    Component *comp = nnet->GetComponent(c);
+    DropoutComponent *dc = dynamic_cast<DropoutComponent*>(comp);
+    if (dc != NULL)
+      dc->SetDropoutProportion(dropout_proportion);
+  }
+}
 
 } // namespace nnet3
 } // namespace kaldi

src/nnet3/nnet-utils.h

@@ -174,6 +174,9 @@ void ConvertRepeatedToBlockAffine(Nnet *nnet);
 /// Info() function (we need this in the CTC code).
 std::string NnetInfo(const Nnet &nnet);
 
+/// This function sets the dropout proportion in all dropout component to 
+/// dropout_proportion value.
+void SetDropoutProportion(BaseFloat dropout_proportion, Nnet *nnet);
 
 } // namespace nnet3
 } // namespace kaldi