Docs for autodiff and custom loss (#333)

* added Google to the list of third party programs

Signed-off-by: NadyaG <Nadezda_Kr@abbyy.com>

* docs for blob

* docs for Dnn

* Update README.md

* fix the comments

* docs for clustering

* misprints

* comments in C++

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* remove hyphen

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* added comments for custom loss, fixes in autodiff

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* fix in format for custom loss

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* trying to fix the build

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* trying to fix the build with version of jinja for notebooks

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* added more on operations in custom loss

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* custom loss tutorial proofreading

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* misprint

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* corrections about custom loss and autodiff

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

* shifted the description to returns

Signed-off-by: Nadezda Gaganova <nadezda.gaganova@abbyy.com>

Co-authored-by: NadyaG <Nadezda_Kr@abbyy.com>
Co-authored-by: Stanislav Angeliuk <59917951+SAngeliuk@users.noreply.github.com>

NeoML/Python/neoml/AutoDiff.py

@@ -22,123 +22,123 @@
 # ----------------------------------------------------------------------------------------------------------------------
 
 def const(math_engine, shape, data):
-    """
+    """Creates a blob of the specified shape filled with `data` value.
     """
     if not isinstance(math_engine, MathEngine):
-        raise ValueError('The `math_engine` must be neoml.MathEngine.')
+        raise ValueError('The `math_engine` should be neoml.MathEngine.')
 
     np_shape = numpy.array(shape, dtype=numpy.int32, copy=False)
 
     if len(np_shape.shape) > 7:
-        raise ValueError('The `shape` must have not more then 7 dimensions.')
+        raise ValueError('The `shape` should have not more than 7 dimensions.')
 
     if numpy.isscalar(data):
         return Blob(PythonWrapper.blob_const(math_engine._internal, np_shape, float(data)))
 
     np_data = numpy.array(data, copy=False, order='C')
 
     if len(np_data.shape) > 7:
-        raise ValueError('The `shape` must have not more then 7 dimensions.')
+        raise ValueError('The `shape` should have not more than 7 dimensions.')
 
     return Blob(PythonWrapper.blob_const(math_engine._internal, np_shape, np_data))
 
 def add(a, b):
-    """Elementwise sum of two blobs or blob with a scalar
+    """Elementwise adds two blobs or a blob and a scalar value.
     """
     if not type(a) is Blob and not type(b) is Blob:
-        raise ValueError('`a` or `b` must be neoml.Blob.')
+        raise ValueError('At least one of `a` and `b` should be neoml.Blob.')
 
     return a + b
 
 def sub(a, b):
-    """Elementwise sub of two blobs or blob with a scalar
+    """Elementwise subtracts two blobs or a blob and a scalar value.
     """
     if not type(a) is Blob and not type(b) is Blob:
-        raise ValueError('`a` or `b` must be neoml.Blob.')
+        raise ValueError('At least one of `a` and `b` should be neoml.Blob.')
 
     return a - b
 
 def mul(a, b):
-    """Elementwise mul of two blobs or blob with a scalar
+    """Elementwise multiplies two blobs or a blob and a scalar value.
     """
     if not type(a) is Blob and not type(b) is Blob:
-        raise ValueError('`a` or `b` must be neoml.Blob.')
+        raise ValueError('At least one of `a` and `b` should be neoml.Blob.')
 
     return a * b
 
 def div(a, b):
-    """Elementwise div of two blobs or blob with a scalar
+    """Elementwise divides two blobs or a blob and a scalar value.
     """
     if not type(a) is Blob and not type(b) is Blob:
-        raise ValueError('`a` or `b` must be neoml.Blob.')
+        raise ValueError('At least one of `a` and `b` should be neoml.Blob.')
 
     return a / b
 
 def max(a, b):
-    """
+    """Takes the elementwise maximum of two blobs or a blob and a scalar value.
     """
     if type(a) is Blob:
         if a.size == 0:
-            raise ValueError("The blob mustn't be empty.")
+            raise ValueError("The blob shouldn't be empty.")
         return Blob(PythonWrapper.blob_max(a._internal, float(b)))
     elif type(b) is Blob:
         if b.size == 0:
-            raise ValueError("The blob mustn't be empty.")
+            raise ValueError("The blob shouldn't be empty.")
         return Blob(PythonWrapper.blob_max(float(a), b._internal))
 
-    raise ValueError('`a` or `b` must be neoml.Blob.')
+    raise ValueError('At least one of `a` and `b` should be neoml.Blob.')
 
 def sum(a):
-    """
+    """Calculates the total sum of blob elements.
     """
     if not type(a) is Blob:
-        raise ValueError('`a` must be neoml.Blob.')
+        raise ValueError('`a` should be neoml.Blob.')
 
     if a.size == 0:
-        raise ValueError("The blobs mustn't be empty.")
+        raise ValueError("The blob shouldn't be empty.")
 
     return Blob(PythonWrapper.blob_sum(a._internal))
 
 def neg(a):
-    """
+    """Returns the negative of a blob or a number.
     """
     return -a;
 
 def abs(a):
-    """
+    """Takes absolute value of each blob element.
     """
     if not type(a) is Blob:
-        raise ValueError('`a` must be neoml.Blob.')
+        raise ValueError('`a` should be neoml.Blob.')
 
     if a.size == 0:
-        raise ValueError("The blobs mustn't be empty.")
+        raise ValueError("The blob shouldn't be empty.")
 
     return Blob(PythonWrapper.blob_abs(a._internal))
 
 def log(a):
-    """
+    """Takes the logarithm of each blob element.
     """
     if not type(a) is Blob:
-        raise ValueError('`a` must be neoml.Blob.')
+        raise ValueError('`a` should be neoml.Blob.')
 
     if a.size == 0:
-        raise ValueError("The blobs mustn't be empty.")
+        raise ValueError("The blob shouldn't be empty.")
 
     return Blob(PythonWrapper.blob_log(a._internal))
 
 def exp(a):
-    """
+    """Takes the exponential of each blob element.
     """
     if not type(a) is Blob:
-        raise ValueError('`a` must be neoml.Blob.')
+        raise ValueError('`a` should be neoml.Blob.')
 
     if a.size == 0:
-        raise ValueError("The blobs mustn't be empty.")
+        raise ValueError("The blob shouldn't be empty.")
 
     return Blob(PythonWrapper.blob_exp(a._internal))
 
 def clip(blob, min_value, max_value):
-    """
+    """Clips each element of the blob so that it fits between the specified limits.
     """
     if not type(blob) is Blob:
         raise ValueError('`blob` must be neoml.Blob.')
@@ -149,32 +149,36 @@ def clip(blob, min_value, max_value):
     return Blob(PythonWrapper.blob_clip(blob._internal, float(min_value), float(max_value)))
 
 def top_k(a, k=1):
-    """
+    """Finds values of the k largest elements in the blob.
+    The result is a blob of size k.
     """
     if not type(a) is Blob:
-        raise ValueError('`a` must be neoml.Blob.')
+        raise ValueError('`a` should be neoml.Blob.')
 
     if int(k) <= 0:
-        raise ValueError('`k` must be > 0.')
+        raise ValueError('`k` should be > 0.')
 
     if a.size == 0:
-        raise ValueError("The blobs mustn't be empty.")
+        raise ValueError("The blob shouldn't be empty.")
 
     return Blob(PythonWrapper.blob_top_k(a._internal, int(k)))
 
 def binary_cross_entropy(labels, preds, fromLogits):
-    """
+    """Calculates binary cross-entropy for two blobs: the first one contains labels, the second one contains predictions.
+    Blobs should be of the same shape.
+    :math:`result = (1 - labels) * x + log(1 + exp(-x))`
+    if `fromLogits` then `x = preds`, else :math:`x = log( clippedPreds / (1 - clippedPreds))`
     """
     if not type(labels) is Blob:
-        raise ValueError('`labels` must be neoml.Blob.')
+        raise ValueError('`labels` should be neoml.Blob.')
 
     if not type(preds) is Blob:
-        raise ValueError('`preds` must be neoml.Blob.')
+        raise ValueError('`preds` should be neoml.Blob.')
 
     if labels.shape != preds.shape:
-        raise ValueError("The blobs must have the same shape.")
+        raise ValueError("The blobs should be of the same shape.")
 
     if labels.size == 0:
-        raise ValueError("The blobs mustn't be empty.")
+        raise ValueError("The blobs shouldn't be empty.")
 
     return Blob(PythonWrapper.blob_binary_cross_entropy(labels._internal, preds._internal, bool(fromLogits)))

NeoML/Python/neoml/Blob.py

@@ -127,47 +127,47 @@ def copy(self, math_engine):
         return Blob(self._internal.copy(math_engine._internal))
 
     def __add__(self, other):
-        """Elementwise sum of two blobs or blob with a scalar
+        """Elementwise adds two blobs or a blob and a scalar value.
         """
         if self.size == 0:
-            raise ValueError("The blob mustn't be empty.")
+            raise ValueError("The blob shouldn't be empty.")
 
         if type(other) is Blob:
             if self.shape != other.shape:
-                raise ValueError("The blobs must have the same shape.")
+                raise ValueError("The blobs should have the same shape.")
             return Blob(PythonWrapper.blob_add(self._internal, other._internal))
 
         return Blob(PythonWrapper.blob_add(self._internal, float(other)))
 
     def __radd__(self, other):
-        """Elementwise sum of two blobs or a scalar with blob
+        """Elementwise adds two blobs or a scalar value and a blob.
         """
         if self.size == 0:
-            raise ValueError("The blob mustn't be empty.")
+            raise ValueError("The blob shouldn't be empty.")
         return Blob(PythonWrapper.blob_add(self._internal, float(other)))
 
     def __sub__(self, other):
-        """Elementwise sub of two blobs or blob with a scalar
+        """Elementwise subtracts two blobs or a blob and a scalar value.
         """
         if self.size == 0:
-            raise ValueError("The blob mustn't be empty.")
+            raise ValueError("The blob shouldn't be empty.")
 
         if type(other) is Blob:
             if self.shape != other.shape:
-                raise ValueError("The blobs must have the same shape.")
+                raise ValueError("The blobs should have the same shape.")
             return Blob(PythonWrapper.blob_sub(self._internal, other._internal))
 
         return Blob(PythonWrapper.blob_sub(self._internal, float(other)))
 
     def __rsub__(self, other):
-        """Elementwise sub of two blobs or a scalar with blob
+        """Elementwise subtracts two blobs or a scalar value and a blob.
         """
         if self.size == 0:
             raise ValueError("The blob mustn't be empty.")
         return Blob(PythonWrapper.blob_sub(float(other), self._internal))
 
     def __mul__(self, other):
-        """Elementwise mul of two blobs or blob with a scalar
+        """Elementwise multiplies two blobs or a blob and a scalar value.
         """
         if self.size == 0:
             raise ValueError("The blob mustn't be empty.")
@@ -180,14 +180,14 @@ def __mul__(self, other):
         return Blob(PythonWrapper.blob_mul(self._internal, float(other)))
 
     def __rmul__(self, other):
-        """Elementwise mul of two blobs or a scalar with blob
+        """Elementwise multiplies two blobs or a scalar value and a blob.
         """
         if self.size == 0:
             raise ValueError("The blob mustn't be empty.")
         return Blob(PythonWrapper.blob_mul(self._internal, float(other)))
 
     def __truediv__(self, other):
-        """Elementwise div of two blobs or blob with a scalar
+        """Elementwise divides two blobs or a blob and a scalar value.
         """
         if self.size == 0:
             raise ValueError("The blob mustn't be empty.")
@@ -200,14 +200,14 @@ def __truediv__(self, other):
         return Blob(PythonWrapper.blob_div(self._internal, float(other)))
 
     def __rtruediv__(self, other):
-        """Elementwise div of two blobs or a scalar with blob
+        """Elementwise divides two blobs or a scalar value and a blob.
         """
         if self.size == 0:
             raise ValueError("The blob mustn't be empty.")
         return Blob(PythonWrapper.blob_div(float(other), self._internal))
 
     def __neg__(self):
-        """Elementwise negative
+        """Takes elementwise negative of the blob.
         """
         if self.size == 0:
             raise ValueError("The blobs mustn't be empty.")

NeoML/Python/neoml/Dnn/Loss.py

@@ -741,15 +741,72 @@ def __init__(self, input_layers, loss_weight=1.0, name=None):
 
 
 class CustomLossCalculatorBase(metaclass=ABCMeta):
-    """
+    """The base class that you should implement to calculate the custom loss function.
     """
     @abstractmethod
     def calc(self, data, labels):
-        """
+        """Calculates the custom loss function.
+        This function may use only the operations supported for autodiff:
+
+        - simple arithmetic: `+ - * /`
+        - the `neoml.AutoDiff.*` functions
+        - `neoml.Autodiff.const` for creating additional blobs filled with given values
+
+        :param neoml.Blob.Blob data: the network response with the probability 
+        distribution of objects over classes. The blob dimensions:
+
+        - **BatchLength** - the number of objects
+        - **Channels** - the object size
+        - all other dimensions equal to 1
+
+        :param neoml.Blob.Blob labels: the correct labels, of the same dimensions
+        as the first blob, and containing 1 in the coordinate of the class to which
+        the corresponding object belongs, 0 in all other places.
+
+        :return: a blob that contains the loss function values 
+        for each object in the batch. This blob will have the same **BatchLength** 
+        as the input blobs, and all its other dimensions should be 1.
+        :rtype: `neoml.Blob.Blob`
         """
 
 class CustomLoss(Loss):
-    """
+    """The layer that calculates a custom loss function.
+
+    :param input_layers: the input layers to be connected. 
+        The integer in each tuple specifies the number of the output.
+        If not set, the first output will be used.
+    :type input_layers: list of object, tuple(object, int)
+    :param neoml.Dnn.CustomLossCalculatorBase loss_calculator: a user-implemented object 
+        that provides the method to calculate the custom loss.
+    :param loss_weight: the multiplier for the loss function value during training.
+    :type loss_weight: float, default=1.0
+    :param name: the layer name.
+    :type name: str, default=None
+
+    .. rubric:: Layer inputs:
+
+    (1) the network response for which you are calculating the loss.
+        It should contain the probability distribution for objects over classes.
+        If you are not going to apply softmax in this layer, each element should already be >= 0, 
+        and the sum over **Height** * **Width** * **Depth** * **Channels** dimension should be equal to 1.
+        The dimensions:
+
+        - **BatchLength** * **BatchWidth** * **ListSize** - the number of objects
+        - **Height** * **Width** * **Depth** * **Channels** - the number of classes
+    
+    (2) the correct class labels. The blob of the same dimensions as the first input, 
+        filled with zeros, where only the coordinate of the class to which 
+        the corresponding object from the first input belongs is be 1.
+    
+    (3) (optional): the objects' weights.
+        The dimensions:
+
+        - **BatchLength**, **BatchWidth**, **ListSize** should be the same as for the first input
+        - the other dimensions should be 1
+
+    .. rubric:: Layer outputs:
+
+    The layer has no output.
     """
     def __init__(self, input_layers, loss_calculator=None, loss_weight=1.0, name=None):
         if type(input_layers) is PythonWrapper.CustomLoss:
@@ -768,7 +825,14 @@ def __init__(self, input_layers, loss_calculator=None, loss_weight=1.0, name=Non
 
 
 def call_loss_calculator(data, labels, loss_calculator):
-    """
+    """Calculates the value of specified custom loss function.
+    
+    :param neoml.Blob.Blob data: the network response.
+
+    :param neoml.Blob.Blob labels: the correct labels.
+
+    :param neoml.Dnn.CustomLossCalculatorBase loss_calculator: a user-implemented object
+        that provides the method to calculate the custom loss.
     """
     data_blob = Blob.Blob(data)
     labels_blob = Blob.Blob(labels)