Parameter layer (#1021)

* fixed hierarchical clusterization with ward (#998) Signed-off-by: Daniyal Aliev <daniial.aliev@abbyy.com> Co-authored-by: Valeriy Fedyunin <valery.fedyunin@abbyy.com> * master commit Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * added_parameter_layer Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * safer set blob Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * kirill's remarks Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * added SetBlobDesc method Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * new serialization + mistakes Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> --------- Signed-off-by: Daniyal Aliev <daniial.aliev@abbyy.com> Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> Co-authored-by: Valeriy Fedyunin <valery.fedyunin@abbyy.com> Co-authored-by: daniyalaliev <daniial.aliev@abbyy.com> Co-authored-by: Stanislav Angeliuk <59917951+SAngeliuk@users.noreply.github.com>
neoml-lib · Jan 18, 2024 · 24fc118 · 24fc118
1 parent b7a874f
commit 24fc118
Show file tree

Hide file tree

Showing 11 changed files with 371 additions and 0 deletions.
diff --git a/NeoML/docs/en/API/NN/ParameterLayer.md b/NeoML/docs/en/API/NN/ParameterLayer.md
@@ -0,0 +1,41 @@
+# CParameterLayer Class
+
+<!-- TOC -->
+
+- [CParameterLayer Class](#CParameterLayer-class)
+    - [Settings](#settings)
+        - [Setting blob](#initialization-weight-blob )
+    - [Trainable parameters](#trainable-parameters)
+        - [Weight blob](#weight-blob)
+    - [Inputs](#inputs)
+    - [Outputs](#outputs)
+
+<!-- /TOC -->
+
+Class implements layer with blob with trainable parameters.
+
+## Settings
+
+### Initialization weight blob
+
+```c++
+void SetBlob(CDnnBlob* _blob);
+```
+Setting blob of trainable parameters.
+
+## Trainable parameters
+
+### Weight blob
+
+```c++
+const CPtr<CDnnBlob>& GetBlob() const;
+```
+The blob with trainable weights.
+
+## Inputs
+
+The layer has no inputs.
+
+## Outputs
+
+The output is blob of weights (with same dimensions as the initiated blob had).
diff --git a/NeoML/docs/ru/API/NN/ParameterLayer.md b/NeoML/docs/ru/API/NN/ParameterLayer.md
@@ -0,0 +1,41 @@
+# Класс CParameterLayer
+
+<!-- TOC -->
+
+- [Класс CParameterLayer](#класс-CParameterLayer)
+    - [Настройки](#настройки)
+        - [Инициализация матрицы весовв](#Инициализация-матрицы-весов)
+    - [Обучаемые параметры](#обучаемые-параметры)
+        - [Матрица весов](#матрица-весов)
+    - [Входы](#входы)
+    - [Выходы](#выходы)
+
+<!-- /TOC -->
+
+Класс реализует слой, содержащий блоб обучаемых параметров.
+
+## Настройки
+
+### Инициализация матрицы весов
+
+```c++
+void SetBlob(CDnnBlob* _blob);
+```
+Установка блоба начальных весов слоя.
+
+## Обучаемые параметры
+
+### Матрица весов
+
+```c++
+const CPtr<CDnnBlob>& GetBlob() const;
+```
+Блоб с обучаемыми весами слоя.
+
+## Входы
+
+У слоя нет входов.
+
+## Выходы
+
+Выходом слоя является блоб весов ( той же размерности, что был установлен, соответственно).
diff --git a/NeoML/include/NeoML/Dnn/Layers/ParameterLayer.h b/NeoML/include/NeoML/Dnn/Layers/ParameterLayer.h
@@ -0,0 +1,47 @@
+/* Copyright © 2024 ABBYY Production LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+--------------------------------------------------------------------------------------------------------------*/
+
+#pragma once
+
+#include <NeoML/NeoMLDefs.h>
+#include <NeoML/Dnn/Dnn.h>
+
+namespace NeoML {
+
+// Layer with blob of trainable parameters ( nn.Parameter analogous )
+class NEOML_API CParameterLayer : public CBaseLayer {
+    NEOML_DNN_LAYER(CParameterLayer)
+public:
+    CParameterLayer( IMathEngine& mathEngine ) :
+        CBaseLayer(mathEngine, "ParameterLayer", true) { paramBlobs.SetSize(1); };
+
+    void Serialize(CArchive& archive) override;
+    void SetBlob(CDnnBlob* _blob);
+    void SetBlobDesc(const CBlobDesc& _desc);
+    const CPtr<CDnnBlob>& GetBlob() const { return paramBlobs[0]; }
+protected:
+    void AllocateOutputBlobs() override;
+    void Reshape() override;
+    void RunOnce() override;
+    void BackwardOnce() override;
+    void LearnOnce() override;
+private:
+    CBlobDesc desc;
+};
+
+// To make it more convenient to create a class object
+NEOML_API CParameterLayer* Parameter(CDnn& dnn, const char* name, CDnnBlob* blob);
+
+} // namespace NeoML
diff --git a/NeoML/include/NeoML/NeoML.h b/NeoML/include/NeoML/NeoML.h
@@ -65,6 +65,7 @@ limitations under the License.
 #include <NeoML/Dnn/Layers/MultichannelLookupLayer.h>
 #include <NeoML/Dnn/Layers/MultiheadAttentionLayer.h>
 #include <NeoML/Dnn/Layers/ObjectNormalizationLayer.h>
+#include <NeoML/Dnn/Layers/ParameterLayer.h>
 #include <NeoML/Dnn/Layers/PoolingLayer.h>
 #include <NeoML/Dnn/Layers/RecurrentLayer.h>
 #include <NeoML/Dnn/Layers/QrnnLayer.h>

diff --git a/NeoML/src/CMakeLists.txt b/NeoML/src/CMakeLists.txt
@@ -50,6 +50,7 @@ set(NeoML_SOURCES_COMPACT
     Dnn/Layers/MultiheadAttentionLayer.cpp
     Dnn/Layers/MultiHingeLossLayer.cpp
     Dnn/Layers/ObjectNormalizationLayer.cpp
+    Dnn/Layers/ParameterLayer.cpp
     Dnn/Layers/PoolingLayer.cpp
     Dnn/Layers/RecurrentLayer.cpp
     Dnn/Layers/RleConvLayer.cpp
@@ -292,6 +293,7 @@ set(NeoML_HEADERS_COMPACT
     ../include/NeoML/Dnn/Layers/MultichannelLookupLayer.h
     ../include/NeoML/Dnn/Layers/MultiheadAttentionLayer.h
     ../include/NeoML/Dnn/Layers/ObjectNormalizationLayer.h
+    ../include/NeoML/Dnn/Layers/ParameterLayer.h
     ../include/NeoML/Dnn/Layers/PoolingLayer.h
     ../include/NeoML/Dnn/Layers/RecurrentLayer.h
     ../include/NeoML/Dnn/Layers/QrnnLayer.h

diff --git a/NeoML/src/Dnn/Dnn.cpp b/NeoML/src/Dnn/Dnn.cpp
@@ -43,6 +43,7 @@ limitations under the License.
 #include <NeoML/Dnn/Layers/MultichannelLookupLayer.h>
 #include <NeoML/Dnn/Layers/MultiheadAttentionLayer.h>
 #include <NeoML/Dnn/Layers/ObjectNormalizationLayer.h>
+#include <NeoML/Dnn/Layers/ParameterLayer.h>
 #include <NeoML/Dnn/Layers/PoolingLayer.h>
 #include <NeoML/Dnn/Layers/RecurrentLayer.h>
 #include <NeoML/Dnn/Layers/QrnnLayer.h>
@@ -285,6 +286,7 @@ REGISTER_NEOML_LAYER( CMatrixMultiplicationLayer, "NeoMLDnnMatrixMultiplicationL
 REGISTER_NEOML_LAYER( CMobileNetV2BlockLayer, "NeoMLDnnMobileNetV2BlockLayer" )
 REGISTER_NEOML_LAYER( CMultiheadAttentionLayer, "NeoMLDnnMultiheadAttentionLayer" )
 REGISTER_NEOML_LAYER( CObjectNormalizationLayer, "NeoMLDnnObjectNormalizationLayer" )
+REGISTER_NEOML_LAYER( CParameterLayer, "NeoMLDnnParameterLayer" )
 REGISTER_NEOML_LAYER( CQrnnFPoolingLayer, "NeoMLDnnQrnnFPoolingLayer" )
 REGISTER_NEOML_LAYER( CQrnnIfPoolingLayer, "NeoMLDnnQrnnIfPoolingLayer" )
 REGISTER_NEOML_LAYER( CQrnnLayer, "NeoMLDnnQrnnLayer" )

diff --git a/NeoML/src/Dnn/Layers/ParameterLayer.cpp b/NeoML/src/Dnn/Layers/ParameterLayer.cpp
@@ -0,0 +1,124 @@
+/* Copyright © 2024 ABBYY Production LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+--------------------------------------------------------------------------------------------------------------*/
+
+#include <common.h>
+#pragma hdrstop
+
+#include <NeoML/Dnn/Layers/ParameterLayer.h>
+#include <NeoMathEngine/NeoMathEngine.h>
+
+namespace NeoML {
+
+void CParameterLayer::AllocateOutputBlobs()
+{
+	outputBlobs[0] = paramBlobs[0];
+}
+
+void CParameterLayer::SetBlob(CDnnBlob* _blob)
+{
+	bool sameBlob = _blob == paramBlobs[0].Ptr();
+	paramBlobs[0] = _blob;
+
+	if (!outputDescs.IsEmpty()) {
+		if (paramBlobs[0]->GetDataType() != outputDescs[0].GetDataType()
+			|| !paramBlobs[0]->GetDesc().HasEqualDimensions(outputDescs[0]))
+		{
+			outputDescs[0] = paramBlobs[0]->GetDesc();
+			ForceReshape();
+		} else {
+			sameBlob = false;
+		}
+	}
+
+	if ( !outputBlobs.IsEmpty() && !sameBlob ) {
+		outputBlobs[0] = 0;
+	}
+}
+
+void CParameterLayer::SetBlobDesc(const CBlobDesc& _desc)
+{
+	bool isReshapeNeeded = desc.GetDataType() == CT_Invalid
+		|| !desc.HasEqualDimensions(_desc)
+		|| desc.GetDataType() != _desc.GetDataType();
+
+	desc = _desc;
+
+	if( isReshapeNeeded ) {
+		paramBlobs[0] = 0;
+		ForceReshape();
+		if( !outputBlobs.IsEmpty() ) {
+			outputBlobs[0] = 0;
+		}
+	}
+}
+
+void CParameterLayer::Reshape()
+{
+	CheckOutputs();
+	CheckLayerArchitecture(GetInputCount() == 0, "layer must not have inputs");
+	CheckLayerArchitecture(GetOutputCount() == 1, "layer has more than 1 output");
+
+	if( paramBlobs[0].Ptr() == nullptr ) {
+		paramBlobs[0] = CDnnBlob::CreateBlob(MathEngine(), desc);
+		InitializeParamBlob(0, *paramBlobs[0], paramBlobs[0]->GetDataSize());
+	}
+
+	outputDescs[0] = paramBlobs[0]->GetDesc();
+}
+
+void CParameterLayer::RunOnce()
+{
+	// Is done while AllocateOutputBlobs
+}
+
+void CParameterLayer::LearnOnce()
+{
+	// Layer's derivative is one => equal to outputDiff
+	paramDiffBlobs[0]->Add( outputDiffBlobs[0] );
+}
+
+void CParameterLayer::BackwardOnce()
+{
+	// Skip for this layer
+}
+
+void CParameterLayer::Serialize(CArchive& archive)
+{
+	archive.SerializeVersion( 0 );
+	CBaseLayer::Serialize(archive);
+
+	if(archive.IsStoring()) {
+		for(TBlobDim d = TBlobDim(0); d < BD_Count; ++d) {
+			archive << desc.DimSize(d);
+		}
+	} else if(archive.IsLoading()) {
+		CBlobDesc desc(CT_Float);
+		for(TBlobDim d = TBlobDim(0); d < BD_Count; ++d) {
+			int size;
+			archive >> size;
+			desc.SetDimSize(d, size);
+		}
+	}
+}
+
+CParameterLayer* Parameter( CDnn& dnn, const char* name, CDnnBlob* blob ) {
+	CPtr<CParameterLayer> param = new CParameterLayer(dnn.GetMathEngine());
+	param->SetName(name);
+	param->SetBlob(blob);
+	dnn.AddLayer(*param);
+	return param;
+}
+
+} // namespace NeoML
diff --git a/NeoML/test/data/LayersSerializationTestData/NeoMLDnnParameterLayer.arch b/NeoML/test/data/LayersSerializationTestData/NeoMLDnnParameterLayer.arch
diff --git a/NeoML/test/src/CMakeLists.txt b/NeoML/test/src/CMakeLists.txt
@@ -26,6 +26,7 @@ target_sources(${PROJECT_NAME} INTERFACE
     ${CMAKE_CURRENT_SOURCE_DIR}/MobileNetV3BlockTest.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/OnnxLayerTest.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/OptimizerFunctionsTest.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ParameterLayerTest.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/PCATest.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/RandomProblem.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/RandomProblem.h

diff --git a/NeoML/test/src/DnnLayersSerializationTest.cpp b/NeoML/test/src/DnnLayersSerializationTest.cpp
@@ -155,6 +155,7 @@ GTEST_TEST( SerializeToFile, BaseLayerSerialization )
 	serializeToFile<CNotLayer>( "NeoMLDnnNotLayer" );
 	serializeToFile<CErfLayer>( "NeoMLDnnErfLayer" );
 	serializeToFile<CLessLayer>( "NeoMLDnnLessLayer" );
+	serializeToFile<CParameterLayer>( "NeoMLDnnParameterLayer" );
 	serializeToFile<CEqualLayer>( "NeoMLDnnEqualLayer" );
 	serializeToFile<CGlobalSumPoolingLayer>( "NeoMLDnnGlobalSumPoolingLayer" );
 	serializeToFile<CWhereLayer>( "NeoMLDnnWhereLayer" );
@@ -277,6 +278,7 @@ GTEST_TEST( SerializeFromFile, BaseLayerSerialization )
 	checkSerializeLayer<CBaseLayer>( "NeoMLDnnNotLayer" );
 	checkSerializeLayer<CBaseLayer>( "NeoMLDnnErfLayer" );
 	checkSerializeLayer<CBaseLayer>( "NeoMLDnnLessLayer" );
+	checkSerializeLayer<CBaseLayer>( "NeoMLDnnParameterLayer" );
 	checkSerializeLayer<CBaseLayer>( "NeoMLDnnEqualLayer" );
 	checkSerializeLayer<CBaseLayer>( "NeoMLDnnGlobalSumPoolingLayer" );
 	checkSerializeLayer<CBaseLayer>( "NeoMLDnnWhereLayer" );
@@ -1977,6 +1979,42 @@ GTEST_TEST( SerializeFromFile, MultiheadAttentionLayerSerialization )
 
 // ====================================================================================================================
 
+// CParameterLayer
+
+#ifdef GENERATE_SERIALIZATION_FILES
+
+static void setSpecificParams(CParameterLayer& layer)
+{
+	const int batchWidth = 4;
+	const int imageWidth = 5;
+	const int channelCount = 6;
+	CPtr<CDnnBlob> paramData = generateBlob(batchWidth, 1, 1, imageWidth, channelCount);
+	layer.SetBlob( paramData );
+}
+
+GTEST_TEST(SerializeToFile, ParameterLayerSerialization)
+{
+	serializeToFile<CParameterLayer>("NeoMLDnnParameterLayer");
+}
+
+#endif // GENERATE_SERIALIZATION_FILES
+
+template<>
+inline void checkSpecificParams<CParameterLayer>(CParameterLayer& layer)
+{
+	const int batchWidth = 4;
+	const int imageWidth = 5;
+	const int channelCount = 6;
+	checkBlob( *(layer.GetBlob()), batchWidth * imageWidth * channelCount );
+}
+
+GTEST_TEST(SerializeFromFile, ParameterLayerSerialization)
+{
+	checkSerializeLayer<CParameterLayer>("NeoMLDnnParameterLayer");
+}
+
+// ====================================================================================================================
+
 // CPositionalEmbeddingLayer
 
 #ifdef GENERATE_SERIALIZATION_FILES