[NeoML]TiedEmbedding access through composite layer (#1031)

* fixed hierarchical clusterization with ward (#998) Signed-off-by: Daniyal Aliev <daniial.aliev@abbyy.com> Co-authored-by: Valeriy Fedyunin <valery.fedyunin@abbyy.com> * master commit Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * initial changes Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * need to test Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * added path tied embedding Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * added path tied embedding Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * need to add tests Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * tests Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * final commit Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> * docs Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> --------- Signed-off-by: Daniyal Aliev <daniial.aliev@abbyy.com> Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com> Co-authored-by: Valeriy Fedyunin <valery.fedyunin@abbyy.com> Co-authored-by: daniyalaliev <daniial.aliev@abbyy.com>
neoml-lib · Feb 27, 2024 · c8ba3be · c8ba3be
1 parent ddbcda7
commit c8ba3be
Show file tree

Hide file tree

Showing 10 changed files with 269 additions and 26 deletions.
diff --git a/NeoML/docs/en/API/NN/TiedEmbeddingsLayer.md b/NeoML/docs/en/API/NN/TiedEmbeddingsLayer.md
@@ -23,6 +23,16 @@ void SetEmbeddingsLayerName( const char* name )
 ```
 Embeddings layer `name`. Only [CMultichannelLookupLayer](DiscreteFeaturesLayers/MultichannelLookupLayer.md) is allowed.
 
+### EmbeddingsLayerPath
+
+```c++
+void SetEmbeddingsLayerPath( const CArray<CString>& path )
+```
+
+If embedding layer is inside of [CompositeLayer](https://github.com/neoml-lib/neoml/blob/master/NeoML/include/NeoML/Dnn/Layers/CompositeLayer.h), then you need to provide full path to it.
+
+Example: {"composite1", "composite2", ..., "embeddingName"}
+
 ### ChannelIndex
 
 ```c++

diff --git a/NeoML/docs/ru/API/NN/TiedEmbeddingsLayer.md b/NeoML/docs/ru/API/NN/TiedEmbeddingsLayer.md
@@ -23,6 +23,16 @@ void SetEmbeddingsLayerName( const char* name )
 ```
 Использовать слой эмбеддингов `name`. Поддерживается только [CMultichannelLookupLayer](DiscreteFeaturesLayers/MultichannelLookupLayer.md).
 
+### EmbeddingsLayerPath
+
+```c++
+void SetEmbeddingsLayerPath( const CArray<CString>& path )
+```
+
+Если слой эмбеддингов находится в [CompositeLayer](https://github.com/neoml-lib/neoml/blob/master/NeoML/include/NeoML/Dnn/Layers/CompositeLayer.h), то надо указать полный путь до него.
+
+Пример: {"composite1", "composite2", ..., "embeddingName"}
+
 ### ChannelIndex
 
 ```c++

diff --git a/NeoML/include/NeoML/Dnn/Dnn.h b/NeoML/include/NeoML/Dnn/Dnn.h
@@ -1,4 +1,4 @@
-/* Copyright © 2017-2023 ABBYY
+/* Copyright © 2017-2024 ABBYY
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -455,6 +455,8 @@ class NEOML_API CDnnLayerGraph {
 	virtual void GetLayerList( CArray<char const*>& layerList ) const = 0;
 	virtual CPtr<CBaseLayer> GetLayer( const char* name ) = 0;
 	virtual CPtr<const CBaseLayer> GetLayer( const char* name ) const = 0;
+	virtual CPtr<CBaseLayer> GetLayer( const CArray<CString>& path ) = 0;
+	virtual CPtr<const CBaseLayer> GetLayer( const CArray<CString>& path ) const = 0;
 	virtual bool HasLayer( const char* name ) const = 0;
 
 	void AddLayer(CBaseLayer& layer);
@@ -517,6 +519,8 @@ class NEOML_API CDnn : public CDnnLayerGraph {
 	void GetLayerList( CArray<const char*>& layerList ) const override;
 	CPtr<CBaseLayer> GetLayer( const char* name ) override;
 	CPtr<const CBaseLayer> GetLayer( const char* name ) const override;
+	CPtr<CBaseLayer> GetLayer(const CArray<CString>& path) override;
+	CPtr<const CBaseLayer> GetLayer(const CArray<CString>& path) const override;
 	bool HasLayer( const char* name ) const override { return layerMap.Has( name ); }
 
 	// Runs the network: all data from the input blobs is used

diff --git a/NeoML/include/NeoML/Dnn/Layers/CompositeLayer.h b/NeoML/include/NeoML/Dnn/Layers/CompositeLayer.h
@@ -1,4 +1,4 @@
-/* Copyright © 2017-2023 ABBYY
+/* Copyright © 2017-2024 ABBYY
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -58,7 +58,9 @@ class NEOML_API CCompositeLayer : public CBaseLayer, public CDnnLayerGraph {
 	int GetLayerCount() const override { return layers.Size(); }
 	void GetLayerList(CArray<const char*>& layerList) const override;
 	CPtr<CBaseLayer> GetLayer(const char* name) override;
+	CPtr<CBaseLayer> GetLayer(const CArray<CString>& path) override;
 	CPtr<const CBaseLayer> GetLayer(const char* name) const override;
+	CPtr<const CBaseLayer> GetLayer(const CArray<CString>& path) const override;
 	bool HasLayer(const char* name) const override;
 
 	// Returns the total size of the output blobs

diff --git a/NeoML/include/NeoML/Dnn/Layers/TiedEmbeddingsLayer.h b/NeoML/include/NeoML/Dnn/Layers/TiedEmbeddingsLayer.h
@@ -1,4 +1,4 @@
-/* Copyright © 2017-2020 ABBYY Production LLC
+/* Copyright © 2017-2024 ABBYY Production LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,6 +20,8 @@ limitations under the License.
 
 namespace NeoML {
 
+class CMultichannelLookupLayer;
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 // Tied embeddings layer.  https://arxiv.org/pdf/1608.05859.pdf
@@ -31,10 +33,18 @@ class NEOML_API CTiedEmbeddingsLayer : public CBaseLayer {
 
 	void Serialize( CArchive& archive ) override;
 
-	// Embeddings layer name from which we take the matrix.
+	// Methods to get/set embeddings layer name from which we take the matrix.
+	// Only CMultichannelLookupLayer is supported.
+	// Use this method if the lookupLayer is in the same level of the dnn (in the same composite layer)
+	const char* GetEmbeddingsLayerName() const { return embeddingPath.Last(); }
+	void SetEmbeddingsLayerName(const char* name) { embeddingPath = { name }; }
+
+	// Methods to get/set embeddings layer path from which we take the matrix.
 	// Only CMultichannelLookupLayer is supported.
-	const char* GetEmbeddingsLayerName() const { return embeddingsLayerName; }
-	void SetEmbeddingsLayerName( const char* name ) { embeddingsLayerName = name; }
+	// Use this method if the lookupLayer is in the nested level of the dnn (in some nested composite layer)
+	const CArray<CString>& GetEmbeddingsLayerPath() const { return embeddingPath; }
+	void SetEmbeddingsLayerPath(const CArray<CString>& path) { path.CopyTo(embeddingPath); }
+
 	// Channel index in embeddings layer.
 	int GetChannelIndex() const { return channelIndex; }
 	void SetChannelIndex( int val );
@@ -48,12 +58,15 @@ class NEOML_API CTiedEmbeddingsLayer : public CBaseLayer {
 	int BlobsForLearn() const override { return TInputBlobs; }
 
 private:
-	// Embedding layer name from which we take the matrix.
-	CString embeddingsLayerName;
+	// Path for embedding layer from which matrix is taken
+	// Now it contains the path as array
+	// So in case of no composite layer it is gonna be { "embeddingName" }
+	CArray<CString> embeddingPath;
 	// Channel index in embedding layer.
 	int channelIndex;
 
 	const CDnnBlob* getEmbeddingsTable() const;
+	CMultichannelLookupLayer* getLookUpLayer() const;
 };
 
 // Tied embeddings.

diff --git a/NeoML/src/Dnn/Dnn.cpp b/NeoML/src/Dnn/Dnn.cpp
@@ -1,4 +1,4 @@
-/* Copyright © 2017-2023 ABBYY
+/* Copyright © 2017-2024 ABBYY
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -452,6 +452,28 @@ CPtr<const CBaseLayer> CDnn::GetLayer( const char* name ) const
 	return layerMap.Get( name );
 }
 
+CPtr<CBaseLayer> CDnn::GetLayer( const CArray<CString>& path)
+{
+	CheckArchitecture(path.Size() > 0, "NULL", "can not find layer - empty path");
+	if (path.Size() == 1) {
+		return GetLayer(path[0]);
+	} else {
+		CheckArchitecture(layerMap.Has(path[0]), path[0], "layer is not in this dnn");
+		CPtr<CCompositeLayer> currComp = CheckCast<CCompositeLayer>( GetLayer(path[0]).Ptr() );
+		for (int i = 1; i < path.Size() - 1; ++i) {
+			CheckArchitecture(currComp->HasLayer(path[i]), path[i], "layer is not in this composite layer");
+			currComp = CheckCast<CCompositeLayer>(currComp->GetLayer(path[i]).Ptr());
+		}
+		CheckArchitecture(currComp->HasLayer(path.Last()), path.Last(), "layer is not contained by this path");
+		return currComp->GetLayer(path.Last());
+	}
+}
+
+CPtr<const CBaseLayer> CDnn::GetLayer(const CArray<CString>& path) const
+{
+	return const_cast<CDnn*>(this)->GetLayer(path);
+}
+
 void CDnn::AddLayerImpl( CBaseLayer& layer )
 {
 	layer.CheckLayerArchitecture( !layerMap.Has( layer.GetName() ), "layer already in this dnn" );

diff --git a/NeoML/src/Dnn/Layers/CompositeLayer.cpp b/NeoML/src/Dnn/Layers/CompositeLayer.cpp
@@ -1,4 +1,4 @@
-/* Copyright © 2017-2023 ABBYY
+/* Copyright © 2017-2024 ABBYY
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -202,6 +202,22 @@ CPtr<const CBaseLayer> CCompositeLayer::GetLayer(const char* name) const
 	return layerMap.Get(name);
 }
 
+CPtr<CBaseLayer> CCompositeLayer::GetLayer(const CArray<CString>& path)
+{
+	CPtr<CCompositeLayer> currComp = this;
+	for(int i = 0; i < path.Size() - 1; ++i ) {
+		CheckArchitecture(currComp->layerMap.Has(path[i]), path[i], "layer is not in this composite layer");
+		currComp = CheckCast<CCompositeLayer>( currComp->GetLayer(path[i]).Ptr() );
+	}
+	CheckArchitecture(currComp->HasLayer(path.Last()), path.Last(), "layer is not contained by this path");
+	return currComp->GetLayer(path.Last());
+}
+
+CPtr<const CBaseLayer> CCompositeLayer::GetLayer(const CArray<CString>& path) const
+{
+	return const_cast<CCompositeLayer*>(this)->GetLayer(path);
+}
+
 bool CCompositeLayer::HasLayer(const char* name) const
 {
 	return layerMap.Has(name);

diff --git a/NeoML/src/Dnn/Layers/TiedEmbeddingsLayer.cpp b/NeoML/src/Dnn/Layers/TiedEmbeddingsLayer.cpp
@@ -1,4 +1,4 @@
-/* Copyright © 2017-2020 ABBYY Production LLC
+/* Copyright © 2017-2024 ABBYY Production LLC
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -36,29 +36,33 @@ void CTiedEmbeddingsLayer::SetChannelIndex( int val )
 	channelIndex = val;
 }
 
-static const int CnnTiedEmbeddingsLayerVersion = 2000;
+static const int CnnTiedEmbeddingsLayerVersion = 2001;
 
 void CTiedEmbeddingsLayer::Serialize( CArchive& archive )
 {
-	archive.SerializeVersion( CnnTiedEmbeddingsLayerVersion, CDnn::ArchiveMinSupportedVersion );
+	int version = archive.SerializeVersion(CnnTiedEmbeddingsLayerVersion, CDnn::ArchiveMinSupportedVersion);
 	CBaseLayer::Serialize( archive );
 
-	archive.Serialize( embeddingsLayerName );
+	if (version < 2001 && archive.IsLoading()) {
+		CString embeddingLayerName;
+		archive.Serialize(embeddingLayerName);
+		embeddingPath = { embeddingLayerName };
+	}
+	else {
+		archive.Serialize(embeddingPath);
+	}
+
 	archive.Serialize( channelIndex );
 }
 
 void CTiedEmbeddingsLayer::Reshape()
 {
 	CheckInputs();
+	const CMultichannelLookupLayer* embeddingsLayer = getLookUpLayer();
 
-	CheckLayerArchitecture( GetDnn()->HasLayer( embeddingsLayerName ),
-		"Network does not contain embeddings layer with that name." );
-	const CMultichannelLookupLayer* embeddingsLayer = dynamic_cast<CMultichannelLookupLayer*>(
-		GetDnn()->GetLayer( embeddingsLayerName ).Ptr() );
 	CheckLayerArchitecture( embeddingsLayer != 0, "The layer is not an embedding layer." );
 
-	const int embeddingsChannelsCount = CheckCast<CMultichannelLookupLayer>(
-		GetDnn()->GetLayer( embeddingsLayerName ) )->GetDimensions().Size();
+	const int embeddingsChannelsCount = embeddingsLayer->GetDimensions().Size();
 	CheckLayerArchitecture( channelIndex < embeddingsChannelsCount,
 		"Wrong channgel index for embeddings" );
 
@@ -129,9 +133,7 @@ void CTiedEmbeddingsLayer::LearnOnce()
 		diffBlob->Clear();
 	}
 
-	CMultichannelLookupLayer* embeddingsLayer =
-		CheckCast<CMultichannelLookupLayer>( GetDnn()->GetLayer( embeddingsLayerName ) );
-
+	CMultichannelLookupLayer* embeddingsLayer = getLookUpLayer();
 	CObjectArray<CDnnBlob> totalDiffBlobs;
 	const int channelsCount = embeddingsLayer->GetDimensions().Size();
 	for( int i = 0; i < channelsCount; i++ ) {
@@ -152,9 +154,15 @@ const CDnnBlob* CTiedEmbeddingsLayer::getEmbeddingsTable() const
 {
 	NeoAssert( channelIndex >= 0 );
 
-	const CMultichannelLookupLayer* embeddingsLayer =
-		CheckCast<CMultichannelLookupLayer>( GetDnn()->GetLayer( embeddingsLayerName ) );
-	return embeddingsLayer->GetEmbeddings( channelIndex );
+	return getLookUpLayer()->GetEmbeddings( channelIndex );
+}
+
+CMultichannelLookupLayer* CTiedEmbeddingsLayer::getLookUpLayer() const
+{
+	CMultichannelLookupLayer* embeddingsLayer;
+	embeddingsLayer = CheckCast<CMultichannelLookupLayer>(
+		const_cast<CDnn*>(GetDnn())->GetLayer(embeddingPath).Ptr());
+	return embeddingsLayer;
 }
 
 CLayerWrapper<CTiedEmbeddingsLayer> TiedEmbeddings( const char* name, int channel )

diff --git a/NeoML/test/src/CMakeLists.txt b/NeoML/test/src/CMakeLists.txt
@@ -36,6 +36,7 @@ target_sources(${PROJECT_NAME} INTERFACE
     ${CMAKE_CURRENT_SOURCE_DIR}/TestFixture.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/TestParams.h
     ${CMAKE_CURRENT_SOURCE_DIR}/TestParams.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/TiedEmbeddingTest.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/TransformerSourceMaskTest.cpp
 )