master commit

Signed-off-by: daniyalaliev <daniial.aliev@abbyy.com>

NeoML/CMakeLists.txt

@@ -20,6 +20,9 @@ endif()
 
 include(Settings)
 include(Version)
+# Set cmake default variables and some usefull variables
+set_global_variables()
+
 if(USE_FINE_OBJECTS)
     include(FineInstall)
 else()
@@ -57,9 +60,6 @@ if(NOT (WIN32 OR CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES
     set(NeoOnnx_BUILD OFF CACHE BOOL "" FORCE)
 endif()
 
-# Set cmake default variables and some usefull variables
-set_global_variables()
-
 if(USE_FINE_OBJECTS)
     find_package(FineObjects)
     if(NOT FineObj_FOUND)

NeoML/include/NeoML/Dnn/DnnBlob.h

@@ -136,6 +136,11 @@ class NEOML_API CDnnBlob : public IObject {
 	// Copies the contents from another blob
 	void CopyFrom(const CDnnBlob* other);
 
+	// Transfers CDnnBlob data from other thread owner to this thread.
+	// By default memory underneath each blob is associated with the thread on which its allocation has occurred.
+	// This method switches this association to the calling thread.
+	void TransferDataToThisThread();
+
 	// Elementwise adds a blob of the same dimensions
 	void Add(const CDnnBlob* other);
 	// Clears the contents

NeoML/include/NeoML/Dnn/DnnSolver.h

@@ -1,4 +1,4 @@
-/* Copyright © 2017-2020 ABBYY Production LLC
+/* Copyright © 2017-2023 ABBYY
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -50,6 +50,9 @@ class NEOML_API CDnnSolver : virtual public IObject {
 	// Upper limit for gradient norm (if set to < 0, that means no limit)
 	float GetMaxGradientNorm() const { return maxGradientNorm; }
 	void SetMaxGradientNorm(float _maxGradientNorm) { maxGradientNorm = _maxGradientNorm; }
+	// Clipping gradient min and max (if set to -FLT_MAX and FLT_MAX, that means no limit)
+	void GetMinMaxGradientClipping( float& min, float& max ) const { min = clipGradientMin; max = clipGradientMax; }
+	void SetMinMaxGradientClipping( float min, float max ) { clipGradientMin = min; clipGradientMax = max; }
 
 	// Serialize to archive
 	virtual void Serialize( CArchive& archive, CDnn& dnn );
@@ -79,6 +82,8 @@ class NEOML_API CDnnSolver : virtual public IObject {
 	float regularizationL2;
 	float regularizationL1;
 	float maxGradientNorm;
+	float clipGradientMin;
+	float clipGradientMax;
 
 	// The blobs sum
 	struct CDiffBlobSum {
@@ -100,8 +105,10 @@ class NEOML_API CDnnSolver : virtual public IObject {
 	// Averages weights over all threads
 	void allReduce( float distributedCoeff );
 
-	// Clips gradients according to the settings
+	// Clips and normalize gradients according to the settings
 	void clipGradients(const CObjectArray<CDnnBlob>& paramDiffBlobs);
+	// Clips gradients
+	void clip( const CObjectArray<CDnnBlob>& paramDiffBlobs );
 
 	// Telling the compiler that we intentionally using two-parameter Serialize instead of one declared in IObject
 	using IObject::Serialize;

NeoML/include/NeoML/Dnn/Layers/TransformerSourceMaskLayer.h

@@ -1,4 +1,4 @@
-/* Copyright � 2017-2022 ABBYY Production LLC
+/* Copyright © 2017-2023 ABBYY
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

NeoML/src/Dnn/BaseLayer.cpp

@@ -417,7 +417,8 @@ void CBaseLayer::reshape()
 
 class CRunOnceTimer {
 public:
-	CRunOnceTimer( bool enable, IMathEngine& mathEngine, int& hitCount, IPerformanceCounters::CCounter::TCounterType& result );
+	CRunOnceTimer( bool enable, IMathEngine& mathEngine, int& hitCount,
+		IPerformanceCounters::CCounter::TCounterType& result );
 	~CRunOnceTimer();
 
 private:
@@ -427,7 +428,7 @@ class CRunOnceTimer {
 
 CRunOnceTimer::CRunOnceTimer( bool enable, IMathEngine& mathEngine, int& hitCount,
 		IPerformanceCounters::CCounter::TCounterType& result ) :
-	counters( enable ? mathEngine.CreatePerformanceCounters() : nullptr ),
+	counters( enable ? mathEngine.CreatePerformanceCounters( true ) : nullptr ),
 	result( result )
 {
 	if( enable ) {

NeoML/src/Dnn/DnnBlob.cpp

@@ -246,6 +246,18 @@ void CDnnBlob::CopyFrom(const CDnnBlob* other)
 	}
 }
 
+void CDnnBlob::TransferDataToThisThread()
+{
+	NeoAssert( dataOwned );
+	NeoAssert( !data.IsNull() );
+	NeoAssert( parent == nullptr );
+	NeoAssert( GetDataType() == CT_Float || GetDataType() == CT_Int );
+
+	const size_t size = GetDataSize()
+		* ( ( GetDataType() == CT_Float ) ? sizeof( float ) : sizeof( int ) );
+	mathEngine.TransferHandleToThisThread( data, size );
+}
+
 void CDnnBlob::Add(const CDnnBlob* other)
 {
 	NeoPresume(other->GetDataSize() == GetDataSize());

NeoML/src/Dnn/DnnSolver.cpp

@@ -162,7 +162,9 @@ CDnnSolver::CDnnSolver( IMathEngine& _mathEngine ) :
 	learningRate( 0.01f ),
 	regularizationL2( 0.f ),
 	regularizationL1( 0.f ),
-	maxGradientNorm( -1.f )
+	maxGradientNorm( -1.f ),
+	clipGradientMin( -FLT_MAX ),
+	clipGradientMax( FLT_MAX )
 {
 }
 
@@ -265,9 +267,33 @@ void CDnnSolver::allReduce( float distributedCoeff )
 	}
 }
 
+void CDnnSolver::clip( const CObjectArray<CDnnBlob>& paramDiffBlobs )
+{
+	if( clipGradientMin <= -FLT_MAX && clipGradientMax >= FLT_MAX ) {
+		return;
+	}
+
+	CFloatHandleStackVar minVar( MathEngine() );
+	minVar.SetValue( clipGradientMin );
+
+	CFloatHandleStackVar maxVar( MathEngine() );
+	maxVar.SetValue( clipGradientMax );
+
+	for( int i = 0; i < paramDiffBlobs.Size(); ++i ) {
+		MathEngine().VectorMinMax( paramDiffBlobs[i]->GetData(), paramDiffBlobs[i]->GetData(),
+			paramDiffBlobs[i]->GetDataSize(), minVar, maxVar );
+	}
+}
+
 void CDnnSolver::clipGradients(const CObjectArray<CDnnBlob>& paramDiffBlobs)
 {
-	if(maxGradientNorm < 0 || paramDiffBlobs.Size() == 0) {
+	if(paramDiffBlobs.Size() == 0) {
+		return;
+	}
+
+	clip( paramDiffBlobs );
+
+	if( maxGradientNorm < 0 ) {
 		return;
 	}
 
@@ -281,6 +307,7 @@ void CDnnSolver::clipGradients(const CObjectArray<CDnnBlob>& paramDiffBlobs)
 			paramDiffBlobs[i]->GetDataSize(), tempVar.GetHandle());
 		MathEngine().VectorAdd(gradVar.GetHandle(), tempVar.GetHandle(), gradVar.GetHandle(), 1);
 	}
+	NeoPresume( isfinite( gradVar.GetValue() ) );
 	MathEngine().VectorSqrt(gradVar.GetHandle(), gradVar.GetHandle(), 1);
 
 	// Calculate scale
@@ -295,11 +322,11 @@ void CDnnSolver::clipGradients(const CObjectArray<CDnnBlob>& paramDiffBlobs)
 	}
 }
 
-static const int DnnSolverVersion = 0;
+static const int DnnSolverVersion = 1;
 
 void CDnnSolver::Serialize( CArchive& archive, CDnn& dnn )
 {
-	archive.SerializeVersion( DnnSolverVersion );
+	const int version = archive.SerializeVersion( DnnSolverVersion );
 	if( archive.IsStoring() ) {
 		CMap<CBaseLayer*, CString> layerPtrToId;
 		mapLayerPtrToId( dnn, layerPtrToId );
@@ -321,6 +348,7 @@ void CDnnSolver::Serialize( CArchive& archive, CDnn& dnn )
 			SerializeBlobs( mathEngine, archive, layerToGradientHistory.GetValue( pos ) );
 		}
 		archive << learningRate << regularizationL1 << regularizationL2 << maxGradientNorm;
+		archive << clipGradientMin << clipGradientMax;
 	} else {
 		CMap<CString, CBaseLayer*> layerIdToPtr;
 		mapLayerIdToPtr( dnn, layerIdToPtr );
@@ -347,6 +375,12 @@ void CDnnSolver::Serialize( CArchive& archive, CDnn& dnn )
 			SerializeBlobs( mathEngine, archive, layerToGradientHistory.GetOrCreateValue( layerIdToPtr[layerId] ) );
 		}
 		archive >> learningRate >> regularizationL1 >> regularizationL2 >> maxGradientNorm;
+		if( version >= 1 ) {
+			archive >> clipGradientMin >> clipGradientMax;
+		} else {
+			clipGradientMin = -FLT_MAX;
+			clipGradientMax = FLT_MAX;
+		}
 	}
 }
 

NeoML/src/Dnn/Layers/ImageAndPixelConversionLayer.cpp

@@ -198,13 +198,23 @@ void CPixelToImageLayer::BackwardOnce()
 void CPixelToImageLayer::SetImageHeight( int newHeight )
 {
 	NeoAssert( newHeight > 0 );
+	if( newHeight == imageHeight ) {
+		return;
+	}
+
 	imageHeight = newHeight;
+	ForceReshape();
 }
 
 void CPixelToImageLayer::SetImageWidth( int newWidth )
 {
 	NeoAssert( newWidth > 0 );
+	if( newWidth == imageWidth ) {
+		return;
+	}
+
 	imageWidth = newWidth;
+	ForceReshape();
 }
 
 CLayerWrapper<CPixelToImageLayer> PixelToImage( int imageHeight, int imageWidth )

NeoML/src/Dnn/Layers/TransformerSourceMaskLayer.cpp

@@ -1,4 +1,4 @@
-/* Copyright � 2017-2022 ABBYY Production LLC
+/* Copyright © 2017-2023 ABBYY
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

NeoML/src/TraditionalML/Score.cpp

@@ -59,7 +59,6 @@ double F1Score( const CArray<CClassificationResult>& classificationResult, const
 	// *Positive means the objects that got the +1 label, *Negative the -1 label
 	double truePositive = 0;
 	double falsePositive = 0;
-	double trueNegative = 0;
 	double falseNegative = 0;
 
 	for( int i = 0; i < classificationResult.Size(); ++i ) {
@@ -72,9 +71,7 @@ double F1Score( const CArray<CClassificationResult>& classificationResult, const
 				}
 				break;
 			case -1:
-				if( GetBinaryClass( problem->GetClass( i ) ) == -1 ) {
-					trueNegative++;
-				} else {
+				if( GetBinaryClass( problem->GetClass( i ) ) != -1 ) {
 					falseNegative++;
 				}
 				break;

NeoML/src/TraditionalML/UnigramTrainer.cpp

@@ -199,7 +199,6 @@ void CUnigramTrainer::fillTrainDict( const CWordDictionary& frequencyDict )
 void CUnigramTrainer::createInitialVocab()
 {
 	CTrieNode<CTrieCounterData> substringTrie;
-	int64_t totalCount = 0;
 
 	for( int i = 0; i < trainDict.Size(); ++i ) {
 		const CString& word = trainDict.GetWord( i );
@@ -225,7 +224,6 @@ void CUnigramTrainer::createInitialVocab()
 					data.End = end;
 				}
 				data.Count += count;
-				totalCount += count;
 
 				if( tokenLength++ == maxSubwordLength ) {
 					break;