From 8e394b1cf9b5f6959e4b860795b25df5f79a6679 Mon Sep 17 00:00:00 2001 From: Boris Zimka <4560910+zimka@users.noreply.github.com> Date: Tue, 7 May 2024 19:56:38 +0200 Subject: [PATCH 1/4] [NeoOnnx] DepthToSpace operator (#1056) * Add DepthToSpaceOperator without separate source code file Signed-off-by: Boris Zimka * Clean DepthToSpaceOperator Signed-off-by: Boris Zimka * Fix review comments Signed-off-by: Boris Zimka --------- Signed-off-by: Boris Zimka --- NeoOnnx/src/CMakeLists.txt | 2 + NeoOnnx/src/Operator.cpp | 2 + .../src/Operators/DepthToSpaceOperator.cpp | 76 +++++++++++++++++++ NeoOnnx/src/Operators/DepthToSpaceOperator.h | 36 +++++++++ .../test/Python/neoml_onnx_backend_test.py | 2 +- 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 NeoOnnx/src/Operators/DepthToSpaceOperator.cpp create mode 100644 NeoOnnx/src/Operators/DepthToSpaceOperator.h diff --git a/NeoOnnx/src/CMakeLists.txt b/NeoOnnx/src/CMakeLists.txt index 7beec1c6dc..c2272ab160 100644 --- a/NeoOnnx/src/CMakeLists.txt +++ b/NeoOnnx/src/CMakeLists.txt @@ -32,6 +32,7 @@ target_sources( ${PROJECT_NAME} PRIVATE Operators/ConvOperator.cpp Operators/ConvTransposeOperator.cpp Operators/CumSumOperator.cpp + Operators/DepthToSpaceOperator.cpp Operators/DropoutOperator.cpp Operators/ExpandOperator.cpp Operators/FlattenOperator.cpp @@ -90,6 +91,7 @@ target_sources( ${PROJECT_NAME} PRIVATE Operators/ConvOperator.h Operators/ConvTransposeOperator.h Operators/CumSumOperator.h + Operators/DepthToSpaceOperator.h Operators/DropoutOperator.h Operators/EltwiseOperator.h Operators/ExpandOperator.h diff --git a/NeoOnnx/src/Operator.cpp b/NeoOnnx/src/Operator.cpp index dd8dd8cca3..a09e4aaa5d 100644 --- a/NeoOnnx/src/Operator.cpp +++ b/NeoOnnx/src/Operator.cpp @@ -33,6 +33,7 @@ limitations under the License. #include "Operators/ConvOperator.h" #include "Operators/ConvTransposeOperator.h" #include "Operators/CumSumOperator.h" +#include "Operators/DepthToSpaceOperator.h" #include "Operators/DropoutOperator.h" #include "Operators/EltwiseOperator.h" #include "Operators/ExpandOperator.h" @@ -130,6 +131,7 @@ REGISTER_OPERATOR( CConvOperator, "Conv" ) REGISTER_OPERATOR( CConvTransposeOperator, "ConvTranspose" ) REGISTER_OPERATOR( CCumSumOperator, "CumSum" ) REGISTER_OPERATOR( CEltwiseOperator, "Div" ) +REGISTER_OPERATOR( CDepthToSpaceOperator, "DepthToSpace" ) REGISTER_OPERATOR( CDropoutOperator, "Dropout" ) REGISTER_OPERATOR( CEluOperator, "Elu" ) REGISTER_OPERATOR( CEltwiseOperator, "Equal" ) diff --git a/NeoOnnx/src/Operators/DepthToSpaceOperator.cpp b/NeoOnnx/src/Operators/DepthToSpaceOperator.cpp new file mode 100644 index 0000000000..9f9954208f --- /dev/null +++ b/NeoOnnx/src/Operators/DepthToSpaceOperator.cpp @@ -0,0 +1,76 @@ +/* Copyright © 2017-2024 ABBYY + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--------------------------------------------------------------------------------------------------------------*/ + +#include "../common.h" +#pragma hdrstop + +#include "NeoOnnxCheck.h" + +#include "onnx.pb.h" + +#include "DepthToSpaceOperator.h" + +using namespace NeoML; + +namespace NeoOnnx { + +CDepthToSpaceOperator::CDepthToSpaceOperator( const onnx::NodeProto& depthToSpace, int opsetVersion ) : + CLayerOperator( depthToSpace, opsetVersion ) +{ + // v1 - original + // v11 - added "mode" attribute + // v13 - bfloat16 is supported + CheckNeoOnnxSupport( OpsetVersion >= 1 && OpsetVersion <= MaxOpsetVersion, "opset version", *this ); + CheckOnnxProtocol( OutputCount() == 1, "operator must have 1 output", *this ); + CheckOnnxProtocol( InputCount() == 1, "operator must have 1 input", *this ); + GetAttribute( "blocksize", blockSize ); + CheckNeoOnnxSupport( blockSize > 1, "blocksize attribute must be more than 1", *this ); + + CString mode {"DCR"}; + if( opsetVersion > 11 ) { + // before v11 DCR was used by default + GetAttribute( "mode", mode ); + }; + + // For DepthToSpace 'mode' can be either DCR (depth-column-row) or CRD (column-row-depth) + // ONNX uses channel-first layout, so depending on mode operator reshapes input as following: + // CRD: 'B (C block_size block_size) H W -> B C (H block_size) (W block_size)' + // DCR: 'B (block_size block_size C) H W -> B C (H block_size) (W block_size)' + // NeoML CDepthToSpaceLayer uses channel-last layout and rearranges input as following: + // 'batch_length B list_size H W 1 (block_size block_size C) -> + // batch_length B list_size (H block_size) (W block_size) C' + // CDepthToSpaceLayer is equivalent to DCR, while CRD is not supported by NeoML + CheckNeoOnnxSupport( mode == "DCR", "DCR is supported, CRD is not supported", *this ); +} + +void CDepthToSpaceOperator::AddLayers( const CTensorArray& inputs, CDnn& dnn, CTensorArray& outputs ) const +{ + CheckNoShapeInputs( inputs ); + CheckOnnxProtocol( inputs[0] != nullptr, "input can't be optional", *this ); + CheckNeoOnnxSupport( inputs[0]->DimCount() == 4, "can convert only 4d input", *this ); + + CPtr userInput = AsUserTensor( + *ConvertTensor( *inputs[0], CNeoMLImageLayoutValidator() ), Name() + "_Source", dnn ); + + CPtr depthToSpace = new CDepthToSpaceLayer( dnn.GetMathEngine() ); + depthToSpace->SetName( Name() ); + depthToSpace->SetBlockSize( blockSize ); + depthToSpace->Connect( 0, *userInput->Layer(), userInput->OutputIndex() ); + dnn.AddLayer( *depthToSpace ); + + outputs.Add( new CUserTensor( userInput->Layout(), CLayerOutput( depthToSpace, 0 ) ) ); +} + +} // namespace NeoOnnx \ No newline at end of file diff --git a/NeoOnnx/src/Operators/DepthToSpaceOperator.h b/NeoOnnx/src/Operators/DepthToSpaceOperator.h new file mode 100644 index 0000000000..ba2b2ce72d --- /dev/null +++ b/NeoOnnx/src/Operators/DepthToSpaceOperator.h @@ -0,0 +1,36 @@ +/* Copyright © 2017-2024 ABBYY + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--------------------------------------------------------------------------------------------------------------*/ + +#pragma once + +#include "../LayerOperator.h" + +namespace NeoOnnx { + +// DepthToSpace operator +class CDepthToSpaceOperator : public CLayerOperator { +public: + CDepthToSpaceOperator( const onnx::NodeProto& depthToSpace, int opsetVersion ); + +protected: + // CLayerOperator methods + void AddLayers( const CTensorArray& inputs, CDnn& dnn, CTensorArray& outputs ) const override; + +private: + int blockSize; +}; + +} // namespace NeoOnnx + diff --git a/NeoOnnx/test/Python/neoml_onnx_backend_test.py b/NeoOnnx/test/Python/neoml_onnx_backend_test.py index 79a6f24513..d47038e429 100644 --- a/NeoOnnx/test/Python/neoml_onnx_backend_test.py +++ b/NeoOnnx/test/Python/neoml_onnx_backend_test.py @@ -48,7 +48,7 @@ backend_test.exclude('test_cos_') # Cos backend_test.exclude('test_cosh_') # Cosh backend_test.exclude('deform_conv') # DeformConv -backend_test.exclude('test_depthtospace_') # DepthToSpace +backend_test.exclude('test_depthtospace_crd_mode_') # DepthToSpace in CRD mode backend_test.exclude('test_dequantizelinear_') # DequantizeLinear backend_test.exclude('test_det_') # Det backend_test.exclude('test_dft_') # DFT From dc39f395e8856a2334fc7ecd02426b7d6460b25d Mon Sep 17 00:00:00 2001 From: buildtech Date: Tue, 7 May 2024 19:57:38 +0200 Subject: [PATCH 2/4] Build Framework (NeoML-master 2.0.228.0): Incrementing version number. --- Build/Inc/ProductBuildNumber.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build/Inc/ProductBuildNumber.h b/Build/Inc/ProductBuildNumber.h index c2c27e640e..5703b5c353 100644 --- a/Build/Inc/ProductBuildNumber.h +++ b/Build/Inc/ProductBuildNumber.h @@ -1,5 +1,5 @@ #define VERINFO_PRODUCT_BRANCH "NeoML-master" #define VERINFO_MAJOR_VERSION 2 #define VERINFO_MINOR_VERSION 0 -#define VERINFO_MODIFICATION_NUMBER 227 +#define VERINFO_MODIFICATION_NUMBER 228 #define VERINFO_BUILD_NUMBER 0 From 1f32caf8f1cba3a31fae7a0d33df907912c9fe6e Mon Sep 17 00:00:00 2001 From: Kirill Golikov Date: Tue, 7 May 2024 18:25:14 +0200 Subject: [PATCH 3/4] [NeoML] CleanUp CSourceLayer Signed-off-by: Kirill Golikov --- NeoML/include/NeoML/Dnn/Layers/ParameterLayer.h | 4 +++- NeoML/include/NeoML/Dnn/Layers/SourceLayer.h | 8 ++++++-- NeoML/src/Dnn/Layers/ParameterLayer.cpp | 9 +++++---- NeoML/src/Dnn/Layers/SourceLayer.cpp | 13 ++++++++++--- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/NeoML/include/NeoML/Dnn/Layers/ParameterLayer.h b/NeoML/include/NeoML/Dnn/Layers/ParameterLayer.h index e71caa5940..0f373796f1 100644 --- a/NeoML/include/NeoML/Dnn/Layers/ParameterLayer.h +++ b/NeoML/include/NeoML/Dnn/Layers/ParameterLayer.h @@ -1,4 +1,4 @@ -/* Copyright © 2024 ABBYY Production LLC +/* Copyright © 2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,12 +31,14 @@ class NEOML_API CParameterLayer : public CBaseLayer { void SetBlob(CDnnBlob* _blob); void SetBlobDesc(const CBlobDesc& _desc); const CPtr& GetBlob() const { return paramBlobs[0]; } + protected: void AllocateOutputBlobs() override; void Reshape() override; void RunOnce() override; void BackwardOnce() override; void LearnOnce() override; + private: CBlobDesc desc; }; diff --git a/NeoML/include/NeoML/Dnn/Layers/SourceLayer.h b/NeoML/include/NeoML/Dnn/Layers/SourceLayer.h index 3519d4d4bb..56b1d66dba 100644 --- a/NeoML/include/NeoML/Dnn/Layers/SourceLayer.h +++ b/NeoML/include/NeoML/Dnn/Layers/SourceLayer.h @@ -1,4 +1,4 @@ -/* Copyright © 2017-2020 ABBYY Production LLC +/* Copyright © 2017-2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,7 +24,8 @@ namespace NeoML { class NEOML_API CSourceLayer : public CBaseLayer { NEOML_DNN_LAYER( CSourceLayer ) public: - explicit CSourceLayer( IMathEngine& mathEngine ) : CBaseLayer( mathEngine, "CCnnSourceLayer", false ), storeBlob( false ) {} + explicit CSourceLayer( IMathEngine& mathEngine ) : + CBaseLayer( mathEngine, "CCnnSourceLayer", false ), storeBlob( false ) {} // Sets the input data blob void SetBlob( CDnnBlob* blob ); @@ -38,9 +39,12 @@ class NEOML_API CSourceLayer : public CBaseLayer { void Serialize( CArchive& archive ) override; + void CleanUp( bool totalCleanUp = false ) override; + protected: CPtr blob; bool storeBlob; + // CBaseLayer class methods void Reshape() override; void RunOnce() override; diff --git a/NeoML/src/Dnn/Layers/ParameterLayer.cpp b/NeoML/src/Dnn/Layers/ParameterLayer.cpp index 85d9d0faba..1d1af90f78 100644 --- a/NeoML/src/Dnn/Layers/ParameterLayer.cpp +++ b/NeoML/src/Dnn/Layers/ParameterLayer.cpp @@ -1,4 +1,4 @@ -/* Copyright © 2024 ABBYY Production LLC +/* Copyright © 2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,9 +31,10 @@ void CParameterLayer::SetBlob(CDnnBlob* _blob) bool sameBlob = _blob == paramBlobs[0].Ptr(); paramBlobs[0] = _blob; - if (!outputDescs.IsEmpty()) { - if (paramBlobs[0]->GetDataType() != outputDescs[0].GetDataType() - || !paramBlobs[0]->GetDesc().HasEqualDimensions(outputDescs[0])) + if( !outputDescs.IsEmpty() ) { + if( _blob != nullptr + && ( _blob->GetDataType() != outputDescs[0].GetDataType() + || !_blob->GetDesc().HasEqualDimensions( outputDescs[0] ) ) ) { outputDescs[0] = paramBlobs[0]->GetDesc(); ForceReshape(); diff --git a/NeoML/src/Dnn/Layers/SourceLayer.cpp b/NeoML/src/Dnn/Layers/SourceLayer.cpp index 1ea2ef1752..63ea8bf908 100644 --- a/NeoML/src/Dnn/Layers/SourceLayer.cpp +++ b/NeoML/src/Dnn/Layers/SourceLayer.cpp @@ -1,4 +1,4 @@ -/* Copyright © 2017-2020 ABBYY Production LLC +/* Copyright © 2017-2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,8 +26,9 @@ void CSourceLayer::SetBlob( CDnnBlob* _blob ) blob = _blob; if( !outputDescs.IsEmpty() ) { - if( blob->GetDataType() != outputDescs[0].GetDataType() - || !blob->GetDesc().HasEqualDimensions( outputDescs[0] ) ) + if( blob != nullptr + && ( blob->GetDataType() != outputDescs[0].GetDataType() + || !blob->GetDesc().HasEqualDimensions( outputDescs[0] ) ) ) { outputDescs[0] = blob->GetDesc(); ForceReshape(); @@ -93,6 +94,12 @@ void CSourceLayer::Serialize( CArchive& archive ) } } +void CSourceLayer::CleanUp( bool totalCleanUp ) +{ + CBaseLayer::CleanUp( totalCleanUp ); + SetBlob( nullptr ); +} + CSourceLayer* Source( CDnn& network, const char* name ) { CPtr source = new CSourceLayer( network.GetMathEngine() ); From c62512797c1aee1aa605b5e24983412f4addb234 Mon Sep 17 00:00:00 2001 From: Kirill Golikov Date: Wed, 8 May 2024 14:51:12 +0200 Subject: [PATCH 4/4] [NeoML] Add CleanUp in network learning tests Signed-off-by: Kirill Golikov --- NeoML/test/src/DnnSolverTest.cpp | 110 ++++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 23 deletions(-) diff --git a/NeoML/test/src/DnnSolverTest.cpp b/NeoML/test/src/DnnSolverTest.cpp index b932be687c..4b756aa393 100644 --- a/NeoML/test/src/DnnSolverTest.cpp +++ b/NeoML/test/src/DnnSolverTest.cpp @@ -1,4 +1,4 @@ -/* Copyright © 2021-2023 ABBYY +/* Copyright © 2021-2024 ABBYY Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ limitations under the License. using namespace NeoML; using namespace NeoMLTest; +namespace NeoMLTest { + // Returns coefficient of neuron with one in and one out. static float getFcCoeff( const CFullyConnectedLayer* fcLayer ) { @@ -28,6 +30,10 @@ static float getFcCoeff( const CFullyConnectedLayer* fcLayer ) return fcLayer->GetWeightsData()->GetData().GetValue(); } +} // namespace NeoMLTest + +//--------------------------------------------------------------------------------------------------------------------- + // Check build/change correctness with gradient accumulation enabled TEST( CDnnSolverTest, NetworkModificationOnGradientAccumulation ) { @@ -150,51 +156,68 @@ TEST( CDnnSolverTest, NetworkModificationOnGradientAccumulation ) EXPECT_EQ( e1, e2 ); } -// Net for weight check. -class CWeightCheckNet { +//--------------------------------------------------------------------------------------------------------------------- + +namespace NeoMLTest { + +// Network for weights check. +class CWeightCheckNet final { public: - CWeightCheckNet(); - void SetSolver( CDnnSolver* solver ) { dnn.SetSolver( solver ); } + CWeightCheckNet( CDnnSolver* solver ); + float RunAndLearnOnce(); void GetWeights( CArray& weights ) const; + void CleanUp(); + private: CRandom random; CDnn dnn; + + CPtr data; + CPtr label; CPtr fc; CPtr loss; + + void setInputs(); }; -CWeightCheckNet::CWeightCheckNet() : +CWeightCheckNet::CWeightCheckNet( CDnnSolver* solver ) : random( 0xAAAAAAAA ), dnn( random, MathEngine() ) { - CPtr data = AddLayer( "data", dnn ); + dnn.SetSolver( solver ); + + data = AddLayer( "data", dnn ); + label = AddLayer( "label", dnn ); + + fc = AddLayer( "fc", { data } ); + fc->SetNumberOfElements( 2 ); + fc->SetZeroFreeTerm( true ); + + loss = AddLayer( "loss", { fc, label } ); + setInputs(); +} + +void CWeightCheckNet::setInputs() +{ { CPtr dataBlob = CDnnBlob::CreateDataBlob( MathEngine(), CT_Float, 1, 1, 2 ); CArray buff( { 0.25f, -0.345f } ); dataBlob->CopyFrom( buff.GetPtr() ); data->SetBlob( dataBlob ); } - - CPtr label = AddLayer( "label", dnn ); { CPtr labelBlob = CDnnBlob::CreateDataBlob( MathEngine(), CT_Int, 1, 1, 1 ); CArray buff( { 0 } ); labelBlob->CopyFrom( buff.GetPtr() ); label->SetBlob( labelBlob ); } - - fc = AddLayer( "fc", { data } ); - fc->SetNumberOfElements( 2 ); - fc->SetZeroFreeTerm( true ); { CPtr weightBlob = CDnnBlob::CreateDataBlob( MathEngine(), CT_Float, 1, 2, 2 ); CArray buff = { -0.5f, 0.9f, 0.3f, -0.7f }; weightBlob->CopyFrom( buff.GetPtr() ); fc->SetWeightsData( weightBlob ); } - - loss = AddLayer( "loss", { fc, label } ); } float CWeightCheckNet::RunAndLearnOnce() @@ -210,23 +233,44 @@ void CWeightCheckNet::GetWeights( CArray& weights ) const weightBlob->CopyTo( weights.GetPtr() ); } +void CWeightCheckNet::CleanUp() +{ + dnn.CleanUp( /*totalCleanUp*/true ); + MathEngine().CleanUp(); + dnn.Random().Reset( 0xAAAAAAAA ); + dnn.GetSolver()->Reset(); + setInputs(); +} + +//--------------------------------------------------------------------------------------------------------------------- + void testSolver( CDnnSolver* solver, const CArray>& expected ) { - CWeightCheckNet net; CArray weights; - net.SetSolver( solver ); + auto check = [&]( CWeightCheckNet& net ) + { for( int i = 0; i < expected.Size(); ++i ) { float loss = net.RunAndLearnOnce(); loss; net.GetWeights( weights ); - ASSERT_EQ( expected[i].Size(), weights.Size() ); + EXPECT_EQ( expected[i].Size(), weights.Size() ); for( int j = 0; j < weights.Size(); ++j ) { - ASSERT_TRUE( FloatEq( expected[i][j], weights[j] ) ); + EXPECT_NEAR( expected[i][j], weights[j], 1e-5 ); } } + }; + + CWeightCheckNet net( solver ); + check( net ); + + net.CleanUp(); + check( net ); } -// ==================================================================================================================== +} // namespace NeoMLTest + +//--------------------------------------------------------------------------------------------------------------------- + // Sgd. TEST( CDnnSolverTest, SgdNoReg ) @@ -339,7 +383,8 @@ TEST( CDnnSolverTest, SgdCompatL2 ) testSolver( sgd, expected ); } -// ==================================================================================================================== +//--------------------------------------------------------------------------------------------------------------------- + // Adam. TEST( CDnnSolverTest, AdamNoReg ) @@ -469,7 +514,8 @@ TEST( CDnnSolverTest, AdamCompatL2 ) testSolver( adam, expected ); } -// ==================================================================================================================== +//--------------------------------------------------------------------------------------------------------------------- + // Nadam. TEST( CDnnSolverTest, NadamNoReg ) @@ -532,6 +578,12 @@ TEST( CDnnSolverTest, NadamL2 ) testSolver( adam, expected ); } +//--------------------------------------------------------------------------------------------------------------------- + +// Serialization. + +namespace NeoMLTest { + static bool checkBlobEquality( CDnnBlob& firstBlob, CDnnBlob& secondBlob ) { if( !firstBlob.HasEqualDimensions( &secondBlob ) ) { @@ -683,6 +735,10 @@ static void solverSerializationTestImpl( CPtr firstSolver, bool trai EXPECT_TRUE( checkLstmEquality( reverse, secondReverse ) ); } +} // namespace NeoMLTest + +// sgd. + TEST( CDnnSimpleGradientSolverTest, Serialization1 ) { CPtr sgd = new CDnnSimpleGradientSolver( MathEngine() ); @@ -707,6 +763,10 @@ TEST( CDnnSimpleGradientSolverTest, Serialization2 ) solverSerializationTestImpl( sgd.Ptr(), false ); } +//--------------------------------------------------------------------------------------------------------------------- + +// Adam. + TEST( CDnnAdaptiveGradientSolverTest, Serialization1 ) { CPtr adam = new CDnnAdaptiveGradientSolver( MathEngine() ); @@ -769,6 +829,10 @@ TEST( CDnnNesterovGradientSolverTest, Serialization2 ) solverSerializationTestImpl( nadam.Ptr(), false ); } +//--------------------------------------------------------------------------------------------------------------------- + +// Lamb. + TEST( CDnnLambGradientSolverTest, Serialization1 ) { CPtr lamb = new CDnnLambGradientSolver( MathEngine() ); @@ -834,7 +898,7 @@ TEST( CDnnLambGradientSolverTest, Serialization4 ) solverSerializationTestImpl( lamb.Ptr(), true ); } -// ==================================================================================================================== +//--------------------------------------------------------------------------------------------------------------------- TEST( CDnnSolverTest, CompositeLearningRate ) {