Add tests for evaluation with initialization in TF (openvinotoolkit#1119

)

* FQ with multiple inbound nodes init

* Run init with evaluation

* update target_init (and extand range), disbale tfrecords shuffle

* split init and sota_checkpoint tests

* split tests

* separate table

* bugfix for report dump

* extend ranges for init

* separate build optimization

* minor revision

* clean up

* deterministic init

* correct thresholds

* object detection seed

* update thresholds

examples/tensorflow/classification/datasets/builder.py

@@ -50,6 +50,7 @@ def __init__(self, config, image_size, num_devices, one_hot, is_train):
                                              DATASET_NUM_CLASSES.get(self._dataset_name))
         self._one_hot = one_hot
         self._cache = False
+        self._shuffle_train = config.get('seed') is None
         self._shuffle_buffer_size = 10000
         self._deterministic_train = False
         self._use_slack = True
@@ -109,7 +110,8 @@ def _pipeline(self, dataset):
             dataset = dataset.cache()
 
         if self.is_train:
-            dataset = dataset.shuffle(self._shuffle_buffer_size)
+            if self._shuffle_train:
+                dataset = dataset.shuffle(self._shuffle_buffer_size)
             dataset = dataset.repeat()
 
         if self._dataset_type == 'tfrecords':

examples/tensorflow/classification/main.py

@@ -45,6 +45,7 @@
 from examples.tensorflow.common.utils import write_metrics
 from examples.tensorflow.common.utils import SummaryWriter
 from examples.tensorflow.common.utils import close_strategy_threadpool
+from examples.tensorflow.common.utils import set_seed
 
 
 def get_argument_parser():
@@ -154,6 +155,8 @@ def run(config):
     if config.metrics_dump is not None:
         write_metrics(0, config.metrics_dump)
 
+    set_seed(config)
+
     model_fn, model_params = get_model(config.model,
                                        input_shape=config.get('input_info', {}).get('sample_size', None),
                                        num_classes=config.get('num_classes', get_num_classes(config.dataset)),

examples/tensorflow/common/argparser.py

@@ -223,6 +223,10 @@ def get_common_argument_parser(**flags):
         action="store_true",
     )
 
+    parser.add_argument(
+        '--seed', default=None, type=int,
+        help='Specific seed for initializing pseudo-random number generators.')
+
     return parser
 
 

examples/tensorflow/common/tfrecords_dataset.py

@@ -25,6 +25,7 @@ def __init__(self, config, is_train):
         self.dataset_dir = config.dataset_dir
         self.is_train = is_train
 
+        self.shuffle_train = config.get('seed') is None
         self.buffer_size = config.get('buffer_size', BUFFER_SIZE)
         self.cycle_length = config.get('cycle_length', CYCLE_LENGTH)
         self.num_parallel_calls = NUM_PARALLEL_CALLS
@@ -50,7 +51,7 @@ def decoder(self):
         pass
 
     def as_dataset(self) -> tf.data.Dataset:
-        dataset = tf.data.Dataset.list_files(self.file_pattern, shuffle=True)
+        dataset = tf.data.Dataset.list_files(self.file_pattern, shuffle=(self.is_train and self.shuffle_train))
 
         dataset = dataset.interleave(
             lambda name: tf.data.TFRecordDataset(name, buffer_size=self.buffer_size),

examples/tensorflow/common/utils.py

@@ -11,11 +11,14 @@
  limitations under the License.
 """
 
+import random
 import time
 import datetime
 import json
 import os
 import tarfile
+
+import numpy as np
 import resource
 from os import path as osp
 from pathlib import Path
@@ -210,3 +213,12 @@ def close_strategy_threadpool(strategy):
     # pylint: disable=protected-access
     if isinstance(strategy, MirroredStrategy):
         atexit.register(strategy._extended._collective_ops._pool.close)
+
+
+def set_seed(config):
+    if config.seed is not None:
+        os.environ['TF_DETERMINISTIC_OPS'] = '1'
+        os.environ['TF_CUDNN_DETERMINISTIC'] = '1'
+        random.seed(config.seed)
+        np.random.seed(config.seed)
+        tf.random.set_seed(config.seed)

examples/tensorflow/object_detection/main.py

@@ -18,7 +18,6 @@
 import tensorflow as tf
 import numpy as np
 
-from examples.tensorflow.common.utils import close_strategy_threadpool
 from nncf.common.accuracy_aware_training import create_accuracy_aware_training_loop
 from nncf.tensorflow import create_compressed_model
 from nncf.tensorflow.helpers.model_manager import TFModelManager
@@ -47,6 +46,8 @@
 from examples.tensorflow.common.utils import write_metrics
 from examples.tensorflow.object_detection.models.model_selector import get_predefined_config
 from examples.tensorflow.object_detection.models.model_selector import get_model_builder
+from examples.tensorflow.common.utils import close_strategy_threadpool
+from examples.tensorflow.common.utils import set_seed
 
 
 def get_argument_parser():
@@ -279,6 +280,8 @@ def run(config):
     if config.metrics_dump is not None:
         write_metrics(0, config.metrics_dump)
 
+    set_seed(config)
+
     # Create dataset
     train_builder, test_builder = get_dataset_builders(config, strategy.num_replicas_in_sync)
     train_dataset = train_builder.build()

nncf/tensorflow/tensor_statistics/reduction.py

@@ -31,7 +31,7 @@ def get_axes(ndims: int, per_channel: bool, channel_axes: Union[int, list, tuple
 def get_reduction_shape_activations(layer: tf.keras.layers.Layer,
                                     channel_axes: Union[int, tuple, list],
                                     use_per_sample_stats: bool) -> ReductionShape:
-    ndims = len(layer.input_shape)
+    ndims = len(layer.get_input_shape_at(0))
     channel_axes_ = channel_axes if isinstance(channel_axes, (list, tuple)) else [channel_axes]
     reduction_shape = get_axes(ndims, layer.per_channel, channel_axes_)
     if use_per_sample_stats:

tests/tensorflow/sota_checkpoints_eval.json

@@ -19,6 +19,7 @@
                     "config": "examples/tensorflow/classification/configs/quantization/inception_v3_imagenet_int8.json",
                     "reference": "inception_v3",
                     "target": 78.36,
+                    "target_init": 76.64,
                     "resume": "inception_v3_int8_w_sym_t_half_a_sym_t",
                     "metric_type": "Acc@1",
                     "model_description": "inception_v3_int8_w_sym_t_half_a_sym_t",
@@ -33,6 +34,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity_quantization/inception_v3_imagenet_rb_sparsity_int8.json",
                     "reference": "inception_v3",
                     "target": 77.58,
+                    "target_init": 76.61,
                     "resume": "inception_v3_int8_w_sym_t_half_a_sym_t_rb_sparsity_61",
                     "metric_type": "Acc@1",
                     "model_description": "inception_v3_int8_w_sym_t_half_a_sym_t_rb_sparsity_61",
@@ -48,6 +50,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity/inception_v3_imagenet_magnitude_sparsity.json",
                     "reference": "inception_v3",
                     "target": 77.87,
+                    "target_init": 77.9,
                     "resume": "inception_v3_sparsity_54",
                     "metric_type": "Acc@1",
                     "model_description": "inception_v3_sparsity_54",
@@ -71,6 +74,7 @@
                     "config": "examples/tensorflow/classification/configs/quantization/mobilenet_v2_imagenet_int8.json",
                     "reference": "mobilenet_v2",
                     "target": 71.66,
+                    "target_init": 60.92,
                     "resume": "mobilenet_v2_int8_w_sym_t_half_a_sym_t",
                     "metric_type": "Acc@1",
                     "model_description": "mobilenet_v2_int8_w_sym_t_half_a_sym_t",
@@ -85,6 +89,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity_quantization/mobilenet_v2_imagenet_rb_sparsity_int8.json",
                     "reference": "mobilenet_v2",
                     "target": 71.0,
+                    "target_init": 61.25,
                     "resume": "mobilenet_v2_int8_w_sym_t_half_a_sym_t_rb_sparsity_52",
                     "metric_type": "Acc@1",
                     "model_description": "mobilenet_v2_int8_w_sym_t_half_a_sym_t_rb_sparsity_52",
@@ -99,6 +104,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity/mobilenet_v2_imagenet_rb_sparsity.json",
                     "reference": "mobilenet_v2",
                     "target": 71.34,
+                    "target_init": 71.85,
                     "resume": "mobilenet_v2_rb_sparsity_50",
                     "metric_type": "Acc@1",
                     "model_description": "mobilenet_v2_rb_sparsity_50",
@@ -112,6 +118,7 @@
                 "tf1_mobilenet_v2_1.0_224_sparsity_35": {
                     "config": "examples/tensorflow/classification/configs/sparsity/mobilenet_v2_hub_imagenet_magnitude_sparsity.json",
                     "target": 71.83,
+                    "target_init": 71.85,
                     "resume": "tf1_mobilenet_v2_1.0_224_sparsity_35",
                     "metric_type": "Acc@1",
                     "model_description": "tf1 mobilenet_v2_sparsity_35",
@@ -131,6 +138,7 @@
                     "config": "examples/tensorflow/classification/configs/quantization/mobilenet_v3_small_imagenet_int8.json",
                     "reference": "mobilenet_v3_small",
                     "target": 67.75,
+                    "target_init": 0.1,
                     "resume": "mobilenet_v3_small_int8_w_sym_ch_half_a_asym_t",
                     "metric_type": "Acc@1",
                     "model_description": "mobilenet_v3_small_int8_w_sym_ch_half_a_asym_t",
@@ -141,6 +149,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity_quantization/mobilenet_v3_small_imagenet_rb_sparsity_int8.json",
                     "reference": "mobilenet_v3_small",
                     "target": 67.55,
+                    "target_init": 0.1,
                     "resume": "mobilenet_v3_small_int8_w_sym_ch_half_a_asym_t_rb_sparsity_42",
                     "metric_type": "Acc@1",
                     "model_description": "mobilenet_v3_small_int8_w_sym_ch_half_a_asym_t_rb_sparsity_42",
@@ -158,6 +167,7 @@
                     "config": "examples/tensorflow/classification/configs/quantization/mobilenet_v3_large_imagenet_int8.json",
                     "reference": "mobilenet_v3_large",
                     "target": 75.02,
+                    "target_init": 0.16,
                     "resume": "mobilenet_v3_large_int8_w_sym_ch_half_a_asym_t",
                     "metric_type": "Acc@1",
                     "model_description": "mobilenet_v3_large_int8_w_sym_ch_half_a_asym_t",
@@ -168,6 +178,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity_quantization/mobilenet_v3_large_imagenet_rb_sparsity_int8.json",
                     "reference": "mobilenet_v3_large",
                     "target": 75.28,
+                    "target_init": 0.14,
                     "resume": "mobilenet_v3_large_int8_w_sym_ch_half_a_asym_t_rb_sparsity_42",
                     "metric_type": "Acc@1",
                     "model_description": "mobilenet_v3_large_int8_w_sym_ch_half_a_asym_t_rb_sparsity_42",
@@ -185,6 +196,7 @@
                     "config": "examples/tensorflow/classification/configs/quantization/resnet50_imagenet_int8.json",
                     "reference": "resnet50",
                     "target": 75.0,
+                    "target_init": 74.57,
                     "resume": "resnet50_int8_w_sym_t_half_a_sym_t",
                     "metric_type": "Acc@1",
                     "model_description": "resnet50_int8_w_sym_t_half_a_sym_t",
@@ -197,6 +209,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity_quantization/resnet50_imagenet_rb_sparsity_int8.json",
                     "reference": "resnet50",
                     "target": 74.3,
+                    "target_init": 74.45,
                     "resume": "resnet50_int8_w_sym_t_half_a_sym_t_rb_sparsity_65",
                     "metric_type": "Acc@1",
                     "model_description": "resnet50_int8_w_sym_t_half_a_sym_t_rb_sparsity_65",
@@ -209,6 +222,7 @@
                     "config": "examples/tensorflow/classification/configs/sparsity/resnet50_imagenet_rb_sparsity.json",
                     "reference": "resnet50",
                     "target": 74.36,
+                    "target_init": 75.04,
                     "resume": "resnet50_rb_sparsity_80",
                     "metric_type": "Acc@1",
                     "model_description": "resnet50_rb_sparsity_80",
@@ -221,6 +235,7 @@
                     "config": "examples/tensorflow/classification/configs/pruning/resnet50_imagenet_pruning_geometric_median.json",
                     "reference": "resnet50",
                     "target": 74.98,
+                    "target_init": 68.03,
                     "resume": "resnet50_pruning_40",
                     "metric_type": "Acc@1",
                     "model_description": "resnet50_pruning_40",
@@ -233,6 +248,7 @@
                     "config": "examples/tensorflow/classification/configs/pruning_quantization/resnet50_imagenet_pruning_geometric_median_int8.json",
                     "reference": "resnet50",
                     "target": 75.08,
+                    "target_init": 66.78,
                     "resume": "resnet50_int8_w_sym_t_half_a_sym_t_pruning_40",
                     "metric_type": "Acc@1",
                     "model_description": "resnet50_int8_w_sym_t_half_a_sym_t_pruning_40",
@@ -266,6 +282,8 @@
                     "config": "examples/tensorflow/object_detection/configs/quantization/retinanet_coco_int8.json",
                     "reference": "retinanet",
                     "target": 33.18,
+                    "target_init": 33.3,
+                    "weights": "retinanet/retinanet.h5",
                     "resume": "retinanet_int8_w_sym_t_half_a_sym_t",
                     "metric_type": "mAP",
                     "model_description": "retinanet_int8_w_sym_t_half_a_sym_t",
@@ -279,6 +297,8 @@
                     "config": "examples/tensorflow/object_detection/configs/sparsity/retinanet_coco_magnitude_sparsity.json",
                     "reference": "retinanet",
                     "target": 33.13,
+                    "target_init": 33.44,
+                    "weights": "retinanet/retinanet.h5",
                     "resume": "retinanet_sparsity_50",
                     "metric_type": "mAP",
                     "model_description": "retinanet_sparsity_50",
@@ -292,6 +312,8 @@
                     "config": "examples/tensorflow/object_detection/configs/pruning/retinanet_coco_pruning.json",
                     "reference": "retinanet",
                     "target": 32.7,
+                    "target_init": 29.17,
+                    "weights": "retinanet/retinanet.h5",
                     "resume": "retinanet_pruning_40",
                     "metric_type": "mAP",
                     "model_description": "retinanet_pruning_40",
@@ -307,6 +329,8 @@
                     "config": "examples/tensorflow/object_detection/configs/pruning_quantization/retinanet_coco_pruning_int8.json",
                     "reference": "retinanet",
                     "target": 32.68,
+                    "target_init": 29.03,
+                    "weights": "retinanet/retinanet.h5",
                     "resume": "retinanet_int8_w_sym_t_half_a_sym_t_pruning_40",
                     "metric_type": "mAP",
                     "model_description": "retinanet_int8_w_sym_t_half_a_sym_t_pruning_40",
@@ -330,6 +354,8 @@
                     "config":"examples/tensorflow/object_detection/configs/quantization/yolo_v4_coco_int8.json",
                     "reference":"yolo_v4",
                     "target":46.30,
+                    "target_init": 45.28,
+                    "weights":"yolo_v4/yolo_v4.h5",
                     "resume":"yolo_v4_int8_w_sym_ch_half_a_asym_t",
                     "metric_type":"mAP",
                     "model_description":"yolo_v4_int8_w_sym_ch_half_a_asym_t",
@@ -344,6 +370,8 @@
                     "config":"examples/tensorflow/object_detection/configs/sparsity/yolo_v4_coco_magnitude_sparsity.json",
                     "reference":"yolo_v4",
                     "target":46.54,
+                    "target_init": 47.04,
+                    "weights":"yolo_v4/yolo_v4.h5",
                     "resume":"yolo_v4_sparsity_50",
                     "metric_type":"mAP",
                     "model_description":"yolo_v4_sparsity_50",
@@ -375,6 +403,7 @@
                     "config": "examples/tensorflow/segmentation/configs/quantization/mask_rcnn_coco_int8.json",
                     "reference": "mask_rcnn_baseline",
                     "target": 37.27,
+                    "weights": "mask_rcnn_baseline",
                     "resume": "mask_rcnn_int8_w_sym_t_half_a_sym_t",
                     "metric_type": "mAP",
                     "model_description": "mask_rcnn_int8_w_sym_t_half_a_sym_t",
@@ -386,6 +415,7 @@
                     "config": "examples/tensorflow/segmentation/configs/sparsity/mask_rcnn_coco_magnitude_sparsity.json",
                     "reference": "mask_rcnn_baseline",
                     "target": 36.93,
+                    "weights": "mask_rcnn_baseline",
                     "resume": "mask_rcnn_sparsity_50",
                     "metric_type": "mAP",
                     "model_description": "mask_rcnn_sparsity_50",