update

Dawn-LX · Mar 17, 2022 · 1cd96cc · 1cd96cc
1 parent c40db26
commit 1cd96cc
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -75,7 +75,48 @@ Put them under the dir of this project (or any other position if you use absolut
         --dataset_type vidor \
         --save_path datasets/GT_json_for_eval/VidORval_gts.json
     ```
+2. Download model weights for different exps [here](), and put them in the `experiments/` dir.
+3. Refer to `experiments/readme.md` for the correspondence between the exp ids and the table ids in our paper.
+4. For **VidVRD**, run the following commands to evaluate different exps: (refer to `tools/eval_vidvrd.py` for more details)
 
+    e.g., for exp1
+    ```
+    python tools/eval_vidvrd.py \
+        --cfg_path experiments/exp1/config_.py \
+        --ckpt_path experiments/exp1/model_epoch_80.pth \
+        --use_pku \
+        --cuda 1 \
+        --save_tag debug
+    ```
+5. For **VidOR**, refer to `tools/eval_vidor.py` for more details. 
+
+    Run the following commands to evaluate BIG-C (i.e., only the classification stage):
+    ```
+    python tools/eval_vidor.py \
+        --eval_cls_only \
+        --cfg_path experiments/exp4/config_.py \
+        --ckpt_path experiments/exp4/model_epoch_60.pth \
+        --save_tag epoch60_debug \
+        --cuda 1
+    ```
+    Run the following commands to evaluate BIG based on the output of cls stage (you need run BIG-C first and save the infer_results).
+    ```
+    python tools/eval_vidor.py \
+        --cfg_path experiments/grounding_weights/config_.py \
+        --ckpt_path experiments/grounding_weights/model_epoch_70.pth \
+        --output_dir experiments/exp4_with_grounding \
+        --cls_stage_result_path experiments/exp4/VidORval_infer_results_topk3_epoch60_debug.pkl \
+        --save_tag with_grd_epoch70 \
+        --cuda 1
+    ```
+    Run the following commands to evaluate the fraction recall (refer to table-6 in our paper, you need run BIG first and save the hit_infos).
+    ```
+    python tools/eval_fraction_recall.py \
+        --cfg_path experiments/grounding_weights/config_.py \
+        --hit_info_path  experiments/exp5_with_grounding/VidORval_hit_infos_aft_grd_with_grd_epoch70.pkl
+    ```
+
+**NOTE** from frame-level bbox annotations to video-level tracklets GTs.
 # Training (TODO)
 
 the code for training is still being organized (an initial version will be completed before March 28, 2022).

diff --git a/experiments/readme.md b/experiments/readme.md
@@ -1,9 +1,16 @@
 
 ## exp1~exp3 are for VidVRD
 
-| exp1 | tabel-1 |   |
-|------|---------|---|
-| exp2 | table-1 |   |
-| exp3 | table-1 |   |
-
-## exp4~exp5 are for VidVRD
+| dir name | table id |  mAP | Notes |
+|------|---------|------|-----|
+| exp1 | table-1 |17.56 | BIG-C, VidVRD, PKU's tracklet
+| exp2 | table-1 |17.67 | BIG-C, VidVRD, PKU's tracklet with I3D
+| exp3 | table-1 |26.08 | BIG-C, VidVRD, MEGA
+| exp4 | table-3 |8.03 | BIG-C, VidOR
+| exp4_with_grounding | table-3 |8.28 | BIG, VidOR
+| exp5 | table-3 |8.29 | BIG-C, VidOR
+| exp5_with_grounding | table-3 |8.54 | BIG, VidOR
+| exp5_with_grounding | table-6 | - | BIG, VidOR, fraction recall
+| exp6 | table-2 |7.05 | Base-C, VidOR
+| exp6_with_grounding | table-2 |7.19 | Base, VidOR
+| grounding_weights | - | -| VidOR, weights for #Bins = 1,5,10
diff --git a/tools/eval_vidor.py b/tools/eval_vidor.py
@@ -23,7 +23,7 @@ def evaluate_cls_stage(
     save_tag="",
     experiment_dir=None,
     gpu_id = 0,
-    save_infer_result = True,
+    save_infer_results = True,
     save_relation_json=False
 ):
 
@@ -121,7 +121,7 @@ def evaluate_cls_stage(
         logger=logger,
         prediction_results=predict_relations
     )
-    if save_infer_result:
+    if save_infer_results:
         save_path =  os.path.join(experiment_dir,'VidORval_infer_results_{}.pkl'.format(save_tag))
         logger.info("saving infer_results into {}...".format(save_path))
         with open(save_path,'wb') as f:
@@ -316,7 +316,7 @@ def evaluate_combined(
                 save_tag=args.save_tag,
                 experiment_dir=args.output_dir,
                 gpu_id = args.cuda,
-                save_infer_result=True,
+                save_infer_results=True,
                 save_relation_json=False
             )
         else:

diff --git a/tools/eval_vidor_our_gt.py b/tools/eval_vidor_our_gt.py
@@ -199,7 +199,7 @@ def evaluate_cls_stage(
 
     '''
     ### table-3 BIG-C RoI
-    python tools/eval_vidor.py \
+    python tools/eval_vidor_our_gt.py \
         --cfg_path experiments/exp4/config_.py \
         --ckpt_path experiments/exp4/model_epoch_60.pth \
         --save_tag epoch60_debug \
@@ -214,7 +214,7 @@ def evaluate_cls_stage(
 
     '''
     ### table-3 BIG-C RoI + Lang
-    python tools/eval_vidor.py \
+    python tools/eval_vidor_our_gt.py \
         --cfg_path experiments/exp5/config_.py \
         --ckpt_path experiments/exp5/model_epoch_60.pth \
         --save_tag epoch60_debug \
@@ -234,7 +234,7 @@ def evaluate_cls_stage(
 
     '''
     ### table-2 Base-C
-    python tools/eval_vidor.py \
+    python tools/eval_vidor_our_gt.py \
         --use_baseline \
         --cfg_path experiments/exp6/config_.py \
         --ckpt_path experiments/exp6/model_epoch_80.pth \

diff --git a/tools/eval_vidvrd.py b/tools/eval_vidvrd.py
@@ -200,13 +200,13 @@ def inference_then_eval(
         )
 
     '''
-    python tools/eval_vidvrd2.py \
+    python tools/eval_vidvrd.py \
         --json_results_path /home/gkf/project/VideoGraph/training_dir_reorganized/vidvrd/model_0v10_pku_i3dclsme2_cachePKUv2/VidORval_predict_relations_topk10_pTrue_epoch70.json
     
     ### exp1
-    python tools/eval_vidvrd2.py \
+    python tools/eval_vidvrd.py \
         --cfg_path experiments/exp1/config_.py \
-        --ckpt_path /home/gkf/project/VideoGraph/training_dir_reorganized/vidvrd/model_0v10_cachePKUv1_rightcatid/model_epoch_80.pth \
+        --ckpt_path experiments/exp1/model_epoch_80.pth \
         --use_pku \
         --cuda 1 \
         --save_tag debug
@@ -218,9 +218,9 @@ def inference_then_eval(
     
     
     ### exp2
-    python tools/eval_vidvrd2.py \
+    python tools/eval_vidvrd.py \
         --cfg_path experiments/exp2/config_.py \
-        --ckpt_path /home/gkf/project/VideoGraph/training_dir_reorganized/vidvrd/model_0v10_pku_i3dclsme2_cachePKUv2/model_epoch_70.pth \
+        --ckpt_path experiments/exp2/model_epoch_70.pth \
         --use_pku \
         --cuda 2 \
         --save_tag debug
@@ -231,9 +231,9 @@ def inference_then_eval(
 
     
     ### exp3
-    python tools/eval_vidvrd2.py \
+    python tools/eval_vidvrd.py \
         --cfg_path experiments/exp3/config_.py \
-        --ckpt_path /home/gkf/project/VidSGG-BIG/experiments/exp3/model_epoch_80.pth \
+        --ckpt_path experiments/exp3/model_epoch_80.pth \
         --cuda 3 \
         --save_tag debug
     

diff --git a/tools/eval_vidvrd_our_gt.py b/tools/eval_vidvrd_our_gt.py
@@ -235,23 +235,23 @@ def replace_state_dict_keys(state_dict):
     )
     '''
     ### exp1
-    python tools/eval_vidvrd.py \
+    python tools/eval_vidvrd_our_gt.py \
         --cfg_path experiments/exp1/config_.py \
         --ckpt_path /home/gkf/project/VideoGraph/training_dir_reorganized/vidvrd/model_0v10_cachePKUv1_rightcatid/model_epoch_80.pth \
         --use_pku \
         --cuda 1 \
         --save_tag debug
     
     ### exp2
-    python tools/eval_vidvrd.py \
+    python tools/eval_vidvrd_our_gt.py \
         --cfg_path experiments/exp2/config_.py \
         --ckpt_path /home/gkf/project/VideoGraph/training_dir_reorganized/vidvrd/model_0v10_pku_i3dclsme2_cachePKUv2/model_epoch_70.pth \
         --use_pku \
         --cuda 2 \
         --save_tag debug
     
     ### exp3
-    python tools/eval_vidvrd.py \
+    python tools/eval_vidvrd_our_gt.py \
         --cfg_path experiments/exp3/config_.py \
         --ckpt_path /home/gkf/project/VidSGG-BIG/experiments/exp3/model_epoch_80.pth \
         --cuda 3 \