Add events for importing/exporting annotations/datasets (#7476)

This can provide useful statistics on the use of this functionality.

Note that I chose to use the same scope for operations on projects,
tasks and jobs, because the import/export operations work more-or-less
the same on each of them and have the same parameters. It's still
possible to distinguish which type of object the operation was performed
on by examining the various `*_id` fields in the event.

changelog.d/20240213_010700_roman_import_export_events.md

@@ -0,0 +1,5 @@
+### Added
+
+- Added `dataset:export` and `dataset:import` events that are logged when
+  the user initiates an export or import of a project, task or job
+  (<https://github.com/opencv/cvat/pull/7476>)

cvat/apps/engine/views.py

@@ -43,6 +43,7 @@
 import cvat.apps.dataset_manager as dm
 import cvat.apps.dataset_manager.views  # pylint: disable=unused-import
 from cvat.apps.engine.cloud_provider import db_storage_to_storage_instance, download_file_from_bucket, export_resource_to_cloud_storage
+from cvat.apps.events.handlers import handle_dataset_export, handle_dataset_import
 from cvat.apps.dataset_manager.bindings import CvatImportError
 from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer
 from cvat.apps.engine.frame_provider import FrameProvider
@@ -2838,6 +2839,8 @@ def _import_annotations(request, rq_id_template, rq_func, db_obj, format_name,
         dependent_job = None
         location = location_conf.get('location') if location_conf else Location.LOCAL
 
+        db_storage = None
+
         if not filename or location == Location.CLOUD_STORAGE:
             if location != Location.CLOUD_STORAGE:
                 serializer = AnnotationFileSerializer(data=request.data)
@@ -2898,6 +2901,9 @@ def _import_annotations(request, rq_id_template, rq_func, db_obj, format_name,
                 result_ttl=settings.IMPORT_CACHE_SUCCESS_TTL.total_seconds(),
                 failure_ttl=settings.IMPORT_CACHE_FAILED_TTL.total_seconds()
             )
+
+        handle_dataset_import(db_obj, format_name=format_name, cloud_storage=db_storage)
+
         serializer = RqIdSerializer(data={'rq_id': rq_id})
         serializer.is_valid(raise_exception=True)
 
@@ -3045,6 +3051,8 @@ def _export_annotations(
             is_annotation_file=is_annotation_file,
         )
         func_args = (db_storage, filename, filename_pattern, callback) + func_args
+    else:
+        db_storage = None
 
     with get_rq_lock_by_user(queue, user_id):
         queue.enqueue_call(
@@ -3056,6 +3064,10 @@ def _export_annotations(
             result_ttl=ttl,
             failure_ttl=ttl,
         )
+
+    handle_dataset_export(db_instance,
+        format_name=format_name, cloud_storage=db_storage, save_images=not is_annotation_file)
+
     return Response(status=status.HTTP_202_ACCEPTED)
 
 def _import_project_dataset(request, rq_id_template, rq_func, db_obj, format_name, filename=None, conv_mask_to_poly=True, location_conf=None):
@@ -3080,6 +3092,8 @@ def _import_project_dataset(request, rq_id_template, rq_func, db_obj, format_nam
             rq_job.delete()
         dependent_job = None
         location = location_conf.get('location') if location_conf else None
+        db_storage = None
+
         if not filename and location != Location.CLOUD_STORAGE:
             serializer = DatasetFileSerializer(data=request.data)
             if serializer.is_valid(raise_exception=True):
@@ -3138,6 +3152,8 @@ def _import_project_dataset(request, rq_id_template, rq_func, db_obj, format_nam
                 result_ttl=settings.IMPORT_CACHE_SUCCESS_TTL.total_seconds(),
                 failure_ttl=settings.IMPORT_CACHE_FAILED_TTL.total_seconds()
             )
+
+        handle_dataset_import(db_obj, format_name=format_name, cloud_storage=db_storage)
     else:
         return Response(status=status.HTTP_409_CONFLICT, data='Import job already exists')
 

cvat/apps/events/event.py

@@ -26,6 +26,7 @@ class EventScopes:
         "comment": ["create", "update", "delete"],
         "annotations": ["create", "update", "delete"],
         "label": ["create", "update", "delete"],
+        "dataset": ["export", "import"],
     }
 
     @classmethod

cvat/apps/events/handlers.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: MIT
 
 from copy import deepcopy
+from typing import Optional, Union
 import traceback
 import rq
 
@@ -474,6 +475,51 @@ def filter_shape_data(shape):
                 payload={"tracks": tracks},
             )
 
+def handle_dataset_io(
+    instance: Union[Project, Task, Job],
+    action: str,
+    *,
+    format_name: str,
+    cloud_storage: Optional[CloudStorage],
+    **payload_fields,
+) -> None:
+    payload={"format": format_name, **payload_fields}
+
+    if cloud_storage:
+        payload["cloud_storage"] = {"id": cloud_storage.id}
+
+    record_server_event(
+        scope=event_scope(action, "dataset"),
+        request_id=request_id(),
+        org_id=organization_id(instance),
+        org_slug=organization_slug(instance),
+        project_id=project_id(instance),
+        task_id=task_id(instance),
+        job_id=job_id(instance),
+        user_id=user_id(instance),
+        user_name=user_name(instance),
+        user_email=user_email(instance),
+        payload=payload,
+    )
+
+def handle_dataset_export(
+    instance: Union[Project, Task, Job],
+    *,
+    format_name: str,
+    cloud_storage: Optional[CloudStorage],
+    save_images: bool,
+) -> None:
+    handle_dataset_io(instance, "export",
+        format_name=format_name, cloud_storage=cloud_storage, save_images=save_images)
+
+def handle_dataset_import(
+    instance: Union[Project, Task, Job],
+    *,
+    format_name: str,
+    cloud_storage: Optional[CloudStorage],
+) -> None:
+    handle_dataset_io(instance, "import", format_name=format_name, cloud_storage=cloud_storage)
+
 def handle_rq_exception(rq_job, exc_type, exc_value, tb):
     oid = rq_job.meta.get("org_id", None)
     oslug = rq_job.meta.get("org_slug", None)

site/content/en/docs/administration/advanced/analytics.md

@@ -125,6 +125,8 @@ Server events:
 
 - `create:label`, `update:label`, `delete:label`
 
+- `export:dataset`, `import:dataset`
+
 Client events:
 
 - `load:cvat`