fix: DEV-1697: Projects list loading issues (#2038)

* fix: DEV-1008: flow for the first user in on-prem and cloud

* fix: DEV-1697: Projects list loading issues

* Speedup DM

* Fix user none

* Fix trailing slash

* Fix tests

* Fix tests again

label_studio/core/utils/sentry.py

@@ -34,6 +34,15 @@ def event_processor(event, hint):
             if message in value:
                 return None
 
+    if last.get('type') == 'OSError':
+        value = last.get('value')
+        messages = [
+            'Too many open files: ',
+        ]
+        for message in messages:
+            if message in value:
+                return None
+
     # special flag inside of logger.error(..., extra={'sentry_skip': True}) to skip error message
     if event.get('extra', {}).get('sentry_skip', False):
         return None

label_studio/data_manager/api.py

@@ -194,6 +194,18 @@ def get_task_serializer_context(request, project):
     def get_task_queryset(self, request, prepare_params):
         return Task.prepared.only_filtered(prepare_params=prepare_params)
 
+    @staticmethod
+    def prefetch(queryset):
+        return queryset.prefetch_related(
+            'annotations', 'predictions', 'annotations__completed_by', 'project',
+            'io_storages_azureblobimportstoragelink',
+            'io_storages_gcsimportstoragelink',
+            'io_storages_localfilesimportstoragelink',
+            'io_storages_redisimportstoragelink',
+            'io_storages_s3importstoragelink',
+            'file_upload'
+        )
+
     def get(self, request):
         # get project
         view_pk = int_from_request(request.GET, 'view', 0) or int_from_request(request.data, 'view', 0)
@@ -227,10 +239,12 @@ def get(self, request):
         if page is not None:
             ids = [task.id for task in page]  # page is a list already
             tasks = list(
-                Task.prepared.annotate_queryset(
-                    Task.objects.filter(id__in=ids),
-                    fields_for_evaluation=fields_for_evaluation,
-                    all_fields=all_fields,
+                self.prefetch(
+                    Task.prepared.annotate_queryset(
+                        Task.objects.filter(id__in=ids),
+                        fields_for_evaluation=fields_for_evaluation,
+                        all_fields=all_fields,
+                    )
                 )
             )
             tasks_by_ids = {task.id: task for task in tasks}
@@ -363,7 +377,7 @@ def get(self, request):
                 "target_syncing": False,
                 "task_count": project.tasks.count(),
                 "annotation_count": Annotation.objects.filter(task__project=project).count(),
-                'config_has_control_tags': len(project.get_control_tags_from_config()) > 0
+                'config_has_control_tags': len(project.get_parsed_config()) > 0
             }
         )
         return Response(data)

label_studio/projects/api.py

@@ -129,7 +129,7 @@ class ProjectListAPI(generics.ListCreateAPIView):
 
     def get_queryset(self):
         projects = Project.objects.filter(organization=self.request.user.active_organization)
-        return ProjectManager.with_counts_annotate(projects)
+        return ProjectManager.with_counts_annotate(projects).prefetch_related('members', 'created_by')
 
     def get_serializer_context(self):
         context = super(ProjectListAPI, self).get_serializer_context()

label_studio/projects/models.py

@@ -534,7 +534,7 @@ def delete_predictions(self):
         return {'deleted_predictions': count}
 
     def get_updated_weights(self):
-        outputs = self.get_parsed_config()
+        outputs = self.get_parsed_config(autosave_cache=False)
         control_weights = {}
         exclude_control_types = ('Filter',)
         for control_name in outputs:
@@ -706,14 +706,14 @@ def django_settings():
     def max_tasks_file_size():
         return settings.TASKS_MAX_FILE_SIZE
 
-    def get_control_tags_from_config(self):
-        return self.get_parsed_config()
+    def get_parsed_config(self, autosave_cache=True):
+        if self.parsed_label_config is None:
+            self.parsed_label_config = parse_config(self.label_config)
 
-    def get_parsed_config(self):
-        if self.parsed_label_config:
-            return self.parsed_label_config
+            if autosave_cache:
+                Project.objects.filter(id=self.id).update(parsed_label_config=self.parsed_label_config)
 
-        return parse_config(self.label_config)
+        return self.parsed_label_config
 
     def get_counters(self):
         """Method to get extra counters data from Manager method with_counts()

label_studio/projects/serializers.py

@@ -22,7 +22,7 @@ class ProjectSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
     """
 
     task_number = serializers.IntegerField(default=None, read_only=True,
-                                        help_text='Total task number in project')
+                                           help_text='Total task number in project')
     total_annotations_number = serializers.IntegerField(default=None, read_only=True,
                                                     help_text='Total annotations number in project including '
                                                               'skipped_annotations_number and ground_truth_number.')
@@ -40,7 +40,8 @@ class ProjectSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
     skipped_annotations_number = serializers.IntegerField(default=None, read_only=True,
                                                       help_text='Skipped by collaborators annotation number in project')
     num_tasks_with_annotations = serializers.IntegerField(default=None, read_only=True, help_text='Tasks with annotations count')
-    created_by = UserSimpleSerializer(default=CreatedByFromContext())
+
+    created_by = UserSimpleSerializer(default=CreatedByFromContext(), help_text='Project owner')
 
     parsed_label_config = SerializerMethodField(default=None, read_only=True,
                                                 help_text='JSON-formatted labeling configuration')
@@ -51,7 +52,7 @@ class ProjectSerializer(DynamicFieldsMixin, serializers.ModelSerializer):
 
     @staticmethod
     def get_config_has_control_tags(project):
-        return len(project.get_control_tags_from_config()) > 0
+        return len(project.get_parsed_config()) > 0
 
     @staticmethod
     def get_parsed_label_config(project):

label_studio/projects/urls.py

@@ -30,7 +30,7 @@
     path('validate/', api.LabelConfigValidateAPI.as_view(), name='label-config-validate'),
 
     # Validate label config for project
-    path('<int:pk>/validate', api.ProjectLabelConfigValidateAPI.as_view(), name='project-label-config-validate'),
+    path('<int:pk>/validate/', api.ProjectLabelConfigValidateAPI.as_view(), name='project-label-config-validate'),
 
     # Project summary
     path('<int:pk>/summary/', api.ProjectSummaryAPI.as_view(), name='project-summary'),

label_studio/tasks/models.py

@@ -437,7 +437,7 @@ def prepare_prediction_result(cls, result, project):
         elif isinstance(result, dict):
             # "value" from result
             # TODO: validate value fields according to project.label_config
-            for tag, tag_info in project.get_control_tags_from_config().items():
+            for tag, tag_info in project.get_parsed_config().items():
                 tag_type = tag_info['type'].lower()
                 if tag_type in result:
                     return [{
@@ -449,7 +449,7 @@ def prepare_prediction_result(cls, result, project):
 
         elif isinstance(result, (str, numbers.Integral)):
             # If result is of integral type, it could be a representation of data from single-valued control tags (e.g. Choices, Rating, etc.)  # noqa
-            for tag, tag_info in project.get_control_tags_from_config().items():
+            for tag, tag_info in project.get_parsed_config().items():
                 tag_type = tag_info['type'].lower()
                 if tag_type in SINGLE_VALUED_TAGS and isinstance(result, SINGLE_VALUED_TAGS[tag_type]):
                     return [{

label_studio/users/admin.py

@@ -14,6 +14,10 @@
 
 class UserAdminShort(UserAdmin):
 
+    add_fieldsets = (
+        (None, {'fields': ('email', 'password1', 'password2')}),
+    )
+
     def __init__(self, *args, **kwargs):
         super(UserAdminShort, self).__init__(*args, **kwargs)
 
@@ -24,10 +28,10 @@ def __init__(self, *args, **kwargs):
         self.ordering = ('email',)
 
         self.fieldsets = ((None, {'fields': ('password', )}),
-                          ('Personal info', {'fields': ('first_name', 'last_name', 'email')}),
+                          ('Personal info', {'fields': ('email', 'username', 'first_name', 'last_name')}),
                           ('Permissions', {'fields': ('is_active', 'is_staff', 'is_superuser',)}),
                           ('Important dates', {'fields': ('last_login', 'date_joined')}))
-        
+
 
 admin.site.register(User, UserAdminShort)
 admin.site.register(Project)

label_studio/users/forms.py

@@ -71,7 +71,7 @@ def clean_email(self):
         email = self.cleaned_data.get('email').lower()
 
         if email and User.objects.filter(email=email).exists():
-            raise forms.ValidationError('User with email already exists')
+            raise forms.ValidationError('User with this email already exists')
 
         return email
 

label_studio/users/serializers.py

@@ -18,6 +18,19 @@ def get_avatar(self, user):
     def get_initials(self, user):
         return user.get_initials()
 
+    def to_representation(self, instance):
+        """ Returns user with cache, this helps to avoid multiple s3/gcs links resolving for avatars """
+
+        uid = instance.id
+        key = 'user_cache'
+
+        if key not in self.context:
+            self.context[key] = {}
+        if uid not in self.context[key]:
+            self.context[key][uid] = super().to_representation(instance)
+
+        return self.context[key][uid]
+
     class Meta:
         model = User
         fields = (