Fix Vips thumbnails aspect ratio and implement thumbnail preview feature on the frontend

requirements.txt → .docker/requirements.txt

@@ -1,10 +1,13 @@
 # This file is just used to get security alerts from GitHub. Make sure the versions match
-# in .docker/worker.dockerfile.
-numpy==1.22.*
+# in worker.dockerfile.
+numpy==1.24.*
 opencv-contrib-python-headless==4.6.0
-scipy==1.10.0
-scikit-learn
-matplotlib==3.5.2
+scipy==1.10.*
+scikit-learn==1.2.*
+matplotlib==3.6.*
 PyExcelerate==0.6.7
-Pillow==10.0.1
+Pillow==10.2.0
 Shapely==1.8.1
+torch==2.1.*
+torchvision==0.16.*
+pandas==1.5.3

.docker/worker.dockerfile

@@ -1,32 +1,37 @@
-# PHP 8.1.13
-#FROM php:8.1-alpine
-FROM php@sha256:f9e31f22bdd89c1334a03db5c8800a5f3b1e1fe042d470adccf58a29672c6202
+# PHP 8.1.27
+# FROM php:8.1
+FROM php@sha256:9b5dfb7deef3e48d67b2599e4d3967bb3ece19fd5ba09cb8e7ee10f5facf36e0
 MAINTAINER Martin Zurowietz <martin@cebitec.uni-bielefeld.de>
 LABEL org.opencontainers.image.source https://github.com/biigle/core
 
-ARG OPENCV_VERSION=4.6.0-r3
-RUN apk add --no-cache \
-        eigen \
+RUN LC_ALL=C.UTF-8 apt-get update \
+    && apt-get install -y --no-install-recommends \
         ffmpeg \
-        lapack \
-        openblas \
-        py3-numpy \
         python3 \
-        py3-opencv="$OPENCV_VERSION"
+        python3-numpy \
+        python3-opencv \
+        python3-scipy \
+        python3-sklearn \
+        python3-matplotlib \
+        python3-shapely \
+        python3-pandas \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -r /var/lib/apt/lists/*
 
 RUN ln -s "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini"
 ADD ".docker/all-php.ini" "$PHP_INI_DIR/conf.d/all.ini"
 
-RUN apk add --no-cache \
-        libxml2 \
-        libzip \
-        openssl \
-        postgresql \
-    && apk add --no-cache --virtual .build-deps \
+RUN LC_ALL=C.UTF-8 apt-get update \
+    && apt-get install -y --no-install-recommends \
         libxml2-dev \
         libzip-dev \
-        postgresql-dev \
-    && docker-php-ext-configure pgsql -with-pgsql=/usr/local/pgsql \
+        libpq-dev \
+    && apt-get install -y --no-install-recommends \
+        libxml2 \
+        libzip4 \
+        postgresql-client \
+    && docker-php-ext-configure pgsql -with-pgsql=/usr/bin/pgsql \
     && docker-php-ext-install -j$(nproc) \
         exif \
         pcntl \
@@ -35,15 +40,29 @@ RUN apk add --no-cache \
         pgsql \
         soap \
         zip \
-    && apk del --purge .build-deps
+    && apt-get purge -y \
+        libxml2-dev \
+        libzip-dev \
+        libpq-dev \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -r /var/lib/apt/lists/*
 
 # Configure proxy if there is any. See: https://stackoverflow.com/a/2266500/1796523
 RUN [ -z "$HTTP_PROXY" ] || pear config-set http_proxy $HTTP_PROXY
-RUN apk add --no-cache yaml \
-    && apk add --no-cache --virtual .build-deps g++ make autoconf yaml-dev \
+
+RUN LC_ALL=C.UTF-8 apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libyaml-dev \
+    && apt-get install -y --no-install-recommends \
+        libyaml-0-2 \
     && pecl install yaml \
-    && docker-php-ext-enable yaml \
-    && apk del --purge .build-deps
+    && printf "\n" | docker-php-ext-enable yaml \
+    && apt-get purge -y \
+        libyaml-dev \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -r /var/lib/apt/lists/*
 
 ARG PHPREDIS_VERSION=5.3.7
 RUN curl -L -o /tmp/redis.tar.gz https://github.com/phpredis/phpredis/archive/${PHPREDIS_VERSION}.tar.gz \
@@ -53,70 +72,38 @@ RUN curl -L -o /tmp/redis.tar.gz https://github.com/phpredis/phpredis/archive/${
     && mv phpredis-${PHPREDIS_VERSION} /usr/src/php/ext/redis \
     && docker-php-ext-install -j$(nproc) redis
 
-ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH}"
-# Install vips from source because the apk package does not have dzsave support! Install
-# libvips and the vips PHP extension in one go so the *-dev dependencies are reused.
-ARG LIBVIPS_VERSION=8.12.2
-ARG PHP_VIPS_EXT_VERSION=1.0.13
-RUN apk add --no-cache --virtual .build-deps \
-        autoconf \
-        automake \
-        build-base \
-        expat-dev \
-        glib-dev \
-        libgsf-dev \
-        libjpeg-turbo-dev \
-        libpng-dev \
-        tiff-dev \
-    && apk add --no-cache \
-        expat \
-        glib \
-        libgsf \
-        libjpeg-turbo \
-        libpng \
-        tiff \
-    && cd /tmp \
-    && curl -L https://github.com/libvips/libvips/releases/download/v${LIBVIPS_VERSION}/vips-${LIBVIPS_VERSION}.tar.gz -o vips-${LIBVIPS_VERSION}.tar.gz \
-    && tar -xzf vips-${LIBVIPS_VERSION}.tar.gz \
-    && cd vips-${LIBVIPS_VERSION} \
-    && ./configure \
-        --without-python \
-        --enable-debug=no \
-        --disable-dependency-tracking \
-        --disable-static \
-    && make -j $(nproc) \
-    && make -s install-strip \
-    && cd /tmp \
-    && curl -L https://github.com/libvips/php-vips-ext/raw/master/vips-${PHP_VIPS_EXT_VERSION}.tgz -o  vips-${PHP_VIPS_EXT_VERSION}.tgz \
-    && echo '' | pecl install vips-${PHP_VIPS_EXT_VERSION}.tgz \
+# ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:${PKG_CONFIG_PATH}"
+
+RUN LC_ALL=C.UTF-8 apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libvips-dev \
+    && apt-get install -y --no-install-recommends \
+        libvips42 \
+    && pecl install vips \
     && docker-php-ext-enable vips \
-    && rm -r /tmp/* \
-    && apk del --purge .build-deps \
-    && rm -rf /var/cache/apk/*
+    && apt-get purge -y \
+        libvips-dev \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -r /var/lib/apt/lists/*
 
 # Unset proxy configuration again.
 RUN [ -z "$HTTP_PROXY" ] || pear config-set http_proxy ""
 
-# Other Python dependencies are added with the OpenCV build above.
-RUN apk add --no-cache py3-scipy py3-scikit-learn py3-matplotlib py3-shapely
-
-# Set this library path so the Python modules are linked correctly.
-# See: https://github.com/python-pillow/Pillow/issues/1763#issuecomment-204252397
-ENV LIBRARY_PATH=/lib:/usr/lib
-# Install Python dependencies. Note that these also depend on some image processing libs
-# that were installed along with vips.
-RUN apk add --no-cache --virtual .build-deps \
-        python3-dev \
-        py3-pip \
-        py3-wheel \
-        build-base \
-        libjpeg-turbo-dev \
-        libpng-dev \
-    && pip3 install --no-cache-dir \
+RUN LC_ALL=C.UTF-8 apt-get update \
+    && apt-get install -y --no-install-recommends \
+        python3-pip \
+    && pip3 install --no-cache-dir --break-system-packages \
         PyExcelerate==0.6.7 \
-        Pillow==10.0.1 \
-    && apk del --purge .build-deps \
-    && rm -rf /var/cache/apk/*
+        Pillow==10.2.0 \
+    && pip3 install --no-cache-dir --break-system-packages --index-url https://download.pytorch.org/whl/cpu \
+        torch==2.1.* \
+        torchvision==0.16.* \
+    && apt-get purge -y \
+        python3-pip \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -r /var/lib/apt/lists/*
 
 WORKDIR /var/www
 

.env.example

@@ -21,12 +21,6 @@ DB_DATABASE="biigle"
 DB_USERNAME="biigle"
 DB_PASSWORD="secret"
 
-VECTOR_DB_HOST="vector_database"
-VECTOR_DB_PORT=5432
-VECTOR_DB_DATABASE="biigle"
-VECTOR_DB_USERNAME="biigle"
-VECTOR_DB_PASSWORD="secret"
-
 VOLUME_ADMIN_STORAGE_DISKS=local
 
 BROADCAST_DRIVER=pusher

.github/dependabot.yml

@@ -0,0 +1,26 @@
+version: 2
+updates:
+    - package-ecosystem: pip
+      directory: "/.docker"
+      schedule:
+          interval: "weekly"
+      # Disable version updates
+      open-pull-requests-limit: 0
+    - package-ecosystem: composer
+      directory: "/"
+      schedule:
+          interval: "weekly"
+      # Disable version updates
+      open-pull-requests-limit: 0
+    - package-ecosystem: npm
+      directory: "/"
+      schedule:
+          interval: "weekly"
+      # Disable version updates
+      open-pull-requests-limit: 0
+    - package-ecosystem: github-actions
+      directory: "/"
+      schedule:
+          interval: "weekly"
+      # Disable version updates
+      open-pull-requests-limit: 0

.github/workflows/test.yml

@@ -37,7 +37,6 @@ jobs:
     - name: Start test database
       run: |
         docker-compose up -d --no-build database_testing
-        docker-compose up -d --no-build vector_database_testing
         sleep 5
 
     - name: Run tests
@@ -80,7 +79,6 @@ jobs:
     - name: Start test database
       run: |
         docker-compose up -d --no-build database_testing
-        docker-compose up -d --no-build vector_database_testing
         sleep 5
 
     - name: Run tests

.github/workflows/update-schema.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Trigger schema update
       run: |
         curl -X POST --fail \
-          -H "Authorization: token ${{ secrets.TOKEN }}" \
+          -H "Authorization: token ${{ secrets.BIIGLE_SCHEMA_API_TOKEN }}" \
           -H "Content-Type: application/json" \
           --data '{"event_type": "build_application"}' \
           https://api.github.com/repos/biigle/schema/dispatches

DEVELOPING.md

@@ -34,7 +34,7 @@ composer create-project biigle/core:dev-dev-modules \
 
 Note the `--ignore-platform-reqs` flag to keep Composer from complaining about missing requirements. These requirements will be met by the Docker containers.
 
-This will set up the project in the `dev-modules` branch of this repository. The `dev-modules` branch is configured with all BIIGLE modules which makes it easy to start module development. If you want to develop a feature directly in `biigle/core`, please switch to the `master` branch (or create your own branch based on `master`).
+This will set up the project in the `dev-modules` branch of this repository. The `dev-modules` branch is configured with all BIIGLE modules which makes it easy to start module development. If you want to develop a feature directly in `biigle/core`, please switch to the `master` branch (or create your own branch based on `master`). Pull requests from a `dev-modules`-based branch will not be accepted.
 
 ### 2. Build and run the application
 

app/Console/Commands/NewUser.php

@@ -39,6 +39,8 @@ public function handle()
 
         if ($this->confirm('Should the user be global admin? [y|N]')) {
             $u->role_id = Role::adminId();
+        } else {
+            $u->role_id = Role::editorId();
         }
 
         if ($this->confirm('Do you wish to auto-generate a password? [y|N]')) {

app/Events/AnnotationLabelAttached.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Biigle\Events;
+
+use Biigle\AnnotationLabel;
+use Illuminate\Contracts\Events\ShouldDispatchAfterCommit;
+use Illuminate\Foundation\Events\Dispatchable;
+
+class AnnotationLabelAttached implements ShouldDispatchAfterCommit
+{
+    use Dispatchable;
+
+    public function __construct(public AnnotationLabel $annotationLabel)
+    {
+        //
+    }
+}

app/Http/Controllers/Api/ImageAnnotationLabelController.php

@@ -2,6 +2,7 @@
 
 namespace Biigle\Http\Controllers\Api;
 
+use Biigle\Events\AnnotationLabelAttached;
 use Biigle\Http\Requests\StoreImageAnnotationLabel;
 use Biigle\ImageAnnotation;
 use Biigle\ImageAnnotationLabel;
@@ -184,6 +185,8 @@ public function store(StoreImageAnnotationLabel $request)
             // should not be returned
             unset($annotationLabel->annotation);
 
+            AnnotationLabelAttached::dispatch($annotationLabel);
+
             return response($annotationLabel, 201);
         } catch (QueryException $e) {
             // Although we check for existence above, this error happened some time.

app/Http/Controllers/Api/VideoAnnotationLabelController.php

@@ -2,6 +2,7 @@
 
 namespace Biigle\Http\Controllers\Api;
 
+use Biigle\Events\AnnotationLabelAttached;
 use Biigle\Http\Requests\DestroyVideoAnnotationLabel;
 use Biigle\Http\Requests\StoreVideoAnnotationLabel;
 use Biigle\VideoAnnotationLabel;
@@ -65,6 +66,8 @@ public function store(StoreVideoAnnotationLabel $request)
             // should not be returned
             unset($annotationLabel->annotation);
 
+            AnnotationLabelAttached::dispatch($annotationLabel);
+
             return response($annotationLabel, 201);
         } catch (QueryException $e) {
             // Although we check for existence above, this error happened some time.

app/Http/Controllers/Views/Videos/VideoController.php

@@ -72,15 +72,36 @@ public function show(Request $request, $id)
             'too-large' => VIDEO::ERROR_TOO_LARGE,
         ]);
 
-        return view('videos.show', compact(
-            'user',
-            'video',
-            'volume',
-            'videos',
-            'shapes',
-            'labelTrees',
-            'annotationSessions',
-            'errors'
-        ));
+        $fileIds = $volume->orderedFiles()->pluck('uuid', 'id');
+
+        $thumbUriTemplate = thumbnail_url(':uuid', config('videos.thumbnail_storage_disk'));
+
+        $spritesThumbnailsPerSprite = config('videos.sprites_thumbnails_per_sprite');
+        $spritesThumbnailInterval = config('videos.sprites_thumbnail_interval');
+        $spritesMaxThumbnails = config('videos.sprites_max_thumbnails');
+        $spritesMinThumbnails = config('videos.sprites_min_thumbnails');
+        $spritesThumbnailWidth = config('videos.sprites_thumbnail_width');
+        $spritesThumbnailHeight = config('videos.sprites_thumbnail_height');
+        return view(
+            'videos.show',
+            compact(
+                'user',
+                'video',
+                'volume',
+                'videos',
+                'shapes',
+                'labelTrees',
+                'annotationSessions',
+                'errors',
+                'fileIds',
+                'thumbUriTemplate',
+                'spritesThumbnailsPerSprite',
+                'spritesThumbnailInterval',
+                'spritesMaxThumbnails',
+                'spritesMinThumbnails',
+                'spritesThumbnailWidth',
+                'spritesThumbnailHeight'
+            )
+        );
     }
 }

app/Http/Requests/StoreVolume.php

@@ -48,7 +48,7 @@ public function rules()
         return [
             'name' => 'required|max:512',
             'media_type' => ['filled', Rule::in(array_keys(MediaType::INSTANCES))],
-            'url' => ['required', 'max:256', new VolumeUrl],
+            'url' => ['required', 'string', 'max:256', new VolumeUrl],
             'files' => [
                 'required',
                 'array',