From 048bc35a425455b2c5db5726d8dbbf53e182cca4 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Thu, 18 Jan 2024 15:47:31 +0100 Subject: [PATCH 1/2] Update for new ProcessAnnotatedFile job of biigle/largo --- src/Jobs/PostprocessVolumeImport.php | 34 ++++++++++------------ tests/Jobs/PostprocessVolumeImportTest.php | 18 ++++++------ 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/Jobs/PostprocessVolumeImport.php b/src/Jobs/PostprocessVolumeImport.php index 583a8ca..7199464 100644 --- a/src/Jobs/PostprocessVolumeImport.php +++ b/src/Jobs/PostprocessVolumeImport.php @@ -2,12 +2,12 @@ namespace Biigle\Modules\Sync\Jobs; -use Biigle\ImageAnnotation; +use Biigle\Image; use Biigle\Jobs\Job; use Biigle\Jobs\ProcessNewVolumeFiles; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; -use Biigle\Modules\Largo\Jobs\GenerateVideoAnnotationPatch; -use Biigle\VideoAnnotation; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedImage; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedVideo; +use Biigle\Video; use Biigle\Volume; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Queue\InteractsWithQueue; @@ -47,24 +47,22 @@ public function handle() ProcessNewVolumeFiles::dispatch($volume); }); - if (class_exists(GenerateImageAnnotationPatch::class)) { - ImageAnnotation::join('images', 'images.id', '=', 'image_annotations.image_id') - ->whereIn('images.volume_id', $this->ids) - ->select('image_annotations.id') - ->eachById(function ($annotation) { - GenerateImageAnnotationPatch::dispatch($annotation) + if (class_exists(ProcessAnnotatedImage::class)) { + Image::whereIn('images.volume_id', $this->ids) + ->whereHas('annotations') + ->eachById(function ($image) { + ProcessAnnotatedImage::dispatch($image) ->onQueue(config('largo.generate_annotation_patch_queue')); - }, 1000, 'image_annotations.id', 'id'); + }, 1000); } - if (class_exists(GenerateVideoAnnotationPatch::class)) { - VideoAnnotation::join('videos', 'videos.id', '=', 'video_annotations.video_id') - ->whereIn('videos.volume_id', $this->ids) - ->select('video_annotations.id') - ->eachById(function ($annotation) { - GenerateVideoAnnotationPatch::dispatch($annotation) + if (class_exists(ProcessAnnotatedVideo::class)) { + Video::whereIn('videos.volume_id', $this->ids) + ->whereHas('annotations') + ->eachById(function ($video) { + ProcessAnnotatedVideo::dispatch($video) ->onQueue(config('largo.generate_annotation_patch_queue')); - }, 1000, 'video_annotations.id', 'id'); + }, 1000); } } } diff --git a/tests/Jobs/PostprocessVolumeImportTest.php b/tests/Jobs/PostprocessVolumeImportTest.php index 57f8143..385f7e9 100644 --- a/tests/Jobs/PostprocessVolumeImportTest.php +++ b/tests/Jobs/PostprocessVolumeImportTest.php @@ -3,8 +3,8 @@ namespace Biigle\Tests\Modules\Sync\Jobs; use Biigle\Jobs\ProcessNewVolumeFiles; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; -use Biigle\Modules\Largo\Jobs\GenerateVideoAnnotationPatch; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedImage; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedVideo; use Biigle\Modules\Sync\Jobs\PostprocessVolumeImport; use Biigle\Tests\ImageAnnotationTest; use Biigle\Tests\ImageTest; @@ -20,13 +20,13 @@ public function testHandleVolumeImages() $job = new PostprocessVolumeImport(collect([$image->volume])); $job->handle(); Queue::assertPushed(ProcessNewVolumeFiles::class); - Queue::assertNotPushed(GenerateImageAnnotationPatch::class); + Queue::assertNotPushed(ProcessAnnotatedImage::class); } public function testHandleImageAnnotationPatches() { - if (!class_exists(GenerateImageAnnotationPatch::class)) { - $this->markTestSkipped('Requires '.GenerateImageAnnotationPatch::class); + if (!class_exists(ProcessAnnotatedImage::class)) { + $this->markTestSkipped('Requires '.ProcessAnnotatedImage::class); } $annotation = ImageAnnotationTest::create(); @@ -34,13 +34,13 @@ public function testHandleImageAnnotationPatches() $job->handle(); // One job for the creation of the annotation and one job by // PostprocessVolumeImport. - $this->assertCount(2, Queue::pushed(GenerateImageAnnotationPatch::class)); + $this->assertCount(2, Queue::pushed(ProcessAnnotatedImage::class)); } public function testHandleVideoAnnotationPatches() { - if (!class_exists(GenerateVideoAnnotationPatch::class)) { - $this->markTestSkipped('Requires '.GenerateVideoAnnotationPatch::class); + if (!class_exists(ProcessAnnotatedVideo::class)) { + $this->markTestSkipped('Requires '.ProcessAnnotatedVideo::class); } $annotation = VideoAnnotationTest::create(); @@ -48,6 +48,6 @@ public function testHandleVideoAnnotationPatches() $job->handle(); // One job for the creation of the annotation and one job by // PostprocessVolumeImport. - $this->assertCount(2, Queue::pushed(GenerateVideoAnnotationPatch::class)); + $this->assertCount(2, Queue::pushed(ProcessAnnotatedVideo::class)); } } From e7841a6cf7dee84107734c10574a7c8951a60d76 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Thu, 18 Jan 2024 15:49:02 +0100 Subject: [PATCH 2/2] Implement delay to (somewhat) ensure order of processing --- src/Jobs/PostprocessVolumeImport.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Jobs/PostprocessVolumeImport.php b/src/Jobs/PostprocessVolumeImport.php index 7199464..8c9c848 100644 --- a/src/Jobs/PostprocessVolumeImport.php +++ b/src/Jobs/PostprocessVolumeImport.php @@ -47,11 +47,16 @@ public function handle() ProcessNewVolumeFiles::dispatch($volume); }); + // Give the ProcessNewVolumeFiles jobs a head start so the file thumbnails are + // generated (mostly) before the annotation thumbnails. + $delay = now()->addSeconds(30); + if (class_exists(ProcessAnnotatedImage::class)) { Image::whereIn('images.volume_id', $this->ids) ->whereHas('annotations') - ->eachById(function ($image) { + ->eachById(function ($image) use ($delay) { ProcessAnnotatedImage::dispatch($image) + ->delay($delay) ->onQueue(config('largo.generate_annotation_patch_queue')); }, 1000); } @@ -59,8 +64,9 @@ public function handle() if (class_exists(ProcessAnnotatedVideo::class)) { Video::whereIn('videos.volume_id', $this->ids) ->whereHas('annotations') - ->eachById(function ($video) { + ->eachById(function ($video) use ($delay) { ProcessAnnotatedVideo::dispatch($video) + ->delay($delay) ->onQueue(config('largo.generate_annotation_patch_queue')); }, 1000); }