Merge pull request #288 from lardbit/refresh-media-recently-updated

Only fetch "recently updated" meda records vs pulling everything every time

src/frontend/src/app/api.service.ts

@@ -131,35 +131,35 @@ export class ApiService {
       this.localStorage.get(this.STORAGE_KEY_WATCH_MOVIES).pipe(
         map(
           (data: any[]) => {
-            this.watchMovies = data;
+            this.watchMovies = data || [];
             return this.watchMovies;
           }),
       ),
       this.localStorage.get(this.STORAGE_KEY_WATCH_TV_SHOWS).pipe(
         map(
           (data: any[]) => {
-            this.watchTVShows = data;
+            this.watchTVShows = data || [];
             return this.watchTVShows;
           }),
       ),
       this.localStorage.get(this.STORAGE_KEY_WATCH_TV_SEASONS).pipe(
         map(
           (data: any[]) => {
-            this.watchTVSeasons = data;
+            this.watchTVSeasons = data || [];
             return this.watchTVSeasons;
           }),
       ),
       this.localStorage.get(this.STORAGE_KEY_WATCH_TV_SEASON_REQUESTS).pipe(
         map(
           (data: any[]) => {
-            this.watchTVSeasonRequests = data;
+            this.watchTVSeasonRequests = data || [];
             return this.watchTVSeasonRequests;
           }),
       ),
       this.localStorage.get(this.STORAGE_KEY_WATCH_TV_EPISODES).pipe(
         map(
           (data: any[]) => {
-            this.watchTVEpisodes = data;
+            this.watchTVEpisodes = data || [];
             return this.watchTVEpisodes;
           }),
       ),
@@ -190,13 +190,20 @@ export class ApiService {
     );
   }
 
-  public fetchWatchMedia(): Observable<any> {
+  public fetchWatchMedia(afterDateUpdated?: string): Observable<any> {
+    let params: any = {};
+
+    // conditionally include an "updated after date" parameter
+    if (afterDateUpdated) {
+      params.date_updated__gte = afterDateUpdated;
+    }
+
     return forkJoin([
-      this.fetchWatchTVShows(),
-      this.fetchWatchTVSeasons(),
-      this.fetchWatchTVSeasonRequests(),
-      this.fetchWatchTVEpisodes(),
-      this.fetchWatchMovies(),
+      this.fetchWatchTVShows(),  // "shows" don't support the "after date updated" parameter
+      this.fetchWatchTVSeasons(params),
+      this.fetchWatchTVSeasonRequests(params),
+      this.fetchWatchTVEpisodes(params),
+      this.fetchWatchMovies(params),
     ]).pipe(
       tap(() => {
         this._updateStorage().subscribe();
@@ -448,8 +455,8 @@ export class ApiService {
     params = params || {};
     const httpParams = new HttpParams({fromObject: params});
     return this.http.get(this.API_URL_WATCH_TV_SEASON, {params: httpParams, headers: this._requestHeaders()}).pipe(
-      map((data: any) => {
-        this.watchTVSeasons = data;
+      map((records: any) => {
+        this._mergeMediaRecords(this.watchTVSeasons, records);
         return this.watchTVSeasons;
       }),
     );
@@ -459,8 +466,8 @@ export class ApiService {
     params = params || {};
     const httpParams = new HttpParams({fromObject: params});
     return this.http.get(this.API_URL_WATCH_TV_SEASON_REQUEST, {params: httpParams, headers: this._requestHeaders()}).pipe(
-      map((data: any) => {
-        this.watchTVSeasonRequests = data;
+      map((records: any) => {
+        this._mergeMediaRecords(this.watchTVSeasonRequests, records);
         return this.watchTVSeasonRequests;
       }),
     );
@@ -471,17 +478,18 @@ export class ApiService {
     const httpParams = new HttpParams({fromObject: params});
 
     return this.http.get(this.API_URL_WATCH_MOVIE, {params: httpParams, headers: this._requestHeaders()}).pipe(
-      map((data: any) => {
-        this.watchMovies = data;
+      map((records: any[]) => {
+        this._mergeMediaRecords(this.watchMovies, records);
         return this.watchMovies;
       }),
     );
   }
 
-  public fetchWatchTVEpisodes() {
-    return this.http.get(this.API_URL_WATCH_TV_EPISODE, {headers: this._requestHeaders()}).pipe(
-      map((data: any) => {
-        this.watchTVEpisodes = data;
+  public fetchWatchTVEpisodes(params: any) {
+    const httpParams = new HttpParams({fromObject: params});
+    return this.http.get(this.API_URL_WATCH_TV_EPISODE, {headers: this._requestHeaders(), params: httpParams}).pipe(
+      map((records: any) => {
+        this._mergeMediaRecords(this.watchTVEpisodes, records);
         return this.watchTVEpisodes;
       }),
     );
@@ -828,6 +836,22 @@ export class ApiService {
     return this.http.post(this.API_URL_BLACKLISTS_DELETE, null, {headers: this._requestHeaders()});
   }
 
+  protected _mergeMediaRecords(existingRecords: any[], updatedRecords: any[]) {
+    // merge new/updated media records
+    updatedRecords.forEach((updatedRecord) => {
+      // find matching record
+      const mediaIndex = existingRecords.findIndex((media) => {
+        return media.id === updatedRecord.id;
+      });
+      // found - update existing value
+      if (mediaIndex >= 0) {
+        existingRecords[mediaIndex] = updatedRecord;
+      } else {
+        existingRecords.push(updatedRecord);
+      }
+    })
+  }
+
   protected _initWebSocket() {
 
     // we can't rely on the server's websocket url because it may be "nefarious" when run in a docker stack,

src/frontend/src/app/app.component.ts

@@ -1,6 +1,8 @@
 import { Component, OnInit } from '@angular/core';
 import { ApiService } from './api.service';
 import { OnPageVisible } from 'angular-page-visibility-v2';
+import * as moment from 'moment';
+import {tap} from "rxjs/operators";
 
 
 @Component({
@@ -10,6 +12,7 @@ import { OnPageVisible } from 'angular-page-visibility-v2';
 })
 export class AppComponent implements OnInit {
   public isCollapsed = true;
+  protected _lastUpdated: string = moment().format();
 
   constructor(
     private apiService: ApiService,
@@ -21,9 +24,15 @@ export class AppComponent implements OnInit {
 
   @OnPageVisible()
   logWhenPageVisible (): void {
-    console.log('page visible, re-fetching data');
+    console.log(`page visible, re-fetching data that was updated after ${this._lastUpdated}`);
     this.apiService.fetchCoreData().subscribe();
-    this.apiService.fetchWatchMedia().subscribe();
+    this.apiService.fetchWatchMedia(this._lastUpdated)
+      .pipe(
+        tap(() => {
+          this._lastUpdated = moment().format();
+        })
+      )
+      .subscribe();
   }
 
   public isStaff(): Boolean {

src/frontend/src/app/media/media-movie.component.ts

@@ -132,7 +132,7 @@ export class MediaMovieComponent implements OnInit, OnDestroy {
 
   public getWatchMovie() {
     return this.apiService.watchMovies.find((watchMovie) => {
-      return watchMovie.tmdb_movie_id === this.result.id;
+      return this.result && watchMovie.tmdb_movie_id === this.result.id;
     });
   }
 

src/nefarious/api/filters.py

@@ -0,0 +1,42 @@
+from django_filters import rest_framework as filters
+from nefarious.models import WatchMovie, WatchTVEpisode, WatchTVSeasonRequest, WatchTVSeason
+
+
+class WatchMovieFilterSet(filters.FilterSet):
+
+    class Meta:
+        model = WatchMovie
+        fields = {
+            'collected': ['exact'],
+            'date_updated': ['gte'],
+        }
+
+
+class WatchTVSeasonFilterSet(filters.FilterSet):
+
+    class Meta:
+        model = WatchTVSeason
+        fields = {
+            'collected': ['exact'],
+            'date_updated': ['gte'],
+        }
+
+
+class WatchTVEpisodeFilterSet(filters.FilterSet):
+
+    class Meta:
+        model = WatchTVEpisode
+        fields = {
+            'collected': ['exact'],
+            'date_updated': ['gte'],
+        }
+
+
+class WatchTVSeasonRequestFilterSet(filters.FilterSet):
+
+    class Meta:
+        model = WatchTVSeasonRequest
+        fields = {
+            'collected': ['exact'],
+            'date_updated': ['gte'],
+        }

src/nefarious/api/viewsets.py

@@ -13,13 +13,14 @@
 
 from nefarious import websocket
 from nefarious.api.mixins import UserReferenceViewSetMixin, BlacklistAndRetryMixin, DestroyTransmissionResultMixin, WebSocketMediaMessageUpdatedMixin
+from nefarious.api.filters import WatchMovieFilterSet, WatchTVSeasonFilterSet, WatchTVSeasonRequestFilterSet, WatchTVEpisodeFilterSet
 from nefarious.api.permissions import IsAuthenticatedDjangoObjectUser
 from nefarious.api.serializers import (
     NefariousSettingsSerializer, WatchTVEpisodeSerializer, WatchTVShowSerializer,
     UserSerializer, WatchMovieSerializer, NefariousPartialSettingsSerializer,
     WatchTVSeasonSerializer, WatchTVSeasonRequestSerializer, TorrentBlacklistSerializer,
 )
-from nefarious.models import NefariousSettings, WatchTVEpisode, WatchTVShow, WatchMovie, WatchTVSeason, WatchTVSeasonRequest, TorrentBlacklist
+from nefarious.models import NefariousSettings, WatchTVEpisode, WatchTVShow, WatchMovie, WatchTVSeason, WatchTVSeasonRequest, TorrentBlacklist, WatchMediaBase
 from nefarious.tasks import watch_tv_episode_task, watch_tv_show_season_task, watch_movie_task, send_websocket_message_task
 from nefarious.utils import (
     verify_settings_jackett, verify_settings_transmission, verify_settings_tmdb,
@@ -31,7 +32,7 @@ class WatchMovieViewSet(WebSocketMediaMessageUpdatedMixin, DestroyTransmissionRe
     queryset = WatchMovie.objects.select_related('user').all()
     serializer_class = WatchMovieSerializer
     filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter,)
-    filter_fields = ('collected',)
+    filterset_class = WatchMovieFilterSet
     permission_classes = (IsAuthenticatedDjangoObjectUser,)
 
     def perform_create(self, serializer):
@@ -85,7 +86,7 @@ class WatchTVSeasonViewSet(WebSocketMediaMessageUpdatedMixin, DestroyTransmissio
     serializer_class = WatchTVSeasonSerializer
     permission_classes = (IsAuthenticatedDjangoObjectUser,)
     filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter,)
-    filter_fields = ('collected',)
+    filterset_class = WatchTVSeasonFilterSet
 
 
 @method_decorator(gzip_page, name='dispatch')
@@ -96,7 +97,7 @@ class WatchTVSeasonRequestViewSet(WebSocketMediaMessageUpdatedMixin, UserReferen
     queryset = WatchTVSeasonRequest.objects.select_related('user').all()
     serializer_class = WatchTVSeasonRequestSerializer
     permission_classes = (IsAuthenticatedDjangoObjectUser,)
-    filter_fields = ('collected',)
+    filterset_class = WatchTVSeasonRequestFilterSet
 
     def perform_create(self, serializer):
         super().perform_create(serializer)
@@ -153,7 +154,7 @@ class WatchTVEpisodeViewSet(WebSocketMediaMessageUpdatedMixin, DestroyTransmissi
     serializer_class = WatchTVEpisodeSerializer
     permission_classes = (IsAuthenticatedDjangoObjectUser,)
     filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter,)
-    filter_fields = ('collected',)
+    filterset_class = WatchTVEpisodeFilterSet
 
     def perform_create(self, serializer):
         super().perform_create(serializer)

src/nefarious/migrations/0076_auto_20240609_1342.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.0.2 on 2024-06-09 13:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('nefarious', '0075_auto_20240114_1715'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='watchmovie',
+            name='date_updated',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='watchtvepisode',
+            name='date_updated',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='watchtvseason',
+            name='date_updated',
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

src/nefarious/migrations/0077_watchtvseasonrequest_date_updated.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.2 on 2024-06-09 13:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('nefarious', '0076_auto_20240609_1342'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='watchtvseasonrequest',
+            name='date_updated',
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

src/nefarious/models.py

@@ -99,6 +99,7 @@ class WatchMediaBase(models.Model):
     """
     user = models.ForeignKey(User, on_delete=models.CASCADE)
     date_added = models.DateTimeField(auto_now_add=True)
+    date_updated = models.DateTimeField(auto_now=True)
     collected = models.BooleanField(default=False)
     collected_date = models.DateTimeField(blank=True, null=True)
     download_path = models.CharField(max_length=1000, blank=True, null=True, unique=True)
@@ -115,6 +116,7 @@ class Meta:
         indexes = [
             models.Index(fields=['collected']),
             models.Index(fields=['collected_date']),
+            models.Index(fields=['date_updated']),
         ]
 
 
@@ -171,6 +173,7 @@ class WatchTVSeasonRequest(models.Model):
     quality_profile_custom = models.CharField(max_length=500, null=True, blank=True, choices=zip(quality.PROFILE_NAMES, quality.PROFILE_NAMES))
     collected = models.BooleanField(default=False)
     date_added = models.DateTimeField(auto_now_add=True)
+    date_updated = models.DateTimeField(auto_now=True)
     release_date = models.DateField(null=True, blank=True)
 
     class Meta: