Displaying progress bar during refresh.

alembic/versions/3b6918aeca4b_file_group.py

@@ -91,7 +91,7 @@ def upgrade():
     session.execute('''
     CREATE TABLE tag_file (
         tag_id INTEGER REFERENCES tag(id),
-        file_group_id BIGINT REFERENCES file_group(id),
+        file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE,
         created_at TIMESTAMP WITH TIME ZONE,
         UNIQUE (tag_id, file_group_id)
     )''')

app/src/Events.js

@@ -21,12 +21,6 @@ function handleEvents(events) {
             return;
         } else if (event === 'global_refresh_completed') {
             eventToast('Refresh completed', 'All files have been refreshed.');
-        } else if (event === 'directory_refresh_started') {
-            const description = message || 'Refresh of directory has started.';
-            eventToast('Refresh started', description);
-        } else if (event === 'directory_refresh_completed') {
-            const description = message || 'Refresh of directory has completed.';
-            eventToast('Refresh completed', description);
         }
 
         if (subject) {

app/src/api.js

@@ -527,6 +527,13 @@ export async function deleteFile(file) {
     await apiPost(`${API_URI}/files/delete`, body);
 }
 
+export async function fetchFilesProgress() {
+    const response = await apiGet(`${API_URI}/files/refresh_progress`);
+    if (response.status === 200) {
+        const json = await response.json();
+        return json['progress'];
+    }
+}
 
 export async function getHotspotStatus() {
     let response = await getSettings();
@@ -641,11 +648,6 @@ export async function clearFailedDownloads() {
     }
 }
 
-export async function getAPIStatus() {
-    let response = await apiGet(`${API_URI}/echo`);
-    return response.status === 200;
-}
-
 export async function getStatistics() {
     let response = await apiGet(`${API_URI}/statistics`);
     if (response.status === 200) {

app/src/components/Files.js

@@ -27,13 +27,13 @@ import {
     textEllipsis,
     useTitle
 } from "./Common";
-import {useQuery, useSearchFiles} from "../hooks/customHooks";
+import {useFilesProgressInterval, useQuery, useSearchFiles} from "../hooks/customHooks";
 import {Route, Routes} from "react-router-dom";
 import {CardPlacholder} from "./Placeholder";
 import {ArchiveCard, ArchiveRowCells} from "./Archive";
 import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
 import {StatusContext, ThemeContext} from "../contexts/contexts";
-import {Button, Card, CardIcon, Icon, Placeholder, Segment} from "./Theme";
+import {Button, Card, CardIcon, Icon, Placeholder, Progress, Segment} from "./Theme";
 import {SelectableTable} from "./Tables";
 import {VideoCard, VideoRowCells} from "./Videos";
 import _ from 'lodash';
@@ -43,13 +43,6 @@ import {useSubscribeEventName} from "../Events";
 import {FilePreviewContext} from "./FilePreview";
 import {taggedImageLabel, TagsContext} from "../Tags";
 
-
-export function FilesPage() {
-    useTitle('Files');
-
-    return <FileBrowser/>;
-}
-
 function EbookCard({file}) {
     const {s} = useContext(ThemeContext);
     let {data} = file;
@@ -481,6 +474,46 @@ export function FilesSearchView({
     />
 }
 
+export function FilesProgress() {
+    const {progress} = useFilesProgressInterval();
+
+    if (!progress) {
+        return;
+    }
+
+    const {refreshing, modeling, indexing, cleanup, indexed, unindexed, total_files, modeled} = progress;
+
+    if (refreshing) {
+        // Default is Discovery / Step 1.
+        let params = {value: 0, total: 3, progress: 'ratio'};
+        let label = 'Refresh: Discovery';
+
+        if (modeling) {
+            params['value'] = modeled;
+            params['total'] = total_files;
+            label = 'Refresh: Modeling';
+        } else if (indexing) {
+            params['value'] = unindexed;
+            params['total'] = unindexed + indexed;
+            label = 'Refresh: Indexing';
+        } else if (cleanup) {
+            params['value'] = 3;
+            params['total'] = 4;
+            label = 'Refresh: Cleanup';
+        }
+        return <Progress active {...params}>{label}</Progress>
+    }
+}
+
+function FilesPage() {
+    useTitle('Files');
+
+    return <>
+        <FilesProgress/>
+        <FileBrowser/>
+    </>;
+}
+
 export function FilesRoute() {
     return <PageContainer>
         <Routes>

app/src/hooks/customHooks.js

@@ -1,6 +1,7 @@
 import {useContext, useEffect, useRef, useState} from "react";
 import {
     fetchDomains,
+    fetchFilesProgress,
     filesSearch,
     getArchive,
     getChannel,
@@ -456,6 +457,34 @@ export const useBrowseFiles = () => {
     return {browseFiles, openFolders, setOpenFolders, fetchFiles};
 }
 
+export const useFilesProgress = () => {
+    const [progress, setProgress] = useState(null);
+
+    const localFetchFilesProgress = async () => {
+        try {
+            let p = await fetchFilesProgress();
+            setProgress(p);
+        } catch (e) {
+            setProgress(null);
+            console.error(e);
+        }
+    }
+
+    useEffect(() => {
+        localFetchFilesProgress();
+    }, []);
+
+    return {progress, fetchFilesProgress: localFetchFilesProgress}
+}
+
+export const useFilesProgressInterval = () => {
+    const {progress, fetchFilesProgress} = useFilesProgress();
+
+    useRecurringTimeout(fetchFilesProgress, 1000 * 3);
+
+    return {progress, fetchFilesProgress};
+}
+
 export const useHotspot = () => {
     const [on, setOn] = useState(null);
     const {status} = useContext(StatusContext);

modules/archive/__init__.py

@@ -6,7 +6,7 @@
 from sqlalchemy import not_
 from sqlalchemy.orm import Session
 
-from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length, \
+from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
     slow_logger
 from wrolpi.db import optional_session, get_db_session
 from wrolpi.downloader import Downloader, Download, DownloadResult
@@ -189,7 +189,7 @@ async def archive_modeler():
         await asyncio.sleep(0)
 
 
-@register_after_refresh
+@register_refresh_cleanup
 @limit_concurrent(1)
 def archive_cleanup():
     # Process all Archives that have not been validated.  Delete any that no longer exist, or are not real Archives.

modules/videos/__init__.py

@@ -2,7 +2,7 @@
 from typing import List, Tuple
 
 from modules.videos.models import Video, Channel
-from wrolpi.common import logger, limit_concurrent, register_modeler, register_after_refresh
+from wrolpi.common import logger, limit_concurrent, register_modeler, register_refresh_cleanup
 from wrolpi.db import get_db_curs, get_db_session
 from wrolpi.files.models import FileGroup
 from wrolpi.vars import PYTEST
@@ -54,7 +54,7 @@ async def video_modeler():
         # Sleep to catch cancel.
         await asyncio.sleep(0)
 
-@register_after_refresh
+@register_refresh_cleanup
 @limit_concurrent(1)
 def video_cleanup():
     logger.info('Claiming Videos for their Channels')

modules/videos/lib.py

@@ -13,7 +13,7 @@
 
 from wrolpi import before_startup
 from wrolpi.captions import extract_captions
-from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh, limit_concurrent
+from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_refresh_cleanup, limit_concurrent
 from wrolpi.dates import Seconds
 from wrolpi.db import get_db_curs, get_db_session, optional_session
 from wrolpi.errors import UnknownDirectory
@@ -367,7 +367,7 @@ def save_channels_config(session: Session = None):
 
 
 @before_startup
-@register_after_refresh
+@register_refresh_cleanup
 @limit_concurrent(1)
 def import_channels_config():
     """Import channel settings to the DB.  Existing channels will be updated."""

wrolpi/common.py

@@ -110,8 +110,8 @@ def log_level_context(level):
     'aiohttp_post',
     'register_modeler',
     'apply_modelers',
-    'register_after_refresh',
-    'apply_after_refresh',
+    'register_refresh_cleanup',
+    'apply_refresh_cleanup',
     'match_paths_to_suffixes',
     'chunks',
     'chunks_by_stem',
@@ -830,6 +830,7 @@ def register_modeler(modeler: callable):
 
 async def apply_modelers():
     for modeler in modelers:
+        logger_.info(f'Applying modeler {modeler.__name__}')
         try:
             await modeler()
         except Exception as e:
@@ -838,17 +839,17 @@ async def apply_modelers():
         await asyncio.sleep(0)
 
 
-after_refresh = []
+REFRESH_CLEANUP = []
 
 
-def register_after_refresh(func: callable):
-    after_refresh.append(func)
+def register_refresh_cleanup(func: callable):
+    REFRESH_CLEANUP.append(func)
     return func
 
 
-async def apply_after_refresh():
-    for func in after_refresh:
-        logger_.info(f'Applying after-refresh {func.__name__}')
+async def apply_refresh_cleanup():
+    for func in REFRESH_CLEANUP:
+        logger_.info(f'Applying refresh cleanup {func.__name__}')
         func()
 
 

wrolpi/events.py

@@ -45,14 +45,6 @@ def send_global_after_refresh_completed(message: str = None):
     def send_ready(message: str = None):
         send_event('ready', message)
 
-    @staticmethod
-    def send_directory_refresh_started(message: str):
-        send_event('directory_refresh_started', message, subject='refresh_directory')
-
-    @staticmethod
-    def send_directory_refresh_completed(message: str):
-        send_event('directory_refresh_completed', message, subject='refresh_directory')
-
 
 def log_event(event: str, message: str = None, action: str = None, subject: str = None):
     log = f'{event=}'

wrolpi/files/api.py

@@ -57,6 +57,16 @@ async def refresh(request: Request):
     return response.empty()
 
 
+@bp.get('/refresh_progress')
+@openapi.definition(
+    summary='Get the progress of the file refresh'
+)
+async def refresh_progress(request: Request):
+    progress = lib.get_refresh_progress()
+    return json_response(dict(
+        progress=progress,
+    ))
+
 @bp.post('/search')
 @openapi.definition(
     summary='Search Files',

wrolpi/files/lib.py

@@ -19,7 +19,7 @@
 from wrolpi.cmd import which
 from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
     partition, cancelable_wrapper, \
-    get_files_and_directories, chunks_by_stem, apply_modelers, apply_after_refresh
+    get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup
 from wrolpi.dates import now, from_timestamp
 from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
 from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
@@ -426,22 +426,26 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
 
         # Add all files in the media directory to the DB.
         paths = paths or [get_media_directory()]
-        await refresh_discover_paths(paths, idempotency)
+        with flags.refresh_discovery:
+            await refresh_discover_paths(paths, idempotency)
         if send_events:
             Events.send_global_refresh_discovery_completed()
 
         # Model all files that have not been indexed.
-        await apply_modelers()
+        with flags.refresh_modeling:
+            await apply_modelers()
         if send_events:
             Events.send_global_refresh_modeling_completed()
 
         # Index the rest of the files that were not indexed by modelers.
-        await apply_indexers()
+        with flags.refresh_indexing:
+            await apply_indexers()
         if send_events:
             Events.send_global_refresh_indexing_completed()
 
         # Cleanup any outdated file data.
-        await apply_after_refresh()
+        with flags.cleanup:
+            await apply_refresh_cleanup()
         if send_events:
             Events.send_global_after_refresh_completed()
 
@@ -769,3 +773,30 @@ def add_file_group_tag(file_group_id: int, tag_name: str, session: Session = Non
 def remove_file_group_tag(file_group_id: int, tag_name: str, session: Session = None):
     file_group, tag = _get_tag(file_group_id, tag_name, session)
     file_group.remove_tag(tag, session)
+
+
+def get_refresh_progress():
+    with get_db_curs() as curs:
+        curs.execute('''
+            SELECT
+                COUNT(id) AS "total_files",
+                COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed",
+                COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
+                COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
+            FROM file_group
+        ''')
+        results = dict(curs.fetchone())
+
+        status = dict(
+            cleanup=flags.cleanup.is_set(),
+            discovery=flags.refresh_discovery.is_set(),
+            indexed=results['indexed'],
+            indexing=flags.refresh_indexing.is_set(),
+            modeled=results['modeled'],
+            modeling=flags.refresh_modeling.is_set(),
+            refreshing=flags.refreshing.is_set(),
+            total_files=results['total_files'],
+            unindexed=results['unindexed'],
+        )
+
+    return status

wrolpi/files/models.py

@@ -75,7 +75,7 @@ class FileGroup(ModelHelper, Base):
     size = Column(BigInteger, default=lambda: 0)
     title = Column(String)
 
-    tag_files: InstrumentedList = relationship('TagFile')
+    tag_files: InstrumentedList = relationship('TagFile', cascade='all')
 
     a_text = deferred(Column(String))
     b_text = deferred(Column(String))

wrolpi/files/pdfs.py

@@ -123,6 +123,7 @@ async def pdf_modeler():
                     file_group.b_text = author
                     file_group.c_text = file_title
                     file_group.d_text = contents
+                    file_group.model = 'pdf'
                 except Exception as e:
                     logger.error(f'Failed to index PDF {pdf_file}', exc_info=e)
                     if PYTEST:
@@ -131,6 +132,9 @@ async def pdf_modeler():
                 # Even if indexing fails, we mark it as indexed.  We won't retry indexing this.
                 file_group.indexed = True
 
+                # Sleep to catch cancel.
+                await asyncio.sleep(0)
+
             if processed:
                 logger.debug(f'pdf_modeler processed {processed} PDFs')