Check for folders that completed since they were scanned: part of #438.

micall/monitor/kive_watcher.py

@@ -31,7 +31,7 @@
     HTTPAdapter = None
 
 logger = logging.getLogger(__name__)
-FOLDER_SCAN_INTERVAL = timedelta(minutes=2)
+FOLDER_SCAN_INTERVAL = timedelta(hours=1)
 SLEEP_SECONDS = 60
 DOWNLOADED_RESULTS = ['remap_counts_csv',
                       'conseq_csv',
@@ -259,10 +259,26 @@ def upload_kive_dataset(self, source_file, dataset_name, cdt, description):
         return dataset
 
     def add_sample_group(self, base_calls, sample_group):
+        """ Add a sample group (main and optional midi sample) to process.
+
+        Also checks to see whether the folder finished processing since the
+        last folder scan.
+        :param base_calls: path to the BaseCalls folder with FASTQ files in it
+        :param SampleGroup sample_group: the sample(s) to add
+        :return: SampleWatcher for the sample group, or None if that folder has
+            already finished processing
+        """
         self.check_session()
         folder_watcher = self.folder_watchers.get(base_calls)
         if folder_watcher is None:
             folder_watcher = self.add_folder(base_calls)
+
+            # Check if folder has finished since it was scanned.
+            done_path = self.get_results_path(folder_watcher) / "doneprocessing"
+            if done_path.exists():
+                del self.folder_watchers[base_calls]
+                return None
+
             self.create_batch(folder_watcher)
             self.upload_filter_quality(folder_watcher)
             shutil.rmtree(self.get_results_path(folder_watcher),

micall/tests/test_kive_watcher.py

@@ -1259,3 +1259,27 @@ def test_add_duplicate_sample(raw_data_with_two_samples,
                                   None)))
 
     assert sample_watcher1 is sample_watcher2
+
+
+def test_add_finished_sample(raw_data_with_two_samples,
+                             mock_open_kive,
+                             default_config):
+    """ The folder finished processing since the folder was scanned. """
+    base_calls = (raw_data_with_two_samples /
+                  "MiSeq/runs/140101_M01234/Data/Intensities/BaseCalls")
+    assert mock_open_kive
+    done_run = base_calls / "../../.."
+    results_path = done_run / "Results/version_0-dev"
+    results_path.mkdir(parents=True)
+    done_path = results_path / "doneprocessing"
+    done_path.touch()
+    kive_watcher = KiveWatcher(default_config)
+
+    sample_watcher1 = kive_watcher.add_sample_group(
+        base_calls=base_calls,
+        sample_group=SampleGroup('2110A',
+                                 ('2110A-V3LOOP_S13_L001_R1_001.fastq.gz',
+                                  None)))
+
+    assert done_path.exists()
+    assert sample_watcher1 is None