diff --git a/.gitignore b/.gitignore index f92c55600..ca6d38cbd 100755 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ /*.bsfs.json .cache .coverage -/.pytest_cache +.pytest_cache /.idea /micall/tests/working/* /micall/tests/microtest/Results diff --git a/.travis.yml b/.travis.yml index a51f29ae9..7444cacfc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - "3.4" + - "3.6" install: "pip install -r requirements-test.txt" before_script: - "cp micall/settings_default.py micall/settings.py" diff --git a/micall/monitor/kive_watcher.py b/micall/monitor/kive_watcher.py index 7acddc87d..9a76f04d8 100644 --- a/micall/monitor/kive_watcher.py +++ b/micall/monitor/kive_watcher.py @@ -81,6 +81,8 @@ class KiveWatcher: def __init__(self, config=None): self.config = config self.session = None + self.current_run_folder = None + self.run_batches = {} # {run_folder: run_batch} def is_full(self): return False @@ -163,7 +165,13 @@ def add_sample_group(self, base_calls, sample_group): self.session.login(self.config.kive_user, self.config.kive_password) run_folder = (base_calls / "../../..").resolve() run_name = '_'.join(run_folder.name.split('_')[:2]) + run_batch = self.run_batches.get(run_folder) + if run_batch is not None: + return + run_batch = self.create_batch(run_name) + self.run_batches[run_folder] = run_batch + read_sizes = parse_read_sizes(run_folder / "RunInfo.xml") read_lengths = [read_sizes.read1, read_sizes.index1, diff --git a/micall/tests/test_kive_watcher.py b/micall/tests/test_kive_watcher.py index d63cffdba..5f4c5894b 100644 --- a/micall/tests/test_kive_watcher.py +++ b/micall/tests/test_kive_watcher.py @@ -90,6 +90,12 @@ def create_raw_data_with_hcv_pair(tmpdir): return create_run(tmpdir, '140101_M01234', '2130A*.fastq') +@pytest.fixture(name='raw_data_with_two_samples') +def create_raw_data_with_two_samples(tmpdir): + # Install samples 2110A and 2120A in a single run. + return create_run(tmpdir, '140101_M01234', '21[12]0A*.fastq') + + @pytest.fixture(name='raw_data_with_two_runs') def create_raw_data_with_two_runs(tmpdir): raw_data = create_run(tmpdir, '140101_M01234', '2000A*.fastq') @@ -185,12 +191,9 @@ def test_starts_empty(): assert not kive_watcher.is_full() -def test(raw_data_with_two_runs, mock_open_kive, default_config): - base_calls = (raw_data_with_two_runs / - "MiSeq/runs/140201_M01234/Data/Intensities/BaseCalls") - default_config.kive_server = 'https://devnull.com:8000' - default_config.kive_user = 'ishmael' - default_config.kive_password = 'moby' +def test_first_sample(raw_data_with_two_samples, mock_open_kive, default_config): + base_calls = (raw_data_with_two_samples / + "MiSeq/runs/140101_M01234/Data/Intensities/BaseCalls") mock_session = mock_open_kive.return_value mock_pipeline = mock_session.get_pipeline.return_value mock_input = mock_pipeline.inputs.__getitem__.return_value @@ -198,33 +201,63 @@ def test(raw_data_with_two_runs, mock_open_kive, default_config): kive_watcher.add_sample_group( base_calls=base_calls, - sample_group=SampleGroup('2010A', - ('2010A-V3LOOP_S3_L001_R1_001.fastq.gz', + sample_group=SampleGroup('2110A', + ('2110A-V3LOOP_S13_L001_R1_001.fastq.gz', None))) mock_open_kive.assert_called_once_with(default_config.kive_server) mock_session.login.assert_called_once_with(default_config.kive_user, default_config.kive_password) mock_session.create_run_batch.assert_called_once_with( - '140201_M01234 v0-dev', - description='MiCall batch for folder 140201_M01234, pipeline version v0-dev.', + '140101_M01234 v0-dev', + description='MiCall batch for folder 140101_M01234, pipeline version v0-dev.', users=[], groups=['Everyone']) mock_session.get_pipeline.assert_called_once_with( default_config.micall_filter_quality_pipeline_id) mock_session.find_datasets.assert_called_once_with( cdt=mock_input.compounddatatype, - name='140201_M01234_quality.csv', + name='140101_M01234_quality.csv', uploaded=True, md5='6861a4a0bfd71b62c0048ff9a4910223') # MD5 of header with no records. add_args_list = mock_session.add_dataset.call_args_list assert len(add_args_list) == 1 - assert add_args_list[0][1]['name'] == '140201_M01234_quality.csv' - assert add_args_list[0][1]['description'] == 'Error rates for 140201_M01234 run.' + assert add_args_list[0][1]['name'] == '140101_M01234_quality.csv' + assert add_args_list[0][1]['description'] == 'Error rates for 140101_M01234 run.' assert add_args_list[0][1]['cdt'] == mock_input.compounddatatype mock_session.run_pipeline.assert_called_once_with( mock_session.get_pipeline.return_value, [mock_session.add_dataset.return_value], - 'MiCall filter quality on 140201_M01234', + 'MiCall filter quality on 140101_M01234', + runbatch=mock_session.create_run_batch.return_value, + groups=['Everyone']) + + +def test(raw_data_with_two_samples, mock_open_kive, default_config): + base_calls = (raw_data_with_two_samples / + "MiSeq/runs/140101_M01234/Data/Intensities/BaseCalls") + mock_session = mock_open_kive.return_value + kive_watcher = KiveWatcher(default_config) + + kive_watcher.add_sample_group( + base_calls=base_calls, + sample_group=SampleGroup('2110A', + ('2110A-V3LOOP_S13_L001_R1_001.fastq.gz', + None))) + kive_watcher.add_sample_group( + base_calls=base_calls, + sample_group=SampleGroup('2120A', + ('2120A-PR_S14_L001_R1_001.fastq.gz', + None))) + + mock_session.create_run_batch.assert_called_once_with( + '140101_M01234 v0-dev', + description='MiCall batch for folder 140101_M01234, pipeline version v0-dev.', + users=[], + groups=['Everyone']) + mock_session.run_pipeline.assert_called_once_with( + mock_session.get_pipeline.return_value, + [mock_session.add_dataset.return_value], + 'MiCall filter quality on 140101_M01234', runbatch=mock_session.create_run_batch.return_value, groups=['Everyone'])