Fix broken error_metrics test, and switch to Pytest.

Still working on #438.

.gitignore

@@ -17,7 +17,7 @@
 /*.bsfs.json
 .cache
 .coverage
-/micall/tests/.pytest_cache
+/.pytest_cache
 /.idea
 /micall/tests/working/*
 /micall/tests/microtest/Results

.travis.yml

@@ -4,7 +4,7 @@ python:
 install: "pip install -r requirements-test.txt"
 before_script:
     - "cp micall/settings_default.py micall/settings.py"
-script: "coverage run --source=micall/core,micall/g2p,micall/resistance,micall/monitor -m unittest discover"
+script: "coverage run --source=micall/core,micall/g2p,micall/resistance,micall/monitor -m pytest"
 
 after_success:
     - pip install codecov

micall/monitor/error_metrics_parser.py

@@ -114,7 +114,7 @@ def write_phix_csv(out_file, records, read_lengths=None, summary=None):
             cycle = record[1]
             previous_cycle += sign
             while previous_cycle*sign < cycle*sign:
-                writer.writerow((record[0], previous_cycle))
+                writer.writerow((record[0], previous_cycle, ''))
                 previous_cycle += sign
             writer.writerow(record)
             summary_index = (sign+1) // 2

micall/monitor/kive_watcher.py

@@ -4,7 +4,6 @@
 from queue import Full
 
 from io import StringIO, BytesIO
-from requests.adapters import HTTPAdapter
 
 from micall.drivers.run_info import parse_read_sizes
 from micall.monitor import error_metrics_parser
@@ -17,6 +16,12 @@
     # Ignore import errors during testing.
     KiveAPI = RunStatus = None
 
+try:
+    from requests.adapters import HTTPAdapter
+except ImportError:
+    # Ignore import errors during testing.
+    HTTPAdapter = None
+
 logger = logging.getLogger(__name__)
 ALLOWED_GROUPS = ['Everyone']
 FOLDER_SCAN_INTERVAL = timedelta(hours=1)
@@ -25,6 +30,8 @@
 def open_kive(server_url):
     if KiveAPI is None:
         raise ImportError('Kive API failed to import. Is it installed?')
+    if HTTPAdapter is None:
+        raise ImportError('requests module failed to import. Is it installed?')
     session = KiveAPI(server_url)
     session.mount('https://', HTTPAdapter(max_retries=20))
     return session

micall/tests/microtest/README.md

@@ -39,3 +39,7 @@ The scenarios that each file tests are:
   reads with partial deletions in codons 53 and 54, 4 reads with insertions
   after codon 56, 3 soft clipped reads in codons 59-64, 1 read with a deletion
   in codons 65-67, followed by 18 clean codons with only 10 reads covering them.
+* 2130A-HCV - full coverage of the whole genome portion of NS5b, with a couple
+  of mutations. See the `make_sample.py` script for details.
+* 2130AMIDI-MidHCV - full coverage of the MIDI portion of NS5b, with a couple
+  of mutations. See the `make_sample.py` script for details.

micall/utils/find_chimera.py

@@ -26,8 +26,8 @@ def parse_args():
 
 def plot(counts, filename, window_size=31):
     window_counts = defaultdict(dict)  # {(seed, consensus): {mid_pos: (agree, disagree)}}
-    for names, agreement_counts in counts.iteritems():
-        for mid_pos in agreement_counts.iterkeys():
+    for names, agreement_counts in counts.items():
+        for mid_pos in agreement_counts.keys():
             hit_count = 0
             window_start = mid_pos - window_size/2
             window_end = window_start + window_size
@@ -44,7 +44,7 @@ def plot(counts, filename, window_size=31):
                                                  disagree_sum/hit_count)
 
     _fig, axes = plt.subplots(nrows=2, ncols=2)
-    for axis_index, names in enumerate(sorted(window_counts.iterkeys())):
+    for axis_index, names in enumerate(sorted(window_counts.keys())):
         axis = axes[axis_index % 2][axis_index / 2]
         seed_name, consensus_name = names
 #         axis.set_yscale('log')
@@ -53,7 +53,7 @@ def plot(counts, filename, window_size=31):
         axis.set_xlabel(consensus_name + ' nuc pos')
         axis.set_ylabel(seed_name + ' match counts')
         name_counts = window_counts[names]
-        x = sorted(name_counts.iterkeys())
+        x = sorted(name_counts.keys())
         y1 = [name_counts[pos][0] for pos in x]
         y2 = [sum(name_counts[pos]) for pos in x]
         axis.step(x, y1, 'r', linewidth=2, where='mid', label='agree')
@@ -90,8 +90,8 @@ def find_consensus(rows, seed_name, region_name):
             if nuc_count > max_count:
                 max_nucs[pos] = nuc
                 max_count = nuc_count
-    max_pos = max_nucs and max(max_nucs.iterkeys()) or 0
-    consensus = ''.join(max_nucs.get(i+1, 'N') for i in xrange(max_pos))
+    max_pos = max_nucs and max(max_nucs.keys()) or 0
+    consensus = ''.join(max_nucs.get(i+1, 'N') for i in range(max_pos))
     return consensus
 
 
@@ -110,13 +110,13 @@ def process_file(sample_name, projects, args):
         combo_rows = {combo: list(group)
                       for combo, group in groupby(region_rows,
                                                   itemgetter('seed', 'region'))}
-    seed_names = {combo[0] for combo in combo_rows.iterkeys()}
+    seed_names = {combo[0] for combo in combo_rows.keys()}
     seed_map = {seed_name: projects.getReference(seed_name)
                 for seed_name in seed_names}
     consensus_map = {combo: find_consensus(rows, *combo)
-                     for combo, rows in combo_rows.iteritems()}
+                     for combo, rows in combo_rows.items()}
     nucleotides = 'ACGT'
-    for combo, rows in combo_rows.iteritems():
+    for combo, rows in combo_rows.items():
         seed_name, _region_name = combo
         consensus = consensus_map[combo]
         for dest_seed_name in seed_names: