Convert logging settings to a Python file, as part of #438.

.gitignore

@@ -1,3 +1,4 @@
+/micall_logging_override.py
 /micall/settings.py
 /micall/monitor/HIV1_COM_2015_genome_DNA.csv
 /micall/monitor/compare_454_samples.csv

micall/core/aln2counts.py

@@ -14,13 +14,11 @@
 from collections import Counter, defaultdict, OrderedDict
 import csv
 from itertools import groupby
-import logging
 from operator import itemgetter
 import os
 
 import gotoh
 
-from micall.core import miseq_logging
 from micall.core.project_config import ProjectConfig, G2P_SEED_NAME
 from micall.utils.big_counter import BigCounter
 from micall.utils.translation import translate, ambig_dict
@@ -30,6 +28,7 @@
 GAP_OPEN_COORD = 40
 GAP_EXTEND_COORD = 10
 CONSENSUS_MIN_COVERAGE = 100
+MAX_CUTOFF = 'MAX'
 
 
 def parse_args():
@@ -70,11 +69,6 @@ def parse_args():
     return parser.parse_args()
 
 
-logger = miseq_logging.init_logging_console_only(logging.DEBUG)
-
-MAX_CUTOFF = 'MAX'
-
-
 class SequenceReport(object):
     """ Hold the data for several reports related to a sample's genetic sequence.
 

micall/core/filter_quality.py

@@ -1,19 +1,14 @@
-#! /usr/bin/env python3.4
-
 import argparse
 import csv
 import itertools
-import logging
 import math
 from operator import itemgetter
 import os
 
-from micall.core import miseq_logging
-
 BAD_ERROR_RATE = 7.5
 
 
-def parseArgs():
+def parse_args():
     parser = argparse.ArgumentParser(
         description='Post-processing of short-read alignments.')
 
@@ -26,8 +21,6 @@ def parseArgs():
 
     return parser.parse_args()
 
-logger = miseq_logging.init_logging_console_only(logging.DEBUG)
-
 
 def direction_grouper(cycle):
     return math.copysign(1, int(cycle['cycle']))
@@ -39,7 +32,9 @@ def report_bad_cycles(quality_csv, bad_cycles_csv, bad_tiles_csv=None):
                             ['tile', 'cycle', 'errorrate'],
                             lineterminator=os.linesep)
     writer.writeheader()
-    if bad_tiles_csv is not None:
+    if bad_tiles_csv is None:
+        tile_writer = None
+    else:
         tile_writer = csv.DictWriter(bad_tiles_csv,
                                      ['tile', 'bad_cycles'],
                                      lineterminator=os.linesep)
@@ -58,12 +53,12 @@ def report_bad_cycles(quality_csv, bad_cycles_csv, bad_tiles_csv=None):
                 if is_bad:
                     writer.writerow(cycle)
                     bad_cycle_count += 1
-        if bad_tiles_csv is not None:
+        if tile_writer is not None:
             tile_writer.writerow(dict(tile=tile, bad_cycles=bad_cycle_count))
 
 
 def main():
-    args = parseArgs()
+    args = parse_args()
     with args.quality_csv, args.bad_cycles_csv:
         report_bad_cycles(args.quality_csv, args.bad_cycles_csv)
 

micall/core/prelim_map.py

@@ -1,7 +1,5 @@
-#! /usr/bin/env python3.4
-
 """
-Shipyard-style bowtie2
+Kive-style bowtie2
 Run bowtie2 on paired-end FASTQ data sets with user-supplied *.bt2
 bowtie2 SAM format output to <stdout> for redirection via subprocess.Popen
 Sort outputs by refname.
@@ -14,7 +12,6 @@
 import os
 import sys
 
-from micall.core import miseq_logging
 from micall.core import project_config
 from micall.utils.externals import Bowtie2, Bowtie2Build, LineCounter
 
@@ -28,7 +25,7 @@
 REF_GAP_OPEN = 10
 REF_GAP_EXTEND = 3
 
-logger = miseq_logging.init_logging_console_only(logging.DEBUG)
+logger = logging.getLogger(__name__)
 line_counter = LineCounter()
 
 

micall/core/remap.py

@@ -1,5 +1,3 @@
-#! /usr/bin/env python3.4
-
 """
 Takes preliminary SAM as CSV input.  Iterative re-mapping of reads from
 original FASTQ files.
@@ -11,17 +9,18 @@
 from collections import Counter, defaultdict
 import csv
 from functools import partial
-import logging
+from logging import getLogger
 import os
 import re
 import shutil
 import sys
 
 # noinspection PyUnresolvedReferences
 from gotoh import align_it
+
 import Levenshtein
 
-from micall.core import miseq_logging, project_config
+from micall.core import project_config
 from micall.core.sam2aln import apply_cigar, merge_pairs, merge_inserts
 from micall.core.prelim_map import BOWTIE_BUILD_PATH, \
     BOWTIE_PATH, BOWTIE_VERSION, READ_GAP_OPEN, READ_GAP_EXTEND, REF_GAP_OPEN, \
@@ -53,7 +52,7 @@
 
 cigar_re = re.compile('[0-9]+[MIDNSHPX=]')  # CIGAR token
 
-logger = miseq_logging.init_logging_console_only(logging.DEBUG)
+logger = getLogger(__name__)
 indel_re = re.compile('[+-][0-9]+')
 line_counter = LineCounter()
 
@@ -262,7 +261,7 @@ def sam_to_conseqs(samfile,
                 drifted_seeds.append((read_counts[name], name))
                 continue
 
-            other_seed = other_dist = None
+            other_seed = other_dist = seed_dist = None
             for seed_name in sorted(new_conseqs.keys()):
                 seed_ref = original_seeds[seed_name]
                 aligned_seed, aligned_conseq, _score = align_it(seed_ref,

micall/monitor/kive_download.py

@@ -16,9 +16,8 @@
     KiveAPI = RunStatus = None
 from requests.adapters import HTTPAdapter
 
-from micall.settings import kive_server_url, kive_user, kive_password, home, \
+from micall.settings import kive_server_url, kive_user, kive_password, \
     kive_pipelines
-from micall.core.miseq_logging import init_logging
 
 
 def parse_args():
@@ -200,9 +199,7 @@ def find_batch_runs(kive, batch_name, batch_size):
 
 
 def main():
-    logger = init_logging(os.path.join(home, 'kive_download.log'),
-                          file_log_level=logging.INFO,
-                          console_log_level=logging.INFO)
+    logger = logging.getLogger(__name__)
     args = parse_args()
 
     logger.info('Starting.')
@@ -232,5 +229,6 @@ def main():
         download_results(runs, args.resultfolder, args.workfolder)
     logger.info('%d runs found (%d unfinished).', len(runs), unfinished_count)
 
+
 if __name__ == '__main__':
     main()

micall/monitor/kive_watcher.py

@@ -186,6 +186,9 @@ def is_full(self):
                            for folder_watcher in self.folder_watchers.values())
         return active_count >= self.config.max_active
 
+    def is_idle(self):
+        return not self.folder_watchers
+
     def get_kive_pipeline(self, pipeline_id):
         self.check_session()
         kive_pipeline = self.pipelines.get(pipeline_id)

micall/utils/remap_fastq_simplify.py

@@ -1,10 +1,8 @@
 import csv
 from collections import defaultdict
-from itertools import imap
 import logging
 import os
 
-from micall.core.miseq_logging import init_logging_console_only
 from micall.core.remap import remap
 from micall.core.prelim_map import prelim_map
 from csv import DictReader
@@ -159,7 +157,7 @@ def get_result(self, censored_map_count, trimmed_map_count):
         return DD.FAIL if diff >= 20 else DD.PASS
 
     def write_simple_fastq(self, filename1, read_indexes):
-        selected_reads = imap(self.reads.__getitem__, read_indexes)
+        selected_reads = map(self.reads.__getitem__, read_indexes)
         filename2 = get_reverse_filename(filename1)
         with open(filename1, 'w') as f1, open(filename2, 'w') as f2:
             for lines in selected_reads:
@@ -219,7 +217,7 @@ def read_fastq(filename, reads):
 
 
 def main():
-    logger = init_logging_console_only(logging.INFO)
+    logger = logging.getLogger(__name__)
     try:
         logger.info('Starting.')
         fname = 'censored1.fastq'