fix: update CI workflow, snakemake wrappers and python / pandas code syntax

.github/workflows/main.yml

@@ -2,19 +2,22 @@ name: Tests
 
 on:
   push:
-    branches: [ main ]
+    branches: 
+      - main
   pull_request:
-    branches: [ main ]
+    branches_ignore: []
 
 jobs:
   formatting:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Checkout submodules
-      uses: textbook/git-checkout-submodule-action@2.0.0
+    - name: Checkout with submodules
+      uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        fetch-depth: 0
     - name: Formatting
-      uses: github/super-linter@v3.16.1
+      uses: github/super-linter@v5
       env:
         VALIDATE_ALL_CODEBASE: false
         DEFAULT_BRANCH: main
@@ -24,13 +27,11 @@ jobs:
   linting:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Checkout submodules
-      uses: textbook/git-checkout-submodule-action@2.0.0
+    - uses: actions/checkout@v3
     - name: Linting
-      uses: snakemake/snakemake-github-action@v1.17.0
+      uses: snakemake/snakemake-github-action@v1.22.0
       with:
-        directory: '.test'
+        directory: .test
         snakefile: workflow/Snakefile
         args: '--lint'
 
@@ -40,14 +41,13 @@ jobs:
       - formatting
       - linting
     steps:
-    - uses: actions/checkout@v1
-    - name: Checkout submodules
-      uses: textbook/git-checkout-submodule-action@2.0.0
-
-    - name: snakemake
-      uses: snakemake/snakemake-github-action@v1.17.0
+    - name: Checkout repository with submodules
+      uses: actions/checkout@v3
+      with:
+        submodules: recursive
+    - name: Test workflow
+      uses: snakemake/snakemake-github-action@v1.22.0
       with:
         directory: .test
         snakefile: workflow/Snakefile
-        args: "--use-conda --show-failed-logs --cores 1 --conda-cleanup-pkgs cache"
-
+        args: "--use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache"

.gitignore

@@ -1,33 +1,10 @@
-*
-!scripts
-!scripts/*
-!scripts/common
-!scripts/common/*
-scripts/.snakemake*
-!Snakefile
-!config.yaml
-!samples.tsv
-!resources
-!resources/*
-!envs
-!envs/*
-!environment.yaml
-!LICENSE
-!README.md
-!rules
-!rules/*
-!.gitignore
-!.gitattributes
-!.editorconfig
-!.travis.yml
-!.test
-!samples.tsv
-!units.tsv
-!schemas
-!schemas/*
-!.test/data
-!.test/samples.tsv
-!.test/units.tsv
-!.test/config.yaml
-!report
-!report/*.rst
+.test/benchmarks/**
+.test/logs/**
+.test/results/**
+.test/resources/**
+.test/.snakemake/**
+benchmarks/**
+logs/**
+resources/**
+results/**
+.snakemake/**

.test/config/config.yaml

@@ -30,7 +30,7 @@ params:
     GenotypeGVCFs: ""
     VariantRecalibrator: ""
   picard:
-    MarkDuplicates: "REMOVE_DUPLICATES=true"
+    MarkDuplicates: "--REMOVE_DUPLICATES true"
   trimmomatic:
     pe:
       trimmer:

config/config.yaml

@@ -37,7 +37,7 @@ params:
     GenotypeGVCFs: ""
     VariantRecalibrator: ""
   picard:
-    MarkDuplicates: "REMOVE_DUPLICATES=true"
+    MarkDuplicates: "--REMOVE_DUPLICATES true"
   trimmomatic:
     pe:
       trimmer:

workflow/rules/annotation.smk

@@ -23,4 +23,4 @@ rule annotate_variants:
         "logs/vep/annotate.log",
     threads: 4
     wrapper:
-        "0.74.0/bio/vep/annotate"
+        "v1.31.1/bio/vep/annotate"

workflow/rules/calling.smk

@@ -30,7 +30,7 @@ rule call_variants:
     params:
         extra=get_call_variants_params,
     wrapper:
-        "0.59.0/bio/gatk/haplotypecaller"
+        "v1.31.1/bio/gatk/haplotypecaller"
 
 
 rule combine_calls:
@@ -44,7 +44,7 @@ rule combine_calls:
     log:
         "logs/gatk/combinegvcfs.{contig}.log",
     wrapper:
-        "0.74.0/bio/gatk/combinegvcfs"
+        "v1.31.1/bio/gatk/combinegvcfs"
 
 
 rule genotype_variants:
@@ -58,7 +58,7 @@ rule genotype_variants:
     log:
         "logs/gatk/genotypegvcfs.{contig}.log",
     wrapper:
-        "0.74.0/bio/gatk/genotypegvcfs"
+        "v1.31.1/bio/gatk/genotypegvcfs"
 
 
 rule merge_variants:
@@ -71,4 +71,4 @@ rule merge_variants:
     log:
         "logs/picard/merge-genotyped.log",
     wrapper:
-        "0.74.0/bio/picard/mergevcfs"
+        "v1.31.1/bio/picard/mergevcfs"

workflow/rules/common.smk

@@ -38,10 +38,11 @@ wildcard_constraints:
 
 ##### Helper functions #####
 
+
 # contigs in reference genome
 def get_contigs():
     with checkpoints.genome_faidx.get().output[0].open() as fai:
-        return pd.read_table(fai, header=None, usecols=[0], squeeze=True, dtype=str)
+        return pd.read_table(fai, header=None, usecols=[0], dtype=str).squeeze("columns")
 
 
 def get_fastq(wildcards):
@@ -70,9 +71,7 @@ def get_trimmed_reads(wildcards):
     if not is_single_end(**wildcards):
         # paired-end sample
         return expand(
-            "results/trimmed/{sample}-{unit}.{group}.fastq.gz",
-            group=[1, 2],
-            **wildcards
+            "results/trimmed/{sample}-{unit}.{group}.fastq.gz", group=[1, 2], **wildcards
         )
     # single end sample
     return "results/trimmed/{sample}-{unit}.fastq.gz".format(**wildcards)

workflow/rules/filtering.smk

@@ -9,7 +9,7 @@ rule select_calls:
     log:
         "logs/gatk/selectvariants/{vartype}.log",
     wrapper:
-        "0.59.0/bio/gatk/selectvariants"
+        "v1.31.1/bio/gatk/selectvariants"
 
 
 rule hard_filter_calls:
@@ -23,7 +23,7 @@ rule hard_filter_calls:
     log:
         "logs/gatk/variantfiltration/{vartype}.log",
     wrapper:
-        "0.74.0/bio/gatk/variantfiltration"
+        "v1.31.1/bio/gatk/variantfiltration"
 
 
 rule recalibrate_calls:
@@ -36,7 +36,7 @@ rule recalibrate_calls:
     log:
         "logs/gatk/variantrecalibrator/{vartype}.log",
     wrapper:
-        "0.74.0/bio/gatk/variantrecalibrator"
+        "v1.31.1/bio/gatk/variantrecalibrator"
 
 
 rule merge_calls:
@@ -53,4 +53,4 @@ rule merge_calls:
     log:
         "logs/picard/merge-filtered.log",
     wrapper:
-        "0.74.0/bio/picard/mergevcfs"
+        "v1.31.1/bio/picard/mergevcfs"

workflow/rules/mapping.smk

@@ -9,7 +9,7 @@ rule trim_reads_se:
     log:
         "logs/trimmomatic/{sample}-{unit}.log",
     wrapper:
-        "0.74.0/bio/trimmomatic/se"
+        "v1.31.1/bio/trimmomatic/se"
 
 
 rule trim_reads_pe:
@@ -27,7 +27,7 @@ rule trim_reads_pe:
     log:
         "logs/trimmomatic/{sample}-{unit}.log",
     wrapper:
-        "0.74.0/bio/trimmomatic/pe"
+        "v1.31.1/bio/trimmomatic/pe"
 
 
 rule map_reads:
@@ -45,28 +45,28 @@ rule map_reads:
         sort_order="coordinate",
     threads: 8
     wrapper:
-        "0.74.0/bio/bwa/mem"
+        "v1.31.1/bio/bwa/mem"
 
 
 rule mark_duplicates:
     input:
-        "results/mapped/{sample}-{unit}.sorted.bam",
+        bams="results/mapped/{sample}-{unit}.sorted.bam",
     output:
         bam=temp("results/dedup/{sample}-{unit}.bam"),
         metrics="results/qc/dedup/{sample}-{unit}.metrics.txt",
     log:
         "logs/picard/dedup/{sample}-{unit}.log",
     params:
-        config["params"]["picard"]["MarkDuplicates"],
+        extra=f'--VALIDATION_STRINGENCY LENIENT --ASSUME_SORT_ORDER coordinate {config["params"]["picard"]["MarkDuplicates"]}',
     wrapper:
-        "0.74.0/bio/picard/markduplicates"
+        "v1.31.1/bio/picard/markduplicates"
 
 
 rule recalibrate_base_qualities:
     input:
         bam=get_recal_input(),
         bai=get_recal_input(bai=True),
-        ref="resources/genome.fasta",
+        kef="resources/genome.fasta",
         dict="resources/genome.dict",
         known="resources/variation.noiupac.vcf.gz",
         known_idx="resources/variation.noiupac.vcf.gz.tbi",
@@ -79,7 +79,7 @@ rule recalibrate_base_qualities:
     resources:
         mem_mb=1024,
     wrapper:
-        "0.74.0/bio/gatk/baserecalibrator"
+        "v1.31.1/bio/gatk/baserecalibrator"
 
 
 rule apply_base_quality_recalibration:
@@ -98,7 +98,7 @@ rule apply_base_quality_recalibration:
     resources:
         mem_mb=1024,
     wrapper:
-        "0.74.0/bio/gatk/applybqsr"
+        "v1.31.1/bio/gatk/applybqsr"
 
 
 rule samtools_index:
@@ -109,4 +109,4 @@ rule samtools_index:
     log:
         "logs/samtools/index/{prefix}.log",
     wrapper:
-        "0.74.0/bio/samtools/index"
+        "v1.31.1/bio/samtools/index"

workflow/rules/qc.smk

@@ -7,7 +7,7 @@ rule fastqc:
     log:
         "logs/fastqc/{sample}-{unit}.log",
     wrapper:
-        "0.74.0/bio/fastqc"
+        "v1.31.1/bio/fastqc"
 
 
 rule samtools_stats:
@@ -18,7 +18,7 @@ rule samtools_stats:
     log:
         "logs/samtools-stats/{sample}-{unit}.log",
     wrapper:
-        "0.74.0/bio/samtools/stats"
+        "v1.31.1/bio/samtools/stats"
 
 
 rule multiqc:
@@ -40,4 +40,4 @@ rule multiqc:
     log:
         "logs/multiqc.log",
     wrapper:
-        "0.74.0/bio/multiqc"
+        "v1.31.1/bio/multiqc"

workflow/rules/ref.smk

@@ -10,7 +10,7 @@ rule get_genome:
         release=config["ref"]["release"],
     cache: True
     wrapper:
-        "0.74.0/bio/reference/ensembl-sequence"
+        "v1.31.1/bio/reference/ensembl-sequence"
 
 
 checkpoint genome_faidx:
@@ -22,7 +22,7 @@ checkpoint genome_faidx:
         "logs/genome-faidx.log",
     cache: True
     wrapper:
-        "0.74.0/bio/samtools/faidx"
+        "v1.31.1/bio/samtools/faidx"
 
 
 rule genome_dict:
@@ -54,7 +54,7 @@ rule get_known_variation:
         type="all",
     cache: True
     wrapper:
-        "0.74.0/bio/reference/ensembl-variation"
+        "v1.31.1/bio/reference/ensembl-variation"
 
 
 rule remove_iupac_codes:
@@ -82,21 +82,21 @@ rule tabix_known_variants:
         "-p vcf",
     cache: True
     wrapper:
-        "0.74.0/bio/tabix"
+        "v1.31.1/bio/tabix/index"
 
 
 rule bwa_index:
     input:
         "resources/genome.fasta",
     output:
-        multiext("resources/genome.fasta", ".amb", ".ann", ".bwt", ".pac", ".sa"),
+        idx=multiext("resources/genome.fasta", ".amb", ".ann", ".bwt", ".pac", ".sa"),
     log:
         "logs/bwa_index.log",
     resources:
         mem_mb=369000,
     cache: True
     wrapper:
-        "0.74.0/bio/bwa/index"
+        "v1.31.1/bio/bwa/index"
 
 
 rule get_vep_cache:
@@ -109,7 +109,7 @@ rule get_vep_cache:
     log:
         "logs/vep/cache.log",
     wrapper:
-        "0.74.0/bio/vep/cache"
+        "v1.31.1/bio/vep/cache"
 
 
 rule get_vep_plugins:
@@ -120,4 +120,4 @@ rule get_vep_plugins:
     params:
         release=config["ref"]["release"],
     wrapper:
-        "0.74.0/bio/vep/plugins"
+        "v1.31.1/bio/vep/plugins"