Merge branch '3-manage-single-and-paired' into 'main'

Resolve "manage single and paired"

Closes #3

See merge request !3

.gitlab-ci.yml

+image: registry.forge.ird.fr/diade/culebront_pipeline/podman/gitlab-runner:0.0.1 # docker image used
+####################
+# TAG stages: we define 3 test steps called install, test, deploy
+stages:
+  - install
+  - test
+#variables:
+#  CI_DEBUG_TRACE: "true"
+
+####################
+# TAG before_script : we list all the commands required before running any test. eg.: clone repo, install soft
+# at each stage is launched
+before_script:
+  - echo ${CI_PROJECT_DIR}
+  - export PATH=$PATH:/root/.local/bin # ajouter les binaires dans le home
+  - export APPTAINER_BINDPATH="${CI_PROJECT_DIR}:${CI_PROJECT_DIR},/root:/root" # bin automatique de l'aptainer pour ne pas avoir a le mettre dans SMK
+  #- python3 -m pip install -U snakemake setuptools build pip
+  #- python3 -m pip install snakecdysis@git+https://forge.ird.fr/phim/sravel/snakecdysis.git@main
+  - python3 -m pip install --user RNAja@git+https://forge.ird.fr/phim/rnaja-pipeline.git@$CI_COMMIT_REF_NAME
+  - RNAja --install_env
+  #- bash -ec 'CPDIR="/root/.local/lib/python3.10/site-packages/culebrONT/containers"; DIR="/usr/local/culebrONT/Containers"; if [[ -d $DIR ]]; then ls $DIR;  echo "copy Containers files in $DIR to $CPDIR"; ln -sf $DIR/* $CPDIR; fi'
+  - RNAja install_local
+
+####### COMMON (Install) #######
+RNAja:installing:
+  stage: install                       # Put the name of stage defined previously
+  script:                              # shell commands executed by the runner
+    - RNAja --install_env
+  tags:
+    - VM
+  rules:                               # Condition if the test is run or not
+    - if: $CI_COMMIT_TAG =~ "/^v.*rc/" # Tests ran if commit message start with v.
+    - if: $CI_COMMIT_MESSAGE =~ "/.*-TEST.*/i"
+
+# @Description TEST single
+RNAja:single:
+  stage: test
+  artifacts:
+    name: RNAja-config_single
+    paths:
+      - test/RNAJA_OUTPUT
+  script:
+    - RNAja --install_env
+    - RNAja test_install -d test
+    - RNAja run_local -t 12 -c ./RNAja/install_files/config_test/RNAja-config_single.yaml;
+  tags:
+    - VM
+  rules:
+    - if: $CI_COMMIT_TAG =~ "/^v.*rc/" # Tests ran if commit message start with v.
+    - if: $CI_COMMIT_MESSAGE =~ "/.*-TEST.*/i"
+
+
+# @Description TEST paired
+RNAja:paired:
+  stage: test
+  artifacts:
+    name: RNAja-config_paired
+    paths:
+      - test/RNAJA_OUTPUT
+  script:
+    - RNAja --install_env
+    - RNAja test_install -d test
+    - RNAja run_local -t 12 -c ./RNAja/install_files/config_test/RNAja-config_paired.yaml
+  tags:
+    - VM
+  rules:
+    - if: $CI_COMMIT_TAG =~ "/^v.*rc/" # Tests ran if commit message start with v.
+    - if: $CI_COMMIT_MESSAGE =~ "/.*-TEST.*/i"

RNAja/__init__.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 from pathlib import Path
 from .global_variables import GIT_URL, DOCS, DATATEST_URL_FILES, SINGULARITY_URL_FILES
+from RNAja.module import Rnaja
 
 logo = Path(__file__).parent.resolve().joinpath('RNAja_logo.png').as_posix()
 

RNAja/global_variables.py

@@ -10,5 +10,6 @@ SINGULARITY_URL_FILES = [('https://itrop.ird.fr/RNAja_utilities/Singularity.RNAj
 
 DATATEST_URL_FILES = ("https://itrop.ird.fr/RNAja_utilities/DATA_TEST.tar.gz", "DATA_TEST.tar.gz")
 
+ALLOW_FASTQ_EXT = (".fastq", ".fq", ".fq.gz", ".fastq.gz")
 
 

RNAja/install_files/config.yaml

 # Input output data options
 DATA:
-    directories:
-        out_dir: "DATA_DIR/RNAJA_OUTPUT"
-    files:
-        reference: "DATA_DIR/DATA_TEST/REF/GCF_000146045.2_R64_genomic.fna"
-        annotation: "DATA_DIR/DATA_TEST/REF/GCF_000146045.2_R64_genomic.gtf"
-        sample_info: "DATA_DIR/sample_info.txt"
-        de_comparisons_file: "DATA_DIR/comparisons.csv"
+    fastq_dir: "DATA/DATA_TEST/FASTQ_PAIRED"
+    reference: "DATA/DATA_TEST/REF/GCF_000146045.2_R64_genomic.fna"
+    annotation: "DATA/DATA_TEST/REF/GCF_000146045.2_R64_genomic.gtf"
+    sample_info: "DATA/DATA_TEST/sample_info.txt"
+    de_comparisons: "DATA/DATA_TEST/treatmentsComparisons.csv"
+    output_dir: "RNAJA_OUTPUT"
+    PAIRED : true # if true should be _R1 / _R2
+
 PARAMS:
+    HISAT2:
+        indexation:
+            prefix: "REF"
     STRINGTIE:
         discovery_mode : true
-

RNAja/install_files/config_test/RNAja-config_paired.yaml

+# Input output data options
+DATA:
+    fastq_dir: "DATA/DATA_TEST/FASTQ_PAIRED"
+    reference: "DATA/DATA_TEST/REF/GCF_000146045.2_R64_genomic.fna"
+    annotation: "DATA/DATA_TEST/REF/GCF_000146045.2_R64_genomic.gtf"
+    sample_info: "DATA/DATA_TEST/sample_info.txt"
+    de_comparisons: "DATA/DATA_TEST/treatmentsComparisons.csv"
+    output_dir: "RNAJA_OUTPUT"
+    PAIRED : true # if true should be _R1 / _R2
+
+PARAMS:
+    HISAT2:
+        indexation:
+            prefix: "REF"
+    STRINGTIE:
+        discovery_mode : true

RNAja/install_files/config_test/RNAja-config_single.yaml

+# Input output data options
+DATA:
+    fastq_dir: "DATA/DATA_TEST/FASTQ_SINGLE"
+    reference: "DATA/DATA_TEST/REF/GCF_000146045.2_R64_genomic.fna"
+    annotation: "DATA/DATA_TEST/REF/GCF_000146045.2_R64_genomic.gtf"
+    sample_info: "DATA/DATA_TEST/sample_info.txt"
+    de_comparisons: "DATA/DATA_TEST/treatmentsComparisons.csv"
+    output_dir: "RNAJA_OUTPUT"
+    PAIRED : false # if true should be _R1 / _R2
+
+PARAMS:
+    HISAT2:
+        indexation:
+            prefix: "REF"
+    STRINGTIE:
+        discovery_mode : true

RNAja/module.py

@@ -8,16 +8,28 @@ import re
 from .global_variables import *
 from snakecdysis import *
 
-
-class RNAja(SnakEcdysis):
+class Rnaja(SnakEcdysis):
     """
     to read file config
     """
-
     def __init__(self, dico_tool, workflow, config):
         super().__init__(**dico_tool, workflow=workflow, config=config)
-        # workflow is available only in __init__
-        # print("\n".join(list(workflow.__dict__.keys())))
-        # print(workflow.__dict__)
 
         # Initialisation of RNAja attributes
+        self.fastq_files_list = None
+        self.fastq_files_ext = None
+        self.fastq_gzip = None
+        self.samples_names_list = None
+
+        # populating fastq files, extension and compression
+        self.fastq_files_list, self.fastq_files_ext, self.fastq_gzip = get_files_ext2(
+            self.get_config_value('DATA', 'fastq_dir'), ALLOW_FASTQ_EXT)
+
+        #populating samples identifiers
+        self.fastq_files_list, self.fastq_files_ext, self.fastq_gzip = get_files_ext2(
+        self.get_config_value('DATA', 'fastq_dir'), ALLOW_FASTQ_EXT)
+        self.samples_names_list = set([Path(elm).stem.split("_")[0] if "_" in elm else Path(
+        elm).stem if not self.fastq_gzip else Path(elm).stem.split(".")[0] for elm in
+                                 self.fastq_files_list])
+
+

pyproject.toml

@@ -18,7 +18,7 @@ description = "RNAja is a pipeline to analyse RNAseq data"
 authors = [
     {name = "Aurore Comte (IRD)",email = "aurore.comte@ird.fr"},
 ]
-dependencies = ["PyYAML", "click>=8.0.3", "cookiecutter", "docutils", "python-gitlab", "snakemake", "tqdm", 'cookiecutter', 'snakecdysis']
+dependencies = ["PyYAML", "click>=8.0.3", "cookiecutter", "docutils", "python-gitlab", "snakemake", "tqdm", 'cookiecutter', 'snakecdysis', 'pandas']
 #dependencies = ['PyYAML', 'pandas', 'matplotlib', 'tabulate', 'ipython', 'biopython', 'numpy',  'argparse', 'snakemake', 'tqdm', 'click>=8.0.3', 'cookiecutter', 'snakecdysis']
 requires-python = ">=3.8"
 readme = "README.md"