From a2ff589d01fd75a10c9ada02b9afd2cd8e668a9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20FERNANDEZ=20NU=C3=91EZ?=
<nicolas.fernandez@ird.fr>
Date: Fri, 21 Mar 2025 16:26:47 +0100
Subject: [PATCH] Fix typo l. 66 Run_GeVarLi.sh (if-fi) v.2025.03_001
---
Run_GeVarLi.sh | 5 +-
workflow/rules/symlinks_renamming.smk | 16 +-
workflow/rules/workflow_reporting.smk | 61 ++---
.../__pycache__/functions.cpython-312.pyc | Bin 6453 -> 6456 bytes
workflow/scripts/functions.py | 3 +-
workflow/scripts/settings.py | 214 ++++++++++++++++++
6 files changed, 243 insertions(+), 56 deletions(-)
create mode 100644 workflow/scripts/settings.py
diff --git a/Run_GeVarLi.sh b/Run_GeVarLi.sh
index 7759f91..b0fc8bf 100755
--- a/Run_GeVarLi.sh
+++ b/Run_GeVarLi.sh
@@ -63,6 +63,7 @@ else # If yes, intern shell source conda
source ~/mambaforge/etc/profile.d/conda.sh 2> /dev/null # local user with mambaforge ¡ Deprecated !
source ~/miniconda3/etc/profile.d/conda.sh 2> /dev/null # local user with miniconda3 ¡ Deprecated !
source /usr/local/bioinfo/miniconda3-23.10.0-1/etc/profile.d/conda.sh 2> /dev/null # iTROP HPC server (conda 23.11.0)
+fi
###############################################################################
### SPINNER ###
@@ -195,7 +196,7 @@ snakemake \
echo -e "\n ${green} > Snakemake: run${nc} \n"
snakemake \
--directory ${workdir}/ \
- --snakefile ${workdir}/workflow/Snakefile\
+ --snakefile ${workdir}/workflow/Snakefile \
--cores ${max_threads} \
--resources mem_gb=${max_memory} \
--rerun-incomplete \
@@ -211,4 +212,4 @@ snakemake \
echo -e "\n Deactivate ${ylo}Workflow-Core${nc} conda environment."
conda deactivate
-###############################################################################
\ No newline at end of file
+###############################################################################
diff --git a/workflow/rules/symlinks_renamming.smk b/workflow/rules/symlinks_renamming.smk
index eb72ba6..f4bfce3 100644
--- a/workflow/rules/symlinks_renamming.smk
+++ b/workflow/rules/symlinks_renamming.smk
@@ -20,21 +20,17 @@
###############################################################################
rule config:
# Aim: Load configuration file
- # Use: yq -r . <CONFIG_FILE>
+ # Use:
message:
"""
~ Configuration ∞ Show analyses settings ~
"""
- conda:
- WORKFLOW
input:
config_file = "config/config.yaml"
output:
- done = temp("done.temp")
- shell:
- "bash workflow/scripts/settings.sh "
- "&& "
- "touch {output.done}"
+ setting_log = "results/10_Reports/settings.log"
+ script:
+ "workflow/scripts/settings.py "
###############################################################################
rule symlinks:
@@ -47,8 +43,8 @@ rule symlinks:
Reads: ___________ R{wildcards.mate}
"""
input:
- valid_fastq = lambda wildcards: os.path.abspath(VALID_FASTQ[wildcards.sample][wildcards.mate]),
- done = "done.temp"
+ setting_log = "results/10_Reports/settings.log",
+ valid_fastq = lambda wildcards: os.path.abspath(VALID_FASTQ[wildcards.sample][wildcards.mate])
output:
symlink = temp("results/symlinks/{sample}_R{mate}.fastq.gz")
log:
diff --git a/workflow/rules/workflow_reporting.smk b/workflow/rules/workflow_reporting.smk
index ee6152f..da31bbb 100644
--- a/workflow/rules/workflow_reporting.smk
+++ b/workflow/rules/workflow_reporting.smk
@@ -50,14 +50,14 @@ rule snakemake_report:
"""
~ Report ∞ Generate a workflow report in HTML format ~
"""
- conda:
- WORKFLOW
+ #conda:
+ # WORKFLOW
params:
#style_sheet = STYLE_SHEET
input:
multiqc = "results/10_Reports/multiqc/",
time = "results/10_Reports/time.log",
- summary = "results/10_Reports/files-summary.txt",
+ summary = "results/10_Reports/files-summary.tsv",
graph = expand("results/10_Reports/graphs/{graph_type}.{ext}",
graph_type = GRAPH_TYPE,
ext = GRAPH_EXT),
@@ -85,7 +85,7 @@ rule snakemake_summary:
input:
final_outputs = get_final_outputs()
output:
- summary = "results/10_Reports/files-summary.txt"
+ summary = "results/10_Reports/files-summary.tsv"
log:
"results/10_Reports/tools-log/files-summary.log"
shell:
@@ -166,52 +166,27 @@ rule log_time:
output:
time_log = "results/10_Reports/time.log"
run:
- time_stamp_start = time.strftime("%Y-%m-%d %H:%M", time.localtime(start_time)) # Get system: analyzes starting time
- time_stamp_end = time.strftime("%Y-%m-%d %H:%M", time.localtime()) # Get date / hour ending analyzes
- elapsed_time = int(time.time() - start_time) # Get SECONDS counter
- hours = elapsed_time // 3600 # /3600 = hours
- minutes = (elapsed_time % 3600) // 60 # %3600 /60 = minutes
- seconds = elapsed_time % 60 # %60 = seconds
- formatted_time = f"{hours:02d}:{minutes:02d}:{seconds:02d}" # Format
- green = "\033[32m" # ANSI green color code
- ylo = "\033[33m" # ANSI yellow color code
- nc = "\033[0m" # ANSI no-color code
+ time_stamp_start = time.strftime("%Y-%m-%d %H:%M", time.localtime(START_TIME))
+ time_stamp_end = time.strftime("%Y-%m-%d %H:%M", time.localtime())
+ elapsed_time = int(time.time() - START_TIME)
+ hours = elapsed_time // 3600
+ minutes = (elapsed_time % 3600) // 60
+ seconds = elapsed_time % 60
+ formatted_time = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+ green = "\033[32m"
+ ylo = "\033[33m"
+ nc = "\033[0m"
message_time = f"""
{green}Start time{nc} _____________ {time_stamp_start}
{green}End time{nc} _______________ {time_stamp_end}
{green}Processing time{nc} ________ {ylo}{formatted_time}{nc}
"""
- print(message_time) # Print time message
- ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') # ANSI escape code
- message_clean = ansi_escape.sub('', message_time) # Clean ANSI escape code
- with open(output.time_log, "w") as f: # Log time message
+ print(message_time)
+ ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+ message_clean = ansi_escape.sub('', message_time)
+ with open(output.time_log, "w") as f:
f.write(message_clean)
-###############################################################################
-rule log_setup:
- # Aim: log user setup
- # Use:
- message:
- """
- ~ Log ∞ User setup ~
- """
- input:
- setup = "config/config.yaml",
- output:
- setup_log = "results/10_Reports/setup.log"
- run:
- import subprocess
- uname = subprocess.check_output(["uname", "-a"]).decode().strip()
- fastq_dir = subprocess.check_output(["yq", "-Mr", ".fastq_dir", input.setup]).decode().strip()
- try:
- fastq_files = subprocess.check_output(["bash", "-c", "ls -1 {}/*.fastq.gz 2>/dev/null | wc -l".format(fastq_dir)]).decode().strip()
- except Exception:
- fastq_files = "0"
- with open(output.setup_log, "w") as f:
- f.write("OS info: " + uname + "\n")
- f.write("Fastq directory: " + fastq_dir + "\n")
- f.write("Number of fastq files: " + fastq_files + "\n")
-
###############################################################################
rule log_environments:
# Aim: copy conda environments files to results
diff --git a/workflow/scripts/__pycache__/functions.cpython-312.pyc b/workflow/scripts/__pycache__/functions.cpython-312.pyc
index 09825b982dd622ec6f2390258070bd704c9da8c8..039460ac12bd9424bf39c41de31f949261905bb6 100644
GIT binary patch
delta 70
zcmdmLw8Mz^G%qg~0}!wm-A(V`$a_(QQGW7$5i!=1%)IpC&EG}RnFRzGScR^$NL*x*
YxFBhDg~fVup2R-R1V%>5B4waP0QJcf2mk;8
delta 67
zcmdmCwAG0BG%qg~0}!k{dn-L?Bkx5KMw!X?MZ}m&3pW1}NoN+|V_+4!&LVM<MdE^_
V`4tw6$%PX8IO7=^C5x1Sx&YCr6MO&w
diff --git a/workflow/scripts/functions.py b/workflow/scripts/functions.py
index fc28f23..e55beb1 100644
--- a/workflow/scripts/functions.py
+++ b/workflow/scripts/functions.py
@@ -172,7 +172,8 @@ def get_final_outputs():
caller = CALLER,
assigner = ASSIGNER))
# logs
- final_outputs.append("results/10_Reports/setup.log")
+ #final_outputs.append("results/10_Reports/setup.log")
+ final_outputs.append("results/10_Reports/settings.log")
final_outputs.append("results/10_Reports/config.log")
final_outputs.append("results/10_Reports/envs/")
# return final_outpus
diff --git a/workflow/scripts/settings.py b/workflow/scripts/settings.py
new file mode 100644
index 0000000..5d9a94b
--- /dev/null
+++ b/workflow/scripts/settings.py
@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import subprocess
+import platform
+import re
+import glob
+import time
+import yaml
+
+# Get working directory
+workdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+
+# Get version
+version_file = os.path.join(workdir, "VERSION.txt")
+try:
+ with open(version_file, "r") as vf:
+ version = vf.read().strip()
+except Exception:
+ version = "N/A"
+
+# Define colors
+blue = "\033[1;34m"
+green = "\033[1;32m"
+red = "\033[1;31m"
+ylo = "\033[1;33m"
+nc = "\033[0m"
+
+# Get system information
+system_platform = platform.system().lower()
+if "darwin" in system_platform:
+ os_type = "osx"
+elif "linux" in system_platform:
+ os_type = "linux"
+elif "bsd" in system_platform:
+ os_type = "bsd"
+elif "sunos" in system_platform:
+ os_type = "solaris"
+elif "windows" in system_platform:
+ os_type = "windows"
+else:
+ os_type = "unknown (" + system_platform + ")"
+
+# Get hardware information
+if os_type == "osx":
+ try:
+ model_name = subprocess.check_output(["sysctl", "-n", "machdep.cpu.brand_string"]).decode().strip()
+ physical_cpu = subprocess.check_output(["sysctl", "-n", "hw.physicalcpu"]).decode().strip()
+ logical_cpu = subprocess.check_output(["sysctl", "-n", "hw.logicalcpu"]).decode().strip()
+ mem_size = subprocess.check_output(["sysctl", "-n", "hw.memsize"]).decode().strip() # en octets
+ ram_gb = int(mem_size) // (1024 ** 3)
+ except Exception:
+ model_name = physical_cpu = logical_cpu = ram_gb = "N/A"
+elif os_type in ["linux", "bsd", "solaris"]:
+ try:
+ lscpu_out = subprocess.check_output(["lscpu"]).decode()
+ model_name = ""
+ physical_cpu = ""
+ threads_cpu = ""
+ for line in lscpu_out.splitlines():
+ if "Model name:" in line:
+ model_name = line.split(":", 1)[1].strip()
+ if line.startswith("CPU(s):"):
+ physical_cpu = line.split(":", 1)[1].strip()
+ if "Thread(s) per core:" in line:
+ threads_cpu = line.split(":", 1)[1].strip()
+ if physical_cpu and threads_cpu:
+ logical_cpu = int(physical_cpu) * int(threads_cpu)
+ else:
+ logical_cpu = "N/A"
+ except Exception:
+ model_name = physical_cpu = logical_cpu = "N/A"
+ try:
+ with open("/proc/meminfo", "r") as meminfo:
+ for line in meminfo:
+ if line.startswith("MemTotal:"):
+ mem_kb = int(re.findall(r'\d+', line)[0])
+ ram_gb = mem_kb // (1024 ** 2)
+ break
+ except Exception:
+ ram_gb = "N/A"
+else:
+ print(f"\nPlease, use a UNIX-like operating system (linux, osx, WSL).")
+ sys.exit(0)
+
+# Get network status
+def is_online():
+ for host in ["google.com", "cloudflare.com"]:
+ try:
+ subprocess.check_call(
+ ["ping", "-c", "1", "-W", "5", host],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ return True
+ except subprocess.CalledProcessError:
+ continue
+ return False
+
+network = "Online" if is_online() else "Offline"
+
+# Get version
+def get_version(cmd, fallback="N/A"):
+ try:
+ return subprocess.check_output(cmd).decode().strip()
+ except Exception:
+ return fallback
+
+conda_version = get_version(["conda", "--version"])
+mamba_version = get_version(["mamba", "--version"]).splitlines()[0]
+snakemake_version = get_version(["snakemake", "--version"])
+
+# Get configuration
+config_file = os.path.join(workdir, "config", "config.yaml")
+with open(config_file, "r") as cf:
+ config_data = yaml.safe_load(cf)
+
+# Get fastq directory
+fastq_dir = config_data.get("fastq_dir", "")
+fastq_files = len(glob.glob(os.path.join(fastq_dir, "*.fastq.gz")))
+fastq_R1 = len(glob.glob(os.path.join(fastq_dir, "*R1*.fastq.gz")))
+fastq_R2 = len(glob.glob(os.path.join(fastq_dir, "*R2*.fastq.gz")))
+
+# Get resources
+resources = config_data.get("resources", {})
+max_threads = resources.get("cpus", "N/A")
+max_memory = resources.get("ram", "N/A")
+
+# Get modules
+modules = config_data.get("modules", {})
+qualities = modules.get("qualities", "N/A")
+keeptrim = modules.get("keeptrim", "N/A")
+cleapping = modules.get("cleapping", "N/A")
+covstats = modules.get("covstats", "N/A")
+consensus = modules.get("consensus", "N/A")
+lineages = modules.get("lineages", "N/A")
+gisaid = modules.get("gisaid", "N/A")
+
+# Get consensus parameters
+consensus_params = config_data.get("consensus", {})
+reference_conf = consensus_params.get("reference", "N/A")
+mapper = consensus_params.get("mapper", "N/A")
+min_cov = consensus_params.get("min_cov", "N/A")
+min_freq = consensus_params.get("min_freq", "N/A")
+assigner = consensus_params.get("assigner", "N/A")
+
+# Get other parameters
+nextclade_dataset = config_data.get("nextclade", {}).get("dataset", "N/A")
+fastqscreen_subset = config_data.get("fastq_screen", {}).get("subset", "N/A")
+cutadapt_clipping = config_data.get("cutadapt", {}).get("clipping", "N/A")
+
+# Get time stamp
+time_stamp_start = time.strftime("%Y-%m-%d %H:%M", time.localtime())
+
+# Create message
+message_settings = f"""{blue}------------------------------------------------------------------------{nc}
+{blue}#####{nc} {red}Configuration{nc} {blue}#####{nc}
+{blue}-------------------------{nc}
+
+{green}Starting time{nc} ______________ {time_stamp_start}
+
+{green}Conda version{nc} ______________ {ylo}{conda_version}{nc}
+{green}Mamba version{nc} ______________ {ylo}{mamba_version}{nc}
+{green}Snakemake version{nc} __________ {ylo}{snakemake_version}{nc}
+
+{green}Max threads{nc} ________________ {red}{max_threads}{nc} of {ylo}{logical_cpu}{nc} threads available
+{green}Max memory{nc} _________________ {red}{max_memory}{nc} of {ylo}{ram_gb}{nc} Gb available
+{green}Jobs memory{nc} ________________ {red}N/A{nc} Gb per job
+
+{green}Network{nc} ____________________ {red}{network}{nc}
+
+{green}Working directory{nc} _________ '{ylo}{workdir}{nc}'
+
+{green}Fastq directory{nc} ___________ '{ylo}{fastq_dir}{nc}'
+{green} > Fastq processed{nc} ________ {red}{fastq_files}{nc} fastq files
+{green} > Forward reads{nc} __________ {red}{fastq_R1}{nc} R1
+{green} > Reverse reads{nc} __________ {red}{fastq_R2}{nc} R2
+
+{blue}Modules:{nc}
+{green} > Quality Control{nc} ________ {red}{qualities}{nc}
+{green} > Keep Trim{nc} ______________ {red}{keeptrim}{nc}
+{green} > Soft Clipping{nc} __________ {red}{cleapping}{nc}
+{green} > Cov Stats{nc} ______________ {red}{covstats}{nc}
+{green} > Consensus{nc} ______________ {red}{consensus}{nc}
+{green} > Lineages{nc} _______________ {red}{lineages}{nc}
+{green} > Gisaid{nc} _________________ {red}{gisaid}{nc}
+
+{blue}Params:{nc}
+{green} > Reference genome{nc} _______ {ylo}{reference_conf}{nc}
+{green} > Mapper{nc} ________________ {ylo}{mapper}{nc}
+{green} > Min coverage{nc} ___________ {red}{min_cov}{nc} X
+{green} > Min allele frequency{nc} ___ {red}{min_freq}{nc}
+{green} > Assigner{nc} ______________ {red}{assigner}{nc}
+{green} - Nextclade dataset{nc} _____ {red}{nextclade_dataset}{nc}
+{green} > Fastq-Screen subset{nc} ____ {red}{fastqscreen_subset}{nc}
+{green} > Cutadapt clipping{nc} ______ {red}{cutadapt_clipping} nt{nc}
+"""
+
+# Print message
+print(message_settings)
+
+# Clean message
+ansi_escape = re.compile(r'\x1B\[[0-9;]*[mK]')
+message_clean = ansi_escape.sub('', message_settings)
+
+# Create log file
+results_dir = os.path.join(workdir, "results", "10_Reports")
+os.makedirs(results_dir, exist_ok=True)
+
+# Write log file
+log_file = os.path.join(results_dir, "settings.log")
+with open(log_file, "w") as logf:
+ logf.write(message_clean)
\ No newline at end of file
--
GitLab