From 2d66b735241521cb35817190994a81bc78621ae4 Mon Sep 17 00:00:00 2001
From: thdurand4 <91953714+thdurand4@users.noreply.github.com>
Date: Tue, 7 Jun 2022 13:40:30 +0200
Subject: [PATCH] Add files via upload
---
effector/CookieCutter.py | 31 ++
.../__pycache__/CookieCutter.cpython-37.pyc | Bin 0 -> 1077 bytes
.../__pycache__/slurm_utils.cpython-37.pyc | Bin 0 -> 10460 bytes
effector/cluster_config_SLURM.yaml | 95 +++++
effector/config.yaml | 27 ++
effector/settings.json | 6 +
effector/slurm-jobscript.sh | 3 +
effector/slurm-status.py | 72 ++++
effector/slurm-submit.py | 61 ++++
effector/slurm_utils.py | 345 ++++++++++++++++++
10 files changed, 640 insertions(+)
create mode 100644 effector/CookieCutter.py
create mode 100644 effector/__pycache__/CookieCutter.cpython-37.pyc
create mode 100644 effector/__pycache__/slurm_utils.cpython-37.pyc
create mode 100644 effector/cluster_config_SLURM.yaml
create mode 100644 effector/config.yaml
create mode 100644 effector/settings.json
create mode 100644 effector/slurm-jobscript.sh
create mode 100644 effector/slurm-status.py
create mode 100644 effector/slurm-submit.py
create mode 100644 effector/slurm_utils.py
diff --git a/effector/CookieCutter.py b/effector/CookieCutter.py
new file mode 100644
index 0000000..19d61df
--- /dev/null
+++ b/effector/CookieCutter.py
@@ -0,0 +1,31 @@
+#
+# Based on lsf CookieCutter.py
+#
+import os
+import json
+
+d = os.path.dirname(__file__)
+with open(os.path.join(d, "settings.json")) as fh:
+ settings = json.load(fh)
+
+
+class CookieCutter:
+
+ SBATCH_DEFAULTS = settings['SBATCH_DEFAULTS']
+ CLUSTER_NAME = settings['CLUSTER_NAME']
+ CLUSTER_CONFIG = settings['CLUSTER_CONFIG']
+ ADVANCED_ARGUMENT_CONVERSION = settings['ADVANCED_ARGUMENT_CONVERSION']
+
+ @staticmethod
+ def get_cluster_option() -> str:
+ cluster = CookieCutter.CLUSTER_NAME
+ if cluster != "":
+ return f"--cluster={cluster}"
+ return ""
+
+ @staticmethod
+ def get_advanced_argument_conversion() -> bool:
+ val = {"yes": True, "no": False}[
+ CookieCutter.ADVANCED_ARGUMENT_CONVERSION
+ ]
+ return val
diff --git a/effector/__pycache__/CookieCutter.cpython-37.pyc b/effector/__pycache__/CookieCutter.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..df18a126968440b7a86cf5b8997acc5f4d3c19fd
GIT binary patch
literal 1077
zcmaJ=OK%e~5VpOWWb>jGRDz;PaLu7zaR<~iO&?Obg(NL#FV;48l1(@3U^|r>l~XIO
z`~Zs79{EeYa^f#=V!Uafs0gn-GoFY2eID;dwOT?j-s4Zlj~#@*+r@ko!B~f>jzBQP
zaDu{_#v$ftNUk03-rz9D^VzDvbBx@O3;cO8^f<mQau1@VcTo8jwa|OCjhMsSOT^p@
z5|$Zmqgw6<?2UqHRdKE3w6E5NN~B%uoDUa_b(rc62#!L`P)NAL@G=Tr&;)c2v;#U1
z+67(U9xt-o6=r!>xO6W726)Pu$799JQ(+|lP&So@5GOHjOtj|GEVMW3oyHE`Y;M&L
z_d0C|?;W-~%@z&n`%Uv;zG@r<Tf5t4rM`Jo4;sx)T5oM1?l*%Dg~(B})!sb_YS`o@
z*Aw}N5C#qeX2TL%0in?le?;eSM<<Sa!pJ%Lg1(YAtO1-87&I-du6C1&0&uTtMA~9$
zuLL$P-1@&EX}kDy$5(?$a^}aqW2J?B;Sa=!`<hK;lrn#<E7D%v_f;C5@DWU(_j<ew
ziT>RySR0?2hkdT;?e0{Jbu7|HaDwtcAd<%(F5&63zpCz6{O+P9?ZC`PFM+`@Bh0xX
zmi#iZ^nt7!bhc_Fbk4XkBo%Y6dB_K@&S{inEN7w&Uh~|Z8L`tS?Q%vV*`JJf3b;jj
z%B2F#Pa#X$4G{6LN2aUy5dW__s8x+eX&Q|<rKUvbNU%v_?FywICQ))5@#b$=T9R3F
zWX@XlWK&T((s6gh^*}J=C@p2h`rTt861iaWZ1e62%Ps4(%~OjYJPY4`gy6+I!CACc
zF+?ci+8@Ta<46xofyL5-vaow`1{E{oiZO?bp@`Fv^ajRFM8u@+=j=88<9Oq}oRNKP
PpPI56_Vfx~!b{FC(4GvB
literal 0
HcmV?d00001
diff --git a/effector/__pycache__/slurm_utils.cpython-37.pyc b/effector/__pycache__/slurm_utils.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..38a06d13029b3157be5a5898bc352b16f1119838
GIT binary patch
literal 10460
zcmb7K+ix7#d7m?vy>YoBDT)#$+8)cY#kIwytp-7$D2geWauiFnDpIktk+T`@8InWp
z?ySztO5|kLNFv3S+bVIIUSYc>Qy@SJ)DJ};`p|!%Kv5J$pXRZB@<Shrz6pKk@B3zU
zDba4ykeD;)HfPR!*WdS@y*N5r((w1TcyIOJ&uH3zP-E~{K;>0j!FPqGNll8DR!{Gu
zF3{FndfTWQBJDHlCeN|z7T<IA9N+Wx0`5ku*e=yeJlAZEw2Sp3*R58$Jz5{-dahMz
zAF3bXdLH$$`WV*>sE^mjxn68dv=7%0bG_7>Y#*r~;rd8xsy$VoLcJ_U?`!p=vLX-R
zeoT(ZaonfngglJ<arufoBB$<)`U!bd9z)NJoR-INe@dQ}C*%x9o|aF^r_u9_d`8;1
z+wxg?68C52bMh4KCtY**Ir;pa;QI^N{d9Jp&F<Cg{^Fik&(%-Km*g2d`@Ec!HQZm2
zza!`6%Xr&qDVDYBmp&t^FI7d7m!9f4Z5P$Ta;Ut{`jt0P*F7I)@hz_<n~stJ8sp3E
z?Vj6dx|e;`cEZqA=sdE}y6fx&zuWUeS3W*)=<{8pZ~1C#t>xccRLZA^b1i>;otG(J
z^8GE(z0{*+o8UkHStv;=F5wFP1VyODS}5-6!6eu8d%BtlMXcS!H%TFNX-HFAADev<
z8oQ><?OHOwn~TM6eoxyI^xxNGeYX&!)i+{87PbmXg~i>HEXLYLnk;={><OWTBfDj(
zQ!N&;u{$b9_QbT-H$#E9Q5$3TL~Ks-o2%vM?3FdU?Y4cjV|#%ebls-6=DE_o>xCQk
z@)f(+@xn89)9>7IRcMF4eeL4<|NQE|{`K1G^{;<MpSz9=;{VfEx$=YiwS}sgh&K{(
zEfKFLVkIig-jXj>XTRUCo_ZwWrRZd@6ZE=WUxDs}S7+~SI-%!x?9<URr@w03P74&r
zoS?V1=54#dIr}?r#Mk22<CS+l)A7z~B`JA<*9k%gyiE+R6DAh8<b;W?+$7)h+g-2a
zCb>0cdtz?-UMDfP+?^n?AP3D2MLL~WE^#$60`!~iwue>LmEY@Dtt6L{rk)QxA_SgO
z?z+33h;}`<<}^cJ{Yq2MVUb6#&j%Zha^<|Ywi=-SmH7?7?aqg?r<{(QuYrPV-uisd
zakktxu6cKD&7}qAgH}(q8@<qL1-0%@GTF$8-QYJiLch^oox&>uDl~mk6huWF7bV<H
zy<(QoQqN<gNyGF<TrqL6@SQ>>)HVeP*+bEX7;a|5_E3kAXuHOqwrN7Ka?yorUdL-Y
zE&I~dE5vC9t?`sQ?{``|w$p67-OvWP*ITaL^#hMsg~^Uu@3q}dSc{%ozIyY<HGBAp
zVBBik!!u3gbwg;)h;-9Bo=l3H{%YEaW|s$4v4@?h0HSGkmEVQzd2Ud{8s>)EBF+@>
z-maqpH>oUSJGf3wm6RPR8-raXC7wj>K%E9Hu!QI`%hH1AlN?GX4PRbGXOKgo>185s
z4Yi?eQ~!{>9W+Rz??Jz|bTtWr>U&y8kF~z=w7$N5l?Z<y+V~pgyr#W#j<wgAX2S16
z>!HtPXvW4pAuX&>hi>Q6(T*<jeJi$NJvPs4x6iz##X1;Oh>qUube-mwy?k+D<<d7A
zuPt6)xOsJDxi;J)X$P60Wn3&a>)7HumA-&MffL#jKhtj$&&4D0H4wxKx-BoP>WN-^
zIT6kQ>O3l)Z+SsT+H~Pdv>>gX)1qohv|yDqsUh7pr`HOD+WD5>bXvg$R0dBelE_D5
zPEFvh$|#n=K{MRxLTiIish81(UBHNU6QkvJs<|Y`tII^l#3bg`bI{v6PAd^xiMVrM
zi^-8M+zTxmR8VO8F=2{HF(GC|S*Q!>d4g8NQr4D>sKlCjku+vk4~@_SW2tVD;oZ~w
zTA16-vk?{@6AXr4#bPV3E_h(E6z5Tbu6m@d`i>;C38g1pl3p_`m2CVyHUQJW_AVQ@
zUD#Y#*=x#g+s<m>w|XH|&4D5I6|}L1*>_xC1W!OM)?nSRvZ`@kGSV1uq0y)o65WS6
zqo>t0zDg0-l0u_F4>cM|$=&Y4Fx`R9D3a#n5Dcdz+k+j`oT{!C&|f!Xvw4u*hts*O
zUI3?;J@ZvePKli}VV=sm2?JFSY7zaHoAh@2BdH{#`v$IH4MoWA50=S@g*pZosl`S^
z?-W8Kt?g7|W7CYaH4*DS0c}k!w6;p>djd{PXwhVtlSXX90h*scg?=R7(xfG=zWJ{C
zw)Sl(;@fDmb^A2+@wP)0#kM=P*Rl5vPY~|3+?0?j8}Ni}*XaayxZ#901n;g7Bw@Sl
zRaZ)QBs{sQoTeMV`nljtw}pnZS79bQJ2q(QIjgOdQxMr6=J7%_*}3C*EgGdoz;3(Q
zQwrwOCMXj?t+P!$5;iom7`TvZ2W}!Nu3$1vvZG&;tL}BSI{sb$ijX!8&V}xFShKIU
zTqkgCax$>McqPAk$M4zDQLlr2a>rnoHG82I_;v>{C>_YY3>|haz-)NW;U}>h2m*V<
zxr5ohz3SR9=}tHOco4#cjCLUL8*bZPs^%4Gmdc|@#@&`nexi|jV{kwbLQ=<Qu<(Yv
z!`Lb*F5weXm<1!>$as~M8n+3kHGm0{+}2&P2#K-QhNpM84@f7kconM#WNtL81l2F;
z1);-|kD=u8p+M3GNg!GMCt#>vXMwyA)E;X03_@Ko5L4fPn2D{NItv^nB<!~|9|BQ9
zRCCgzy98Rp_zfP11<%HfY@AF9L^d{eEwc8TIxa)zA2L=8bA;%C2|pZ;=VkFBP@m9t
z${|o6#NW)szX<XFH;BIl@jo7xWC^Iy*c=JVv2_n7>M<p--nCc>&OvELw@M!hRetQb
zk>B{-KzYhh%^qqGS!r;X@!>#*g4AY_IKzk5>?RR@@JFiuDz0D>g{x6SRM%6l2;*Wv
zInC&)R45vE2dI7DjeOHz9Wu+n3dAt>YbR0Jo*~Ej&SW3Vis2psdi80rO=!1sumqS`
z6&x!BZV0VcRU<J2x3!j3u+PRmvPcXl`2jg1V+lYM!eru20U1dRs4IHm*Y348cxVYQ
z7%Py*lDTa(+P>_u=`A-P%uXvEQ7c&Xu{S?Q%~L2617Gf<!u}S9^lj=Sx9GGfs40+}
zAh)uV6o+Jk(f&3q@Ew|KD!i{M)GaikXNG!}vW^iham)UG{~WH!zS5EIHf!D&&j8%c
z`Ky}%t%|I8y^?7%bd<J~j1Stw-6SUMFUj?~(g^{`H=KZ&z#C#MRNth{7HP8<w&F?k
zWvZ9<w?arDl{j^S<}wis%mqb<(x={}A*S3u`Nj^iX%&-#7g1=&m_@E>LeFJo4kltk
zF99Fuzz1X`PU%!v6w?f)kIdKzu1mOrKSTkrr|l8QyAP$;3HUL@3vcOt1FCLFf`n%Y
zMZ~}nFd}AX#U?zPuBLV-p!6n83mF*Wy!MuMdkSirhjKs2lypW?z{kc0ATbvf;~Z36
zXCDXI?q}+Kd+KdGn}I`fDKQYUM6X<in;}#0ll?}pV>eT8J-{Te#0dE?mdQYkxl%19
zX46L$ViI^uEH8B10W*a|40RY^%|uB#N4A^USIyT;#A^x()DE*MDW)$*l&45v6N_D8
zB7phY!TLdVjb`jh{pEXjAfVMXaA*ow8JeeqKVxjnZK1jt8*+wplk9?lE08D%dV_Kr
zg5)tmX~ZcBK)$DcAhv5j!D3H?-8gwm|3Cxs0Po({-WO{+1>*N%P6@1&@xA>5tO(Et
z4Ey(D)TUu$qy>~mKs*P{Q*Pk*6p+9Zc#`%*qzazqy#RYp5%zvS4#IqZ+O+mS`#EFq
zeL6sQU%~W5c&MF6P2HwqDAB;-L}JF_A7f;I!(SZ4i_nV4OxnH$-G3~}0}Z;445HCy
zw-?MIu9^#-U<+o^Sa(CUMKAmwimJ&xtd9-8uhDk8UD*93?{|sIf%+QFrOiFDTee(N
zU%VsAgQPwgOhoVsrcQ{b;h6!+?_%t8BxFT9i;*<KA`-GnO-hhI6r1`keDa<MYN$h%
zD2GBpEsa*KXnqlS2EZS`7j}CgM7<>mTtniQDdM(~!m$bH0@rOr*lUTlv_f{3TMd7!
zYB6S3uVMoXg_9f)1xbOh6>xWePiKs*qrrR{G7E5r-`=2U4Sw#guuee1kA_c9KOdoG
zjCPYc>5uFx$uXnjZ=kXc+1<X7+8M~}Sy&d>770rN>!NQJ)fvDN<0I`89X*h0$g2L4
zMtC?)2i>l~E>J{bMMo~Db|8({>;W6M@s=euk22Ed=I}uz14$0tb~_SLS|y9`2J`P=
zIF(KkfW&~l9{ltpnJPc|<lkU=7L38zvILaGF`<5d{vqed#16xQU!W0cYxsblAp5m*
zN{V&F((B?V)DM?NwZ0J|P>YfK>YHzAomrSrv1!1Fek(Jgh1g<)38XPFm=qKMXPGeF
z=FU`T?dCu>b5GyRON$IG)_DQz6tK$2zhRXenSto!a+>IX0;U-agaHy^LD)l-!G5h)
z&!yCeN^^4*t5O886j_`a<A}IgWJ)A@&`rwC4Y#?~NCh{UxCGGa%InI9nF&&qoRe<T
zhd57xL~xL<qmaych8o4iZA#QYMS;Fgqes0<1+gtLnr)dFP-#VJ3q=m`An!_P&ZdD~
z|2>9N8w}5=!wOfxwkZbEQ?SERLVbW4L)MXG4^y8103c0D3yB2F2$Zt&>RYCoiGj*^
zgpwb6_e`h*7$aeVxfb<+rRjJsdlL2+V|gC4LW5fcS>#o7@U38foC8!xT#k7KcvvgW
z%Mx&)n1sdeBbJN{*g+vKz~|=W2#sJT{~YFb3$h#+5Iv6WkDyig1cHEM>ToumePY#q
zF)qZ#EfbLX_+zUaii>!9EPGm`^eo;v+|3E(5C{ceEn0az-3QiK<uxWajy=3ThP5WM
zwX87R4L?H(+<VY?&<WoaoASuVdcPD~Sn1Pngu;Nn#j8xw8~W&pM|iKF@_UZPBUt%Z
zY9Zp0xFqxNXxeT$F40#Lf2z~b(&9GUgGV77W%=YLf_-Y({+gZH5JX{|`a^i@Q^;qp
zBR|ohWJQ(@La2?z9SLJCu{sdgfMq=?4*Qpq5}XAPo`dY|0k4D%5niu0NK7G5T2i^T
zc<s#_-)>xAywSLH{brI&+tCDNDqZ9vhWX^#Xsl{qKn%P+$S}`FGu5wVpF#0Umh(nj
zKmlujQOxg<SG2e7C*NbwM$h9};3awxpPCZG?(~okP(%-Ytk+FU>}om6r_0PnleFf3
zz|XU0Bm0Sc99Tpmq=ylvZ$3zk@s%sr7O!5pc4eh;?aI>4mBr;~h6rTOs$PdG^fvBj
zd6agIjy(RoJix1uDhk9@k0?6TfZHEtjfPxdu$LU?!3;NMS)MFq1qg>!x&iv8DRk58
ztoexvoVfu2>3Xf?5WSF-*Iox{tRRII<vQhU;S>=>{2uM%_o?^;6m^TKmgKXKQ8qf0
z9E1b;D5gh!bm3bISFSExyt>!`gX*Ix8(dEI2K5rZkU#yPQJZm)=fV-eep)!ou?LXM
zf5dx&*H9oVV7ENMApnH|W%#E8?xvVA%H|l%Ad-mYarRy_1`MJ>0R%x)Q&jYVQ4)%P
zS@i06XTN^^yu|6!1v>heqt*ssV%j`MO^}bM!sFZ#z@T?z+HoqH|Ls;d6+QiHA2=1A
z<?-Kc6VLH8zhS2npPO`Q=n@8S#vl^^3yKi1mrubaLh%5m=>y8Or2jgfeIe!gK5`v|
z<n9-cF(JpH!+_$>$nP>G^RjksOAqQu|K%y;0ay-Dy_<iKo{tI2ZR{55><nhS1Z-F!
zQz(m*I4PS<d)N(7>uB08(+t#cg2qpb@)IR8+Q1nw+Sq{^{1e_-;WttO{ZRCumj*{;
zq>5Q`1U4cgTP;ocE;O6SAZF%+4}8w01GbGJ_-oDxvW}+UDL{G<nkQZRWVX`DGsEh+
zOju5y;l!_PpJc|K%VzEm+b2l~=XjK!-){}(p3GJt!14vd!=<wCU^|S&hDj?W7B8IS
zv-c+XY+pkGAgfKx=U9jm@$fPbfjx@MiXb25X4DDv<MwQNdx%@AzZ^;w&bOTQs&p<K
z#ajQ3F)e!bfMt0s(8=a5v6qQXv~eKeb&+a)g0Shy-zQk=Gm0=j<524_P*Z<N#V(3y
zz&<28*|{OlxsJdINlVI+cI=m{qiHPsWqM>21x^{g7RHkidRfLq^%WYaECe{0A}`3`
zjX#*pVN&EQ3|m9C$$v_#{TT|x9!!XH)IpR{l<i74_#BOkbew3U{u~8<Fpz<@5*-Xy
z0ME)~tx&62m-R8hvz`bvNINg#3NE4u0iA${p^tQ$jBrAP99<Dg1}26M0wvQ4G|r<#
zYqPLt&^gv_Q8Mz5Nf*rMg&T+=Jr}9r&-s6jyE0c29mXl*ns3j&xsMO*Xp+q0&;wn#
zwfs``){E>PswFjoEhZD>s|M~UjYrfEF`#}#1s(P<9#%i5+I=cW<QNtskjZCHs`sh$
z0ToPB_CQ3_eW_EF+&N7Y9fWIi$T|r-&m&C(!}LeHqG*NM0YToKGQr&zq7uXqXDAjy
z4Dl6#6Ioaxq!vOPWu!4GPWf?6LDnffq`=|FRX{eJ_sblC#19aEkp1eL60jC<4FD8;
zF^T?tYsU^tn<KIS$c8m!5v@^KdT8ucFiP<Yz55W}{dYp^=aDC_gnA4+DYPFU(S#(2
z-p{|A9~_W`W4q&WM3(tJDl2i0ZRrF!Z-CE~T|OL664a#MiagZ&#husUq8x+0JrYjs
z3Dm~{Ad7MW>5-y5JguFe`y}@rnbtOsreB5G_%@78QM#*BOZN!STL1s-G8UNm;gd9?
zwP+XYGw4?;1;}bCE^Z!+0ss5Oa5_9rz9T$=nb;kYa0sAzJl(CnWzoEtjL~%T-cWus
zS;>-R8R*Gu;IE&=f=*{l8SteP1Iz*+kg%=Yhr7X<0SYDSOXx8532bDSqKj-T&J7NT
z=nOuCF~c)=+8=(GK`;g?M-h8q)9)wfWX}ms^LaU?IM^OO5@({xO!HiO&bOaOTxrjB
zZFL#)yMp6iE5PA8&R2p^BKC~R-@SD8=JLwojdQh^N@^REBE!BFt8vtjC^2w0RWA-f
zsrnhZ)laGTkcyw6s8<-WH7+kMtlYe@xLhB}YK`T^l_$J}iP`KWCQwP^Kk^go_Q|;D
zOR4M8Q<GXow1DqME*hVXy@+iTm3`7Sfa@RA=RGWX6Z)xsfl74d|HieS_a)UC%DEFc
zlaVWJ`qfGrfm>Z1gM{iSEXV|Nl~R=LBeI04F(UJT--n?1Aq`m9@jHeNAI2u;^()sG
zlM&?d+W3_Re)i%jN_nR7ynz^;?N_}($+|`}xTDC#Bo?wVtsS=HX`<Y~S-bipdhmHF
z{uo7dgl#6RJqW9U6ie$}ss5U#kpV<1pVMNT2_SbKI*BgVl%fe_4dG$_mg?+bevaBJ
zpkMGRDq5bB<Q1d@beyY=>A=45IFo4Ez{*s!fsiR#J1L$<Nj*8tnS%YCLR-;lRdJcT
z|A4KtX_k$sS<n0U9Sj{?>u9AGo&91J&4lt3IJTioNNW2@a8l|Nhd(??%EM~S^ZB$Q
zIkex!KegFEndD>0WO9Gp9b`~~^k`yye~5ne1r*0Y4?Crl@HN`R1u7P)AXt%R$-7je
z+$QJwIK#tP6OI$qQ5yIq739L%w=i*1$t5oA%WzUSpB^AxAhHDCKtWl=GOqN$jGwAJ
m_OF+X!r?-`Fj5#VO%+ZR#!91w6156n89aff&B9ZK(*FWB{keAl
literal 0
HcmV?d00001
diff --git a/effector/cluster_config_SLURM.yaml b/effector/cluster_config_SLURM.yaml
new file mode 100644
index 0000000..1f045d5
--- /dev/null
+++ b/effector/cluster_config_SLURM.yaml
@@ -0,0 +1,95 @@
+__default__:
+ cpus-per-task : 4
+ mem-per-cpu : 10G
+ partition : long
+ output : 'slurm_logs/stdout/{rule}/{wildcards}.o'
+ error : 'slurm_logs/error/{rule}/{wildcards}.e'
+ job-name : '{rule}.{wildcards}'
+
+
+rename_protein:
+ cpus-per-task: 1
+ partition: fast
+
+phobius:
+ cpus-per-task: 5
+ mem-per-cpu: 10G
+ partition: fast
+
+signalP:
+ cpus-per-task: 10
+ mem-per-cpu: 10G
+ partition: fast
+
+targetp:
+ cpus-per-task: 10
+ mem-per-cpu: 10G
+ partition: fast
+
+predgpi:
+ cpus-per-task: 10
+ mem-per-cpu: 10G
+ partition: fast
+
+parse_phobius:
+ cpus-per-task: 1
+ partition: fast
+
+parse_signalp:
+ cpus-per-task: 1
+ partition: fast
+
+parse_targetp:
+ cpus-per-task: 1
+ partition: fast
+
+parse_predgpi:
+ cpus-per-task: 1
+ partition: fast
+
+intersect_tools:
+ cpus-per-task: 1
+ partition: fast
+
+fasta_intersect:
+ cpus-per-task: 1
+ partition: fast
+
+tmhmm:
+ cpus-per-task: 5
+ partition: fast
+
+parse_tmhmm:
+ cpus-per-task: 1
+ partition: fast
+
+tmhmm_fasta:
+ cpus-per-task: 1
+ partition: fast
+
+wolfpsort:
+ cpus-per-task: 10
+ mem-per-cpu: 10G
+ partition: fast
+
+parse_wolfpsort:
+ cpus-per-task: 1
+ partition: fast
+
+id_tofasta_secreted:
+ cpus-per-task: 1
+ partition: fast
+
+hmmer_pfam:
+ cpus-per-task: 8
+ mem-per-cpu: 5G
+ partition: fast
+
+effectorP:
+ cpus-per-task: 10
+ mem-per-cpu: 10G
+ partition: fast
+
+count_effector:
+ cpus-per-task: 1
+ partition: fast
diff --git a/effector/config.yaml b/effector/config.yaml
new file mode 100644
index 0000000..7ea6574
--- /dev/null
+++ b/effector/config.yaml
@@ -0,0 +1,27 @@
+restart-times: 0
+jobscript: "slurm-jobscript.sh"
+cluster: "slurm-submit.py"
+cluster-status: "slurm-status.py"
+max-jobs-per-second: 1
+max-status-checks-per-second: 10
+local-cores: 1
+jobs: 200
+latency-wait: 60000000
+use-envmodules: true
+use-singularity: false
+rerun-incomplete: false
+printshellcmds: true
+
+# Example resource configuration
+# default-resources:
+# - runtime=100
+# - mem_mb=6000
+# - disk_mb=1000000
+# # set-threads: map rule names to threads
+# set-threads:
+# - single_core_rule=1
+# - multi_core_rule=10
+# # set-resources: map rule names to resources in general
+# set-resources:
+# - high_memory_rule:mem_mb=12000
+# - long_running_rule:runtime=1200
diff --git a/effector/settings.json b/effector/settings.json
new file mode 100644
index 0000000..daf6024
--- /dev/null
+++ b/effector/settings.json
@@ -0,0 +1,6 @@
+{
+ "SBATCH_DEFAULTS": "--export=ALL",
+ "CLUSTER_NAME": "",
+ "CLUSTER_CONFIG": "/shared/home/tdurand/effector/cluster_config_SLURM.yaml",
+ "ADVANCED_ARGUMENT_CONVERSION": "no"
+}
diff --git a/effector/slurm-jobscript.sh b/effector/slurm-jobscript.sh
new file mode 100644
index 0000000..391741e
--- /dev/null
+++ b/effector/slurm-jobscript.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+# properties = {properties}
+{exec_job}
diff --git a/effector/slurm-status.py b/effector/slurm-status.py
new file mode 100644
index 0000000..6dc2323
--- /dev/null
+++ b/effector/slurm-status.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import re
+import subprocess as sp
+import shlex
+import sys
+import time
+import logging
+from CookieCutter import CookieCutter
+
+logger = logging.getLogger("__name__")
+
+STATUS_ATTEMPTS = 20
+
+jobid = sys.argv[1]
+
+cluster = CookieCutter.get_cluster_option()
+
+for i in range(STATUS_ATTEMPTS):
+ try:
+ sacct_res = sp.check_output(shlex.split(f"sacct {cluster} -P -b -j {jobid} -n"))
+ res = {
+ x.split("|")[0]: x.split("|")[1]
+ for x in sacct_res.decode().strip().split("\n")
+ }
+ break
+ except sp.CalledProcessError as e:
+ logger.error("sacct process error")
+ logger.error(e)
+ except IndexError as e:
+ logger.error(e)
+ pass
+ # Try getting job with scontrol instead in case sacct is misconfigured
+ try:
+ sctrl_res = sp.check_output(
+ shlex.split(f"scontrol {cluster} -o show job {jobid}")
+ )
+ m = re.search(r"JobState=(\w+)", sctrl_res.decode())
+ res = {jobid: m.group(1)}
+ break
+ except sp.CalledProcessError as e:
+ logger.error("scontrol process error")
+ logger.error(e)
+ if i >= STATUS_ATTEMPTS - 1:
+ print("failed")
+ exit(0)
+ else:
+ time.sleep(1)
+
+status = res[jobid]
+
+if status == "BOOT_FAIL":
+ print("failed")
+elif status == "OUT_OF_MEMORY":
+ print("failed")
+elif status.startswith("CANCELLED"):
+ print("failed")
+elif status == "COMPLETED":
+ print("success")
+elif status == "DEADLINE":
+ print("failed")
+elif status == "FAILED":
+ print("failed")
+elif status == "NODE_FAIL":
+ print("failed")
+elif status == "PREEMPTED":
+ print("failed")
+elif status == "TIMEOUT":
+ print("failed")
+elif status == "SUSPENDED":
+ print("running")
+else:
+ print("running")
diff --git a/effector/slurm-submit.py b/effector/slurm-submit.py
new file mode 100644
index 0000000..a8a780b
--- /dev/null
+++ b/effector/slurm-submit.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Snakemake SLURM submit script.
+"""
+from snakemake.utils import read_job_properties
+
+import slurm_utils
+from CookieCutter import CookieCutter
+
+# cookiecutter arguments
+SBATCH_DEFAULTS = CookieCutter.SBATCH_DEFAULTS
+CLUSTER = CookieCutter.get_cluster_option()
+CLUSTER_CONFIG = CookieCutter.CLUSTER_CONFIG
+ADVANCED_ARGUMENT_CONVERSION = CookieCutter.get_advanced_argument_conversion()
+
+RESOURCE_MAPPING = {
+ "time": ("time", "runtime", "walltime"),
+ "mem": ("mem", "mem_mb", "ram", "memory"),
+ "mem-per-cpu": ("mem-per-cpu", "mem_per_cpu", "mem_per_thread"),
+ "nodes": ("nodes", "nnodes"),
+ "partition": ("partition", "queue"),
+}
+
+# parse job
+jobscript = slurm_utils.parse_jobscript()
+job_properties = read_job_properties(jobscript)
+
+sbatch_options = {}
+cluster_config = slurm_utils.load_cluster_config(CLUSTER_CONFIG)
+
+# 1) sbatch default arguments and cluster
+sbatch_options.update(slurm_utils.parse_sbatch_defaults(SBATCH_DEFAULTS))
+sbatch_options.update(slurm_utils.parse_sbatch_defaults(CLUSTER))
+
+# 2) cluster_config defaults
+sbatch_options.update(cluster_config["__default__"])
+
+# 3) Convert resources (no unit conversion!) and threads
+sbatch_options.update(
+ slurm_utils.convert_job_properties(job_properties, RESOURCE_MAPPING)
+)
+
+# 4) cluster_config for particular rule
+sbatch_options.update(cluster_config.get(job_properties.get("rule"), {}))
+
+# 5) cluster_config options
+sbatch_options.update(job_properties.get("cluster", {}))
+
+# 6) Advanced conversion of parameters
+if ADVANCED_ARGUMENT_CONVERSION:
+ sbatch_options = slurm_utils.advanced_argument_conversion(sbatch_options)
+
+# 7) Format pattern in snakemake style
+sbatch_options = slurm_utils.format_values(sbatch_options, job_properties)
+
+# ensure sbatch output dirs exist
+for o in ("output", "error"):
+ slurm_utils.ensure_dirs_exist(sbatch_options[o]) if o in sbatch_options else None
+
+# submit job and echo id back to Snakemake (must be the only stdout)
+print(slurm_utils.submit_job(jobscript, **sbatch_options))
diff --git a/effector/slurm_utils.py b/effector/slurm_utils.py
new file mode 100644
index 0000000..d43c070
--- /dev/null
+++ b/effector/slurm_utils.py
@@ -0,0 +1,345 @@
+#!/usr/bin/env python3
+import os
+import sys
+from os.path import dirname
+import re
+import math
+import argparse
+import subprocess as sp
+from io import StringIO
+
+from snakemake import io
+from snakemake.io import Wildcards
+from snakemake.utils import SequenceFormatter
+from snakemake.utils import AlwaysQuotedFormatter
+from snakemake.utils import QuotedFormatter
+from snakemake.exceptions import WorkflowError
+from snakemake.logging import logger
+
+from CookieCutter import CookieCutter
+
+
+def _convert_units_to_mb(memory):
+ """If memory is specified with SI unit, convert to MB"""
+ if isinstance(memory, int) or isinstance(memory, float):
+ return int(memory)
+ siunits = {"K": 1e-3, "M": 1, "G": 1e3, "T": 1e6}
+ regex = re.compile(r"(\d+)({})$".format("|".join(siunits.keys())))
+ m = regex.match(memory)
+ if m is None:
+ logger.error(
+ (
+ f"unsupported memory specification '{memory}';"
+ " allowed suffixes: [K|M|G|T]"
+ )
+ )
+ sys.exit(1)
+ factor = siunits[m.group(2)]
+ return int(int(m.group(1)) * factor)
+
+
+def parse_jobscript():
+ """Minimal CLI to require/only accept single positional argument."""
+ p = argparse.ArgumentParser(description="SLURM snakemake submit script")
+ p.add_argument("jobscript", help="Snakemake jobscript with job properties.")
+ return p.parse_args().jobscript
+
+
+def parse_sbatch_defaults(parsed):
+ """Unpack SBATCH_DEFAULTS."""
+ d = parsed.split() if type(parsed) == str else parsed
+ args = {}
+ for keyval in [a.split("=") for a in d]:
+ k = keyval[0].strip().strip("-")
+ v = keyval[1].strip() if len(keyval) == 2 else None
+ args[k] = v
+ return args
+
+
+def load_cluster_config(path):
+ """Load config to dict
+
+ Load configuration to dict either from absolute path or relative
+ to profile dir.
+ """
+ if path:
+ path = os.path.join(dirname(__file__), os.path.expandvars(path))
+ dcc = io.load_configfile(path)
+ else:
+ dcc = {}
+ if "__default__" not in dcc:
+ dcc["__default__"] = {}
+ return dcc
+
+
+# adapted from format function in snakemake.utils
+def format(_pattern, _quote_all=False, **kwargs): # noqa: A001
+ """Format a pattern in Snakemake style.
+ This means that keywords embedded in braces are replaced by any variable
+ values that are available in the current namespace.
+ """
+ fmt = SequenceFormatter(separator=" ")
+ if _quote_all:
+ fmt.element_formatter = AlwaysQuotedFormatter()
+ else:
+ fmt.element_formatter = QuotedFormatter()
+ try:
+ return fmt.format(_pattern, **kwargs)
+ except KeyError as ex:
+ raise NameError(
+ f"The name {ex} is unknown in this context. Please "
+ "make sure that you defined that variable. "
+ "Also note that braces not used for variable access "
+ "have to be escaped by repeating them "
+ )
+
+
+# adapted from Job.format_wildcards in snakemake.jobs
+def format_wildcards(string, job_properties):
+ """ Format a string with variables from the job. """
+
+ class Job(object):
+ def __init__(self, job_properties):
+ for key in job_properties:
+ setattr(self, key, job_properties[key])
+
+ job = Job(job_properties)
+ if "params" in job_properties:
+ job._format_params = Wildcards(fromdict=job_properties["params"])
+ else:
+ job._format_params = None
+ if "wildcards" in job_properties:
+ job._format_wildcards = Wildcards(fromdict=job_properties["wildcards"])
+ else:
+ job._format_wildcards = None
+ _variables = dict()
+ _variables.update(
+ dict(params=job._format_params, wildcards=job._format_wildcards)
+ )
+ if hasattr(job, "rule"):
+ _variables.update(dict(rule=job.rule))
+ try:
+ return format(string, **_variables)
+ except NameError as ex:
+ raise WorkflowError(
+ "NameError with group job {}: {}".format(job.jobid, str(ex))
+ )
+ except IndexError as ex:
+ raise WorkflowError(
+ "IndexError with group job {}: {}".format(job.jobid, str(ex))
+ )
+
+
+# adapted from ClusterExecutor.cluster_params function in snakemake.executor
+def format_values(dictionary, job_properties):
+ formatted = dictionary.copy()
+ for key, value in list(formatted.items()):
+ if key == "mem":
+ value = str(_convert_units_to_mb(value))
+ if isinstance(value, str):
+ try:
+ formatted[key] = format_wildcards(value, job_properties)
+ except NameError as e:
+ msg = "Failed to format cluster config " "entry for job {}.".format(
+ job_properties["rule"]
+ )
+ raise WorkflowError(msg, e)
+ return formatted
+
+
+def convert_job_properties(job_properties, resource_mapping=None):
+ options = {}
+ if resource_mapping is None:
+ resource_mapping = {}
+ resources = job_properties.get("resources", {})
+ for k, v in resource_mapping.items():
+ options.update({k: resources[i] for i in v if i in resources})
+
+ if "threads" in job_properties:
+ options["cpus-per-task"] = job_properties["threads"]
+ return options
+
+
+def ensure_dirs_exist(path):
+ """Ensure output folder for Slurm log files exist."""
+ di = dirname(path)
+ if di == "":
+ return
+ if not os.path.exists(di):
+ os.makedirs(di, exist_ok=True)
+ return
+
+
+def format_sbatch_options(**sbatch_options):
+ """Format sbatch options"""
+ options = []
+ for k, v in sbatch_options.items():
+ val = ""
+ if v is not None:
+ val = f"={v}"
+ options.append(f"--{k}{val}")
+ return options
+
+
+def submit_job(jobscript, **sbatch_options):
+ """Submit jobscript and return jobid."""
+ options = format_sbatch_options(**sbatch_options)
+ try:
+ cmd = ["sbatch"] + ["--parsable"] + options + [jobscript]
+ res = sp.check_output(cmd)
+ except sp.CalledProcessError as e:
+ raise e
+ # Get jobid
+ res = res.decode()
+ try:
+ jobid = re.search(r"(\d+)", res).group(1)
+ except Exception as e:
+ raise e
+ return jobid
+
+
+def advanced_argument_conversion(arg_dict):
+ """Experimental adjustment of sbatch arguments to the given or default partition."""
+ # Currently not adjusting for multiple node jobs
+ nodes = int(arg_dict.get("nodes", 1))
+ if nodes > 1:
+ return arg_dict
+ partition = arg_dict.get("partition", None) or _get_default_partition()
+ constraint = arg_dict.get("constraint", None)
+ ncpus = int(arg_dict.get("cpus-per-task", 1))
+ runtime = arg_dict.get("time", None)
+ memory = _convert_units_to_mb(arg_dict.get("mem", 0))
+ config = _get_cluster_configuration(partition, constraint, memory)
+ mem = arg_dict.get("mem", ncpus * min(config["MEMORY_PER_CPU"]))
+ mem = _convert_units_to_mb(mem)
+ if mem > max(config["MEMORY"]):
+ logger.info(
+ f"requested memory ({mem}) > max memory ({max(config['MEMORY'])}); "
+ "adjusting memory settings"
+ )
+ mem = max(config["MEMORY"])
+
+ # Calculate available memory as defined by the number of requested
+ # cpus times memory per cpu
+ AVAILABLE_MEM = ncpus * min(config["MEMORY_PER_CPU"])
+ # Add additional cpus if memory is larger than AVAILABLE_MEM
+ if mem > AVAILABLE_MEM:
+ logger.info(
+ f"requested memory ({mem}) > "
+ f"ncpus x MEMORY_PER_CPU ({AVAILABLE_MEM}); "
+ "trying to adjust number of cpus up"
+ )
+ ncpus = int(math.ceil(mem / min(config["MEMORY_PER_CPU"])))
+ if ncpus > max(config["CPUS"]):
+ logger.info(
+ f"ncpus ({ncpus}) > available cpus ({max(config['CPUS'])}); "
+ "adjusting number of cpus down"
+ )
+ ncpus = min(int(max(config["CPUS"])), ncpus)
+ adjusted_args = {"mem": int(mem), "cpus-per-task": ncpus}
+
+ # Update time. If requested time is larger than maximum allowed time, reset
+ if runtime:
+ runtime = time_to_minutes(runtime)
+ time_limit = max(config["TIMELIMIT_MINUTES"])
+ if runtime > time_limit:
+ logger.info(
+ f"time (runtime) > time limit {time_limit}; " "adjusting time down"
+ )
+ adjusted_args["time"] = time_limit
+
+ # update and return
+ arg_dict.update(adjusted_args)
+ return arg_dict
+
+
+timeformats = [
+ re.compile(r"^(?P<days>\d+)-(?P<hours>\d+):(?P<minutes>\d+):(?P<seconds>\d+)$"),
+ re.compile(r"^(?P<days>\d+)-(?P<hours>\d+):(?P<minutes>\d+)$"),
+ re.compile(r"^(?P<days>\d+)-(?P<hours>\d+)$"),
+ re.compile(r"^(?P<hours>\d+):(?P<minutes>\d+):(?P<seconds>\d+)$"),
+ re.compile(r"^(?P<minutes>\d+):(?P<seconds>\d+)$"),
+ re.compile(r"^(?P<minutes>\d+)$"),
+]
+
+
+def time_to_minutes(time):
+ """Convert time string to minutes.
+
+ According to slurm:
+
+ Acceptable time formats include "minutes", "minutes:seconds",
+ "hours:minutes:seconds", "days-hours", "days-hours:minutes"
+ and "days-hours:minutes:seconds".
+
+ """
+ if not isinstance(time, str):
+ time = str(time)
+ d = {"days": 0, "hours": 0, "minutes": 0, "seconds": 0}
+ regex = list(filter(lambda regex: regex.match(time) is not None, timeformats))
+ if len(regex) == 0:
+ return
+ assert len(regex) == 1, "multiple time formats match"
+ m = regex[0].match(time)
+ d.update(m.groupdict())
+ minutes = (
+ int(d["days"]) * 24 * 60
+ + int(d["hours"]) * 60
+ + int(d["minutes"])
+ + math.ceil(int(d["seconds"]) / 60)
+ )
+ assert minutes > 0, "minutes has to be greater than 0"
+ return minutes
+
+
+def _get_default_partition():
+ """Retrieve default partition for cluster"""
+ cluster = CookieCutter.get_cluster_option()
+ cmd = f"sinfo -O partition {cluster}"
+ res = sp.check_output(cmd.split())
+ m = re.search(r"(?P<partition>\S+)\*", res.decode(), re.M)
+ partition = m.group("partition")
+ return partition
+
+
+def _get_cluster_configuration(partition, constraints=None, memory=0):
+ """Retrieve cluster configuration.
+
+ Retrieve cluster configuration for a partition filtered by
+ constraints, memory and cpus
+
+ """
+ try:
+ import pandas as pd
+ except ImportError:
+ print(
+ "Error: currently advanced argument conversion "
+ "depends on 'pandas'.", file=sys.stderr
+ )
+ sys.exit(1)
+
+ if constraints:
+ constraint_set = set(constraints.split(","))
+ cluster = CookieCutter.get_cluster_option()
+ cmd = f"sinfo -e -o %all -p {partition} {cluster}".split()
+ try:
+ output = sp.Popen(" ".join(cmd), shell=True, stdout=sp.PIPE).communicate()
+ except Exception as e:
+ print(e)
+ raise
+ data = re.sub("^CLUSTER:.+\n", "", re.sub(" \\|", "|", output[0].decode()))
+ df = pd.read_csv(StringIO(data), sep="|")
+ try:
+ df["TIMELIMIT_MINUTES"] = df["TIMELIMIT"].apply(time_to_minutes)
+ df["MEMORY_PER_CPU"] = df["MEMORY"] / df["CPUS"]
+ df["FEATURE_SET"] = df["AVAIL_FEATURES"].str.split(",").apply(set)
+ except Exception as e:
+ print(e)
+ raise
+ if constraints:
+ constraint_set = set(constraints.split(","))
+ i = df["FEATURE_SET"].apply(lambda x: len(x.intersection(constraint_set)) > 0)
+ df = df.loc[i]
+ memory = min(_convert_units_to_mb(memory), max(df["MEMORY"]))
+ df = df.loc[df["MEMORY"] >= memory]
+ return df
--
GitLab