Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
mpievolbio-scicomp
RepinPop
Commits
515ede26
Commit
515ede26
authored
Dec 15, 2021
by
Carsten
Browse files
debugging logging and rq.
parent
456a0d16
Pipeline
#258327
failed with stages
in 6 minutes and 11 seconds
Changes
7
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
app/__init__.py
View file @
515ede26
...
...
@@ -6,12 +6,22 @@ from flask_wtf.csrf import CSRFProtect
from
flask_mail
import
Mail
from
flask_mongoengine
import
MongoEngine
,
MongoEngineSessionInterface
from
redis
import
Redis
import
datetime
import
logging
import
datetime
logger
=
logging
.
getLogger
(
'rarefan'
)
formatter
=
logging
.
Formatter
(
'%(asctime)s [%(levelname)s] %(module)s: %(message)s'
)
timestamp
=
datetime
.
datetime
.
now
().
strftime
(
format
=
"%Y%m%d-%H%M%S"
)
handler
=
logging
.
FileHandler
(
"/tmp/rarefan.log"
.
format
(
timestamp
))
handler
.
setFormatter
(
formatter
)
handler
.
setLevel
(
logging
.
DEBUG
)
logger
.
addHandler
(
handler
)
app
=
Flask
(
__name__
,
instance_relative_config
=
True
,
static_url_path
=
'/static'
)
upload_dir
=
os
.
path
.
join
(
app
.
static_folder
,
'uploads'
)
app
.
testing
=
app
.
debug
=
Fals
e
app
.
testing
=
app
.
debug
=
Tru
e
# email
app
.
config
.
from_object
(
Config
)
...
...
app/routes.py
View file @
515ede26
...
...
@@ -21,9 +21,12 @@ from app.tasks.rarefan import rarefan_task
from
app.tasks.tree
import
tree_task
,
empty_task
from
app.tasks.zip
import
zip_task
from
app.tasks.email
import
email_task
from
app.tasks.redis_tests
import
example
from
app.callbacks.callbacks
import
on_success
,
on_failure
import
logging
import
datetime
import
copy
import
os
import
shlex
...
...
@@ -31,19 +34,9 @@ import shutil
import
stat
import
subprocess
import
tempfile
import
logging
from
Bio
import
SeqIO
import
datetime
logger
=
logging
.
getLogger
(
__name__
)
formatter
=
logging
.
Formatter
(
'%(asctime)s [%(levelname)s] %(module)s: %(message)s'
)
timestamp
=
datetime
.
datetime
.
now
().
strftime
(
format
=
"%Y%m%d-%H%M%S"
)
handler
=
logging
.
FileHandler
(
"/tmp/rarefan.log"
.
format
(
timestamp
))
handler
.
setFormatter
(
formatter
)
handler
.
setLevel
(
logging
.
DEBUG
)
logger
.
addHandler
(
handler
)
logger
=
logging
.
getLogger
(
'rarefan'
)
def
get_status_code
(
run_id_path
):
# Check if the run has finished.
...
...
@@ -62,13 +55,13 @@ def validate_fasta(filename):
:param filename: The filename of the file to validate.
"""
logg
ing
.
info
(
"Validating fasta file %s."
,
filename
)
logg
er
.
info
(
"Validating fasta file %s."
,
filename
)
with
open
(
filename
,
'r'
)
as
fp
:
fasta
=
SeqIO
.
parse
(
fp
,
"fasta"
)
is_fasta
=
any
(
fasta
)
if
not
is_fasta
:
logg
ing
.
warning
(
"%s is not a valid fasta file."
,
filename
)
logg
er
.
warning
(
"%s is not a valid fasta file."
,
filename
)
return
is_fasta
...
...
@@ -76,10 +69,9 @@ def validate_fasta(filename):
def
index
():
return
render_template
(
"index.html"
)
@
app
.
route
(
'/upload'
,
methods
=
[
'GET'
,
'POST'
])
def
upload
():
logg
ing
.
debug
(
"upload/%s"
,
request
.
method
)
logg
er
.
debug
(
"upload/%s"
,
request
.
method
)
if
request
.
method
==
'POST'
:
#upload_form.validate_on_submit():
session
[
'tmpdir'
]
=
tempfile
.
mkdtemp
(
suffix
=
None
,
...
...
@@ -88,7 +80,7 @@ def upload():
)
seqs
=
[
v
for
k
,
v
in
request
.
files
.
items
()
if
k
.
startswith
(
'file'
)]
logg
ing
.
info
(
"Uploading %s."
,
str
(
seqs
))
logg
er
.
info
(
"Uploading %s."
,
str
(
seqs
))
dna_extensions
=
[
'fn'
,
'fna'
,
'fastn'
,
'fas'
,
'fasta'
]
aa_extensions
=
[
'fa'
,
'faa'
]
...
...
@@ -116,7 +108,7 @@ def upload():
session
[
'tree_names'
]
=
tree_names
for
k
,
v
in
session
.
items
():
logg
ing
.
debug
(
"session[%s] = %s"
,
k
,
str
(
v
))
logg
er
.
debug
(
"session[%s] = %s"
,
k
,
str
(
v
))
return
redirect
(
url_for
(
'submit'
,
_method
=
'GET'
))
...
...
@@ -147,7 +139,7 @@ def upload():
@
app
.
route
(
'/submit'
,
methods
=
[
'GET'
,
'POST'
])
def
submit
():
logg
ing
.
debug
(
"submit/%s"
,
request
.
method
)
logg
er
.
debug
(
"submit/%s"
,
request
.
method
)
submit_form
=
SubmitForm
()
strain_names
=
session
.
get
(
'strain_names'
)
...
...
@@ -159,13 +151,13 @@ def submit():
tmpdir
=
session
[
'tmpdir'
]
session
[
'outdir'
]
=
os
.
path
.
join
(
tmpdir
,
'out'
)
logg
ing
.
debug
(
"tmpdir: %s"
,
tmpdir
)
logg
ing
.
debug
(
"tmpdir: %s"
,
session
[
'tmpdir'
])
logg
er
.
debug
(
"tmpdir: %s"
,
tmpdir
)
logg
er
.
debug
(
"tmpdir: %s"
,
session
[
'tmpdir'
])
# If there is already an outdir, this must be a rerun!
old_run_id
=
None
if
os
.
path
.
isdir
(
session
[
'outdir'
]):
logg
ing
.
warning
(
"Rerun, creating new directory"
)
logg
er
.
warning
(
"Rerun, creating new directory"
)
rerun_tmpdir
=
tempfile
.
mkdtemp
(
suffix
=
None
,
prefix
=
""
,
...
...
@@ -195,12 +187,12 @@ def submit():
session
[
'analyse_repins'
]
=
request
.
form
.
get
(
'analyse_repins'
)
session
[
'email'
]
=
request
.
form
.
get
(
'email'
,
None
)
logg
ing
.
info
(
"Session parameters:"
)
logg
ing
.
info
(
"tmpdir: %s"
,
session
[
'tmpdir'
])
logg
ing
.
info
(
"outdir: %s"
,
session
[
'outdir'
])
logg
ing
.
info
(
"reference_strain: %s"
,
session
[
'reference_strain'
])
logg
ing
.
info
(
"treefile: %s"
,
session
[
'treefile'
])
logg
ing
.
info
(
"email: %s"
,
session
[
'email'
])
logg
er
.
info
(
"Session parameters:"
)
logg
er
.
info
(
"tmpdir: %s"
,
session
[
'tmpdir'
])
logg
er
.
info
(
"outdir: %s"
,
session
[
'outdir'
])
logg
er
.
info
(
"reference_strain: %s"
,
session
[
'reference_strain'
])
logg
er
.
info
(
"treefile: %s"
,
session
[
'treefile'
])
logg
er
.
info
(
"email: %s"
,
session
[
'email'
])
# Store session in db.
run_id
=
os
.
path
.
basename
(
session
[
'tmpdir'
])
...
...
@@ -232,7 +224,11 @@ def submit():
notification_is_sent
=
False
,
parent_run
=
old_run_id
)
dbjob
.
save
()
logger
.
debug
(
"Constructed dbjob with job ID %s."
,
dbjob
.
run_id
)
logger
.
debug
(
"Attempting to save dbjob in DB."
)
success
=
dbjob
.
save
()
logger
.
debug
(
"Return code is %s"
,
str
(
success
))
# If one of the server provided rayt files was selected, copy it to the working dir. In the dropdown menu,
# the server provided rayts are listed without filename extension, so have to append that here.
...
...
@@ -240,7 +236,7 @@ def submit():
if
session
[
'query_rayt'
]
in
[
'yafM_Ecoli'
,
'yafM_SBW25'
]:
query_rayt_fname
=
query_rayt_fname
+
".faa"
src
=
os
.
path
.
join
(
app
.
static_folder
,
"rayts"
,
session
[
'query_rayt'
]
+
".faa"
)
logg
ing
.
debug
(
"Copying rayt from %s to %s."
,
src
,
query_rayt_fname
)
logg
er
.
debug
(
"Copying rayt from %s to %s."
,
src
,
query_rayt_fname
)
shutil
.
copyfile
(
src
,
query_rayt_fname
)
if
not
os
.
path
.
isfile
(
query_rayt_fname
):
...
...
@@ -266,6 +262,8 @@ def submit():
}
)
logger
.
debug
(
"Constructed rarefan job %s."
,
str
(
rarefan_job
))
run_tree_task
=
len
(
dbjob
.
setup
[
'strain_names'
])
>=
4
if
run_tree_task
:
...
...
@@ -290,6 +288,7 @@ def submit():
connection
=
app
.
redis
,
)
logger
.
debug
(
"Constructed tree job %s."
,
str
(
tree_job
))
zip_job
=
RQJob
.
create
(
zip_task
,
depends_on
=
[
rarefan_job
,
tree_job
],
on_success
=
on_success
,
...
...
@@ -298,6 +297,7 @@ def submit():
connection
=
app
.
redis
,
kwargs
=
{
'run_dir'
:
session
[
'tmpdir'
]},
)
logger
.
debug
(
"Constructed zip job %s."
,
str
(
zip_job
))
email_job
=
RQJob
.
create
(
email_task
,
depends_on
=
[
'rarefan_job'
,
...
...
@@ -309,6 +309,8 @@ def submit():
kwargs
=
{
'run_id'
:
run_id
},
)
logger
.
debug
(
"Constructed email job %s."
,
str
(
email_job
))
# Enqueue the jobs
app
.
queue
.
enqueue_job
(
rarefan_job
)
app
.
queue
.
enqueue_job
(
tree_job
)
...
...
@@ -347,7 +349,7 @@ def results():
run_id
=
request
.
form
.
get
(
'run_id'
)
if
run_id
is
not
None
:
logg
ing
.
debug
(
run_id
)
logg
er
.
debug
(
run_id
)
dbjob
=
DBJob
.
objects
.
get_or_404
(
run_id
=
run_id
)
...
...
@@ -401,7 +403,7 @@ def files(req_path):
# Remove trailing '/'
if
req_path
.
endswith
(
'/'
):
req_path
=
req_path
[:
-
1
]
logg
ing
.
warning
(
"Request dir is %s in (%s)."
,
req_path
,
os
.
path
.
dirname
(
req_path
))
logg
er
.
warning
(
"Request dir is %s in (%s)."
,
req_path
,
os
.
path
.
dirname
(
req_path
))
# Save the target for the 'back to results' link.
tmp_dir
=
session
.
get
(
'tmpdir'
,
None
)
...
...
@@ -451,8 +453,8 @@ def rerun():
run_id
=
request
.
args
[
'run_id'
]
do_repins
=
request
.
args
.
get
(
'do_repins'
,
None
)
dbjob
=
DBJob
.
objects
.
get_or_404
(
run_id
=
run_id
)
logg
ing
.
debug
(
"Found job %s"
,
str
(
dbjob
.
id
))
logg
ing
.
debug
(
"Job run_id = %s"
,
str
(
dbjob
.
run_id
))
logg
er
.
debug
(
"Found job %s"
,
str
(
dbjob
.
id
))
logg
er
.
debug
(
"Job run_id = %s"
,
str
(
dbjob
.
run_id
))
submit_form
=
SubmitForm
()
...
...
@@ -484,8 +486,8 @@ def rerun():
session
[
'email'
]
=
dbjob
.
setup
.
get
(
'email'
)
submit_form
.
email
.
data
=
dbjob
.
setup
.
get
(
'email'
)
logg
ing
.
debug
(
"DO_REPINS? %s"
,
do_repins
)
logg
ing
.
debug
(
"analyse_repins= %s"
,
submit_form
.
analyse_repins
.
data
)
logg
er
.
debug
(
"DO_REPINS? %s"
,
do_repins
)
logg
er
.
debug
(
"analyse_repins= %s"
,
submit_form
.
analyse_repins
.
data
)
# Update do_repins if requested.
if
do_repins
is
not
None
:
if
do_repins
in
[
'y'
,
'1'
,
1
,
True
]:
...
...
@@ -493,7 +495,7 @@ def rerun():
else
:
submit_form
.
analyse_repins
.
data
=
session
[
'analyse_repins'
]
=
None
logg
ing
.
debug
(
'session = %s'
,
str
(
session
))
logg
er
.
debug
(
'session = %s'
,
str
(
session
))
return
render_template
(
'submit.html'
,
...
...
@@ -511,3 +513,12 @@ def plot():
# return redirect('http://localhost:7238?run_id={}'.format(run_id))
return
redirect
(
'http://rarefan.evolbio.mpg.de/shiny/analysis?run_id={}'
.
format
(
run_id
))
@
app
.
route
(
'/test_task'
)
def
test_task
():
job
=
app
.
queue
.
enqueue
(
example
,
10
)
logger
.
info
(
job
.
result
)
return
redirect
(
url_for
(
'index'
))
app/tasks/rarefan.py
View file @
515ede26
...
...
@@ -4,10 +4,11 @@ module rarefan: Implementation of the function that runs the rarefan java code a
import
os
,
sys
,
shutil
,
shlex
import
subprocess
import
logging
from
app.models
import
Job
as
DBJob
from
rq
import
get_current_job
import
logging
logging
.
getLogger
(
'rarefan'
)
from
app.utilities.rarefan_cli
import
rarefan_command
...
...
app/tasks/redis_tests.py
View file @
515ede26
import
time
import
logging
logging
.
basicConfig
(
level
=
logging
.
DEBUG
)
import
logging
logger
=
logging
.
getLogger
()
def
example
(
seconds
):
logging
.
info
(
"************ Starting task *****************"
)
seconds
=
int
(
seconds
)
for
i
in
range
(
seconds
):
logg
ing
.
debug
(
i
)
logg
er
.
debug
(
i
)
time
.
sleep
(
1
)
logging
.
info
(
"************ Task complete *****************"
)
logging
.
debug
(
"Done"
)
# example(10)
app/tasks/tree.py
View file @
515ede26
...
...
@@ -7,7 +7,7 @@ from app.models import Job as DBJob
from
rq
import
get_current_job
import
logging
logging
.
basicConfig
(
level
=
logging
.
DEBUG
)
logging
.
getLogger
(
'rarefan'
)
def
tree_task
(
run_dir
,
treefile
=
None
):
""" Generate a phylogenetic tree from all DNA sequence files in given directory.
...
...
app/tasks/zip.py
View file @
515ede26
...
...
@@ -6,6 +6,8 @@ import shutil
from
app.models
import
Job
as
DBJob
from
rq
import
get_current_job
import
logging
logging
.
getLogger
(
'rarefan'
)
def
zip_task
(
run_dir
):
...
...
app/utilities/__init__.py
View file @
515ede26
from
.
import
checkers
from
.
import
rarefan_cli
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment