Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
Sen2Chain
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
ESPACE-DEV
Sen2Chain
Commits
5c2edfb0
Commit
5c2edfb0
authored
2 years ago
by
TGermain
Browse files
Options
Downloads
Patches
Plain Diff
Jobs docstrings
parent
f3a3b30d
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
sen2chain/jobs.py
+47
-2
47 additions, 2 deletions
sen2chain/jobs.py
with
47 additions
and
2 deletions
sen2chain/jobs.py
+
47
−
2
View file @
5c2edfb0
...
@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
...
@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
class
Jobs
:
class
Jobs
:
"""
"""
Class to manage
created job
s
Class to manage
jobs routine
s
"""
"""
def
__init__
(
self
):
def
__init__
(
self
):
...
@@ -106,7 +106,16 @@ class Jobs:
...
@@ -106,7 +106,16 @@ class Jobs:
class
Job
:
class
Job
:
"""
"""
Class to manage job
Class to create, edit or delete a job routine. A job consist of a group of tasks. A task is a succession of sen2chain main Tile processings :
- Download L1C
- Process L2A
- Process cloudmask (only 1 per task)
- Process Indices (Any, using previously produced cloudmask)
Both L1C and L2A can be removed once the task is performed.
Job editing can be done manually in sen2chain_data/config/jobs/yourjobname.cfg
A job can be launched immediately or scheduled with the cron command line.
"""
"""
# logger.propagate = False
# logger.propagate = False
...
@@ -148,6 +157,12 @@ class Job:
...
@@ -148,6 +157,12 @@ class Job:
self
.
tasks
=
pd
.
DataFrame
(
first_row
)
self
.
tasks
=
pd
.
DataFrame
(
first_row
)
def
task_add
(
self
,
row
:
dict
=
None
):
def
task_add
(
self
,
row
:
dict
=
None
):
"""
Add a task to the job. If row : dict = None, default settings are used
:param row: Dictionnary of task attributes.
:type row: dict
"""
if
not
row
:
if
not
row
:
logger
.
info
(
"
No row provided, using default
"
)
logger
.
info
(
"
No row provided, using default
"
)
row
=
pd
.
DataFrame
(
row
=
pd
.
DataFrame
(
...
@@ -172,6 +187,12 @@ class Job:
...
@@ -172,6 +187,12 @@ class Job:
logger
.
info
(
"
\n
{}
"
.
format
(
self
.
tasks
))
logger
.
info
(
"
\n
{}
"
.
format
(
self
.
tasks
))
def
task_edit
(
self
,
task_id
:
int
=
None
,
**
kwargs
):
def
task_edit
(
self
,
task_id
:
int
=
None
,
**
kwargs
):
"""
Edit existing task from job routine. Any task attribute can be provided.
:param task_id: Task number ID. Default to none
:type task_id: int
"""
if
task_id
is
None
:
if
task_id
is
None
:
logger
.
info
(
logger
.
info
(
"
Please provide task_number to edit, if no task in job, create_task first
"
"
Please provide task_number to edit, if no task in job, create_task first
"
...
@@ -194,6 +215,12 @@ class Job:
...
@@ -194,6 +215,12 @@ class Job:
logger
.
info
(
"
Task_number not found
"
)
logger
.
info
(
"
Task_number not found
"
)
def
task_remove
(
self
,
task_id
:
int
=
None
):
def
task_remove
(
self
,
task_id
:
int
=
None
):
"""
Remove task from job.
:param task_id: Tas number ID
:type task_id: int
"""
if
task_id
is
None
:
if
task_id
is
None
:
logger
.
info
(
"
Please provide task_number to remove
"
)
logger
.
info
(
"
Please provide task_number to remove
"
)
else
:
else
:
...
@@ -205,6 +232,9 @@ class Job:
...
@@ -205,6 +232,9 @@ class Job:
logger
.
info
(
"
Task_number not found
"
)
logger
.
info
(
"
Task_number not found
"
)
def
save
(
self
):
def
save
(
self
):
"""
Save all task edits to job. Job file located in sen2chain_data/config/jobs/yourjob.cfg
"""
# save task to disk
# save task to disk
with
open
(
str
(
self
.
_config_path
),
"
w
"
)
as
ict
:
with
open
(
str
(
self
.
_config_path
),
"
w
"
)
as
ict
:
comments_header
=
"
\n
"
.
join
(
comments_header
=
"
\n
"
.
join
(
...
@@ -242,6 +272,9 @@ class Job:
...
@@ -242,6 +272,9 @@ class Job:
self
.
tasks
.
to_csv
(
ict
,
index
=
False
,
sep
=
"
;
"
)
self
.
tasks
.
to_csv
(
ict
,
index
=
False
,
sep
=
"
;
"
)
def
get_cron_status
(
self
):
def
get_cron_status
(
self
):
"""
Return cron status of selected job : enabled, disabled or absent from cron.
"""
iter
=
list
(
self
.
_cron
.
find_comment
(
"
sen2chain_job_
"
+
self
.
jid
))
iter
=
list
(
self
.
_cron
.
find_comment
(
"
sen2chain_job_
"
+
self
.
jid
))
if
iter
:
if
iter
:
for
job
in
iter
:
for
job
in
iter
:
...
@@ -276,6 +309,9 @@ class Job:
...
@@ -276,6 +309,9 @@ class Job:
def
cron_enable
(
def
cron_enable
(
self
,
self
,
):
):
"""
Enable job in cron.
"""
# enable job in cron
# enable job in cron
self
.
save
()
self
.
save
()
self
.
create_python_script
()
self
.
create_python_script
()
...
@@ -304,6 +340,9 @@ class Job:
...
@@ -304,6 +340,9 @@ class Job:
self
.
get_cron_status
()
self
.
get_cron_status
()
def
cron_disable
(
self
):
def
cron_disable
(
self
):
"""
Disable job from cron.
"""
# disable / commenting job in cron
# disable / commenting job in cron
iter
=
list
(
self
.
_cron
.
find_comment
(
"
sen2chain_job_
"
+
self
.
jid
))
iter
=
list
(
self
.
_cron
.
find_comment
(
"
sen2chain_job_
"
+
self
.
jid
))
if
iter
:
if
iter
:
...
@@ -314,6 +353,9 @@ class Job:
...
@@ -314,6 +353,9 @@ class Job:
self
.
get_cron_status
()
self
.
get_cron_status
()
def
cron_remove
(
self
):
def
cron_remove
(
self
):
"""
Remove job from cron.
"""
# remove job from cron
# remove job from cron
iter
=
list
(
self
.
_cron
.
find_comment
(
"
sen2chain_job_
"
+
self
.
jid
))
iter
=
list
(
self
.
_cron
.
find_comment
(
"
sen2chain_job_
"
+
self
.
jid
))
if
iter
:
if
iter
:
...
@@ -446,6 +488,9 @@ class Job:
...
@@ -446,6 +488,9 @@ class Job:
clean_before
:
bool
=
False
,
clean_before
:
bool
=
False
,
clean_after
:
bool
=
False
,
clean_after
:
bool
=
False
,
):
):
"""
Run job. Tasks are executed on
"""
if
self
.
logs
:
if
self
.
logs
:
self
.
_log_folder_path
.
mkdir
(
exist_ok
=
True
)
self
.
_log_folder_path
.
mkdir
(
exist_ok
=
True
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment