Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
# -*- coding:utf-8 -*-
""" lance les traitements sur les fichiers L1C qui ont
été téléchargés mais qui n'ont pas été traités
"""
import logging
import datetime
import pandas as pd
from sen2chain import Tile, L1cProduct, L2aProduct
from sen2chain import l2a_multiprocessing, cld_version_probability_iterations_reprocessing_multiprocessing, idx_multiprocessing
import os, shutil, sys
import glob
import math
from itertools import chain
try:
arg = sys.argv[1]
except:
arg = []
print(arg)
logger = logging.getLogger("Traitement L2A")
logging.basicConfig(level=logging.INFO)
logger.setLevel(logging.INFO)
# default nombre de jours à requeter à partir de today
# si le champs n'est pas renseingné ds le csv
delta_t = 15
# liste de tuiles à traiter
fwd = os.path.dirname(os.path.realpath(__file__))
tiles_file = fwd + "/99_traitement_l2a_multiprocessing.csv"
tiles_list = pd.read_csv(tiles_file, sep = ';', na_values="", na_filter=False, comment='#')
for index, row in tiles_list.iterrows():
if not row.start_time:
tiles_list.at[index, "start_time"] = (datetime.datetime.now()-datetime.timedelta(days=delta_t)).strftime('%Y-%m-%d')
if not row.end_time:
tiles_list.at[index, "end_time"] = (datetime.datetime.now()+datetime.timedelta(days=1)).strftime('%Y-%m-%d')
if not row.indices:
tiles_list.at[index, "indices"] = 'NDVI'
logger.info("\n{}".format(tiles_list))
# Traitement des L1C en L2A
l1c_process_list = []
for index, row in tiles_list.iterrows():
t = Tile(row.tile)
l1c_process_list.append(list(p.identifier for p in t.l2a_missings.filter_dates(date_min = row.start_time, date_max = row.end_time)))
l1c_process_list = list(chain.from_iterable(l1c_process_list))
logger.info("l1c_process_list ({} files): {}\n".format(len(l1c_process_list), l1c_process_list))
l2a_res = False
if l1c_process_list:
if not arg:
l2a_res = l2a_multiprocessing(l1c_process_list, nb_proc=12)
# Traitement des L2A (clouds)
cm_version = "cm001"
probability = 1
iterations = 5
reprocess = False
cld_l2a_process_list = []
for index, row in tiles_list.iterrows():
t = Tile(row.tile)
if not reprocess:
l2a_list = [p.identifier for p in t.cloudmasks_missing(cm_version = cm_version,
probability = probability,
iterations = iterations,
)\
.filter_dates(date_min = row.start_time, date_max = row.end_time)]
else:
l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = row.start_time, date_max = row.end_time)]
for j in l2a_list:
cld_l2a_process_list.append([j,
cm_version,
probability,
iterations,
reprocess,
])
logger.info("\ncld_l2a_process_list ({} files): {}".format(len(cld_l2a_process_list), cld_l2a_process_list))
cld_res = False
if cld_l2a_process_list:
if not arg:
cld_res = cld_version_probability_iterations_reprocessing_multiprocessing(cld_l2a_process_list, nb_proc=8)
# Traitement des L2A (indices)
reprocess = False
nodata_clouds = True
quicklook = False
cm_version = "cm001"
probability = 1
iterations = 5
indices_l2a_process_list = []
for index, row in tiles_list.iterrows():
t = Tile(row.tile)
indices_list = row.indices.split("/")
for i in indices_list:
if not reprocess:
l2a_list = [p.identifier for p in t.missing_indices(i,
nodata_clouds = nodata_clouds,
cm_version = cm_version,
probability = probability,
iterations = iterations,
).filter_dates(date_min = row.start_time, date_max = row.end_time)]
else:
l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = row.start_time, date_max = row.end_time)]
for j in l2a_list:
#~ indices_l2a_process_list.append([j, i])
indices_l2a_process_list.append([j,
i,
reprocess,
nodata_clouds,
quicklook,
cm_version,
probability,
iterations])
logger.info("\nindices_l2a_process_list ({} files): {}".format(len(indices_l2a_process_list), indices_l2a_process_list))
indices_res = False
if indices_l2a_process_list:
if not arg:
indices_res = idx_multiprocessing(indices_l2a_process_list, nb_proc=8)