Skip to content

Commit d625ea6

Browse files
author
Cresson Remi
committed
Merge branch '5-pyotb_update' into 'master'
FIX: follow API changes in pyotb Closes #5 See merge request umr-tetis/rt_decloud!7
2 parents 4f71f0b + 17ba084 commit d625ea6

14 files changed

Lines changed: 58 additions & 72 deletions

‎.gitlab-ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ variables:
44
GPU_IMAGE_NAME: $CI_REGISTRY_IMAGE:gpu
55
DOCKER_BUILDKIT: 1
66
DOCKER_DRIVER: overlay2
7-
CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:cpu-basic-dev
8-
GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:gpu-basic-dev
7+
CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-cpu-dev
8+
GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-gpu-dev
99

1010
workflow:
1111
rules:

‎Dockerfile‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# Decloud dockerfile
22
# To build the docker image for cpu, do the following:
33
#
4-
# docker build --build-arg "BASE_IMAGE=mdl4eo/otbtf3.0:cpu-basic-dev" .
4+
# docker build --build-arg "BASE_IMAGE=mdl4eo/otbtf3.3.2:cpu-dev" .
55
#
6-
ARG BASE_IMAGE=mdl4eo/otbtf3.0:gpu-dev
6+
ARG BASE_IMAGE=mdl4eo/otbtf3.3.2:gpu-dev
77
FROM $BASE_IMAGE
88
LABEL description="Decloud docker image"
99
LABEL maintainer="Remi Cresson [at] inrae [dot] fr"
@@ -18,9 +18,6 @@ RUN apt upgrade -y
1818
COPY docker/requirements.txt /tmp/requirements.txt
1919
RUN python3 -m pip install -r /tmp/requirements.txt
2020

21-
# Fix Mosaic app in OTB <= 7.4
22-
RUN cd /src/otb/otb/ && git config remote.origin.fetch refs/heads/*:refs/remotes/origin/* && git fetch origin && git -c user.name=decloud-docker -c user.email=decloud@decloud.xyz cherry-pick 44ec133646c46d6772450bbf5fb3ec54e282f56b
23-
2421
# Build remote modules
2522
RUN cd /src/otb/otb/Modules/Remote/ && git clone https://gitlab.irstea.fr/remi.cresson/SimpleExtractionTools.git
2623
RUN cd /src/otb/otb/Modules/Remote/ && git clone https://gitlab.irstea.fr/remi.cresson/mlutils.git

‎decloud/analysis/images_stats.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import json
3232
import numpy as np
3333
from matplotlib import pyplot as plt
34-
import gdal
34+
from osgeo import gdal
3535

3636
parser = argparse.ArgumentParser(description="dataset test")
3737
parser.add_argument("--input_dir", help="A directory containing S1 or S2 files", required=True)

‎decloud/core/system.py‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,16 +41,13 @@ def get_commit_hash():
4141
""" Return the git hash of the repository """
4242
repo = git.Repo(os.path.dirname(os.path.realpath(__file__)), search_parent_directories=True)
4343

44-
commit_hash = "nohash"
4544
try:
46-
commit_hash = repo.head.object.hexsha[0:5]
47-
except (ValueError, TypeError) as e:
48-
print(f"Unable to get commit hash! {e}")
49-
50-
try:
51-
commit_hash = repo.active_branch.name + "_" + commit_hash
52-
except (ValueError, TypeError) as e:
53-
print(f"Unable to get branch name! {e}")
45+
commit_hash = repo.active_branch.name + "_" + repo.head.object.hexsha[0:5]
46+
except (TypeError, ValueError, BrokenPipeError):
47+
try:
48+
commit_hash = 'DETACHED_' + repo.head.object.hexsha[0:5]
49+
except (ValueError, BrokenPipeError):
50+
commit_hash = 'DETACHED'
5451

5552
return commit_hash
5653

‎decloud/preprocessing/dem_prepare.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import argparse
2626
import logging
2727
import elevation
28-
import gdal
28+
from osgeo import gdal
2929
import decloud.preprocessing.constants as constants
3030
import otbApplication as otb
3131
from decloud.core import system

‎decloud/production/inference.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def inference(sources, sources_scales, pad, ts, savedmodel_dir, out_tensor, out_
6666

6767
# Setup TensorFlowModelServe
6868
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
69-
infer = pyotb.TensorflowModelServe(execute=False)
69+
parameters = {}
7070

7171
# Inputs
7272
for i, (placeholder, source) in enumerate(sources.items()):
@@ -80,18 +80,18 @@ def get_key(key):
8080
if placeholder in sources_scales:
8181
src_rfield = int(rfield / sources_scales[placeholder])
8282

83-
infer.set_parameters({get_key("il"): [source],
84-
get_key("rfieldx"): src_rfield,
85-
get_key("rfieldy"): src_rfield,
86-
get_key("placeholder"): placeholder})
83+
parameters.update({get_key("il"): [source],
84+
get_key("rfieldx"): src_rfield,
85+
get_key("rfieldy"): src_rfield,
86+
get_key("placeholder"): placeholder})
8787

8888
# Model
89-
infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
90-
"output.names": [padded_tensor_name(out_tensor, pad)],
91-
"output.efieldx": efield, "output.efieldy": efield,
92-
"optim.tilesizex": efield, "optim.tilesizey": efield,
93-
"optim.disabletiling": 1})
94-
infer.Execute()
89+
parameters.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
90+
"output.names": [padded_tensor_name(out_tensor, pad)],
91+
"output.efieldx": efield, "output.efieldy": efield,
92+
"optim.tilesizex": efield, "optim.tilesizey": efield,
93+
"optim.disabletiling": True})
94+
infer = pyotb.TensorflowModelServe(parameters)
9595

9696
# Post Processing
9797
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image

‎decloud/production/monthly_synthesis_processor_s2.py‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,10 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
7474

7575
# Setup TensorFlowModelServe
7676
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
77-
infer = pyotb.App("TensorflowModelServe", execute=False)
77+
infer_params = {}
7878

7979
# Setup BandMath for post processing
80-
bm = pyotb.App("BandMath", execute=False)
80+
bm_params = {}
8181
mask_expr = "0"
8282

8383
# Inputs
@@ -93,41 +93,42 @@ def get_key(key):
9393
if placeholder in sources_scales:
9494
src_rfield = int(rfield / sources_scales[placeholder])
9595

96-
infer.set_parameters({get_key("il"): [source]})
96+
infer_params.update({get_key("il"): [source]})
9797

9898
# Update post processing BandMath expression
9999
if placeholder != 'dem' and '20m' not in placeholder:
100100
nodatavalue = nodatavalues[placeholder]
101101
n_channels = pyotb.get_nbchannels(source)
102102
mask_expr += "||"
103103
mask_expr += "&&".join(["im{}b{}=={}".format(k + 1, b, nodatavalue) for b in range(1, 1 + n_channels)])
104-
bm.set_parameters(il=[source])
104+
bm_params.update({'il': source})
105105
k += 1
106106

107-
infer.set_parameters({get_key("rfieldx"): src_rfield,
108-
get_key("rfieldy"): src_rfield,
109-
get_key("placeholder"): placeholder})
107+
infer_params.update({get_key("rfieldx"): src_rfield,
108+
get_key("rfieldy"): src_rfield,
109+
get_key("placeholder"): placeholder})
110110

111111
# Model
112-
infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
113-
"output.names": [padded_tensor_name(out_tensor, pad)],
114-
"output.efieldx": efield, "output.efieldy": efield,
115-
"optim.tilesizex": efield, "optim.tilesizey": efield,
116-
"optim.disabletiling": 1})
117-
infer.Execute()
112+
infer_params.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
113+
"output.names": [padded_tensor_name(out_tensor, pad)],
114+
"output.efieldx": efield, "output.efieldy": efield,
115+
"optim.tilesizex": efield, "optim.tilesizey": efield,
116+
"optim.disabletiling": True})
117+
infer = pyotb.TensorflowModelServe(infer_params)
118118

119119
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
120120
if out_pixeltype == otbApplication.ImagePixelType_uint16:
121-
n_channels = pyotb.get_nbchannels(infer.out)
121+
n_channels = pyotb.get_nbchannels(infer)
122122
exp = ';'.join([f'(im1b{b}<=1 ? 1 : im1b{b})' for b in range(1, 1 + n_channels)])
123-
rmzeros = pyotb.App("BandMathX", il=[infer.out], exp=exp)
123+
rmzeros = pyotb.App("BandMathX", il=[infer], exp=exp)
124124
rmzeros.SetParameterOutputImagePixelType("out", out_pixeltype)
125125
else:
126126
rmzeros = infer
127127

128128
# Mask for post processing
129129
mask_expr += "?0:255"
130-
bm.set_parameters(exp=mask_expr)
130+
bm_params.update({'exp': mask_expr})
131+
bm = pyotb.BandMath(bm_params)
131132

132133
# Closing post processing mask to remove small groups of NoData pixels
133134
closing = pyotb.App("BinaryMorphologicalOperation", bm, filter="closing", foreval=255, structype="box",

‎decloud/production/monthly_synthesis_processor_s2s1.py‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,10 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
7474

7575
# Setup TensorFlowModelServe
7676
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
77-
infer = pyotb.App("TensorflowModelServe", execute=False)
77+
infer_params = {}
7878

7979
# Setup BandMath for post processing
80-
bm = pyotb.App("BandMath", execute=False)
80+
bm_params = {}
8181
mask_expr = "0"
8282

8383
# Inputs
@@ -93,28 +93,28 @@ def get_key(key):
9393
if placeholder in sources_scales:
9494
src_rfield = int(rfield / sources_scales[placeholder])
9595

96-
infer.set_parameters({get_key("il"): [source]})
96+
infer_params.update({get_key("il"): [source]})
9797

9898
# Update post processing BandMath expression
9999
if placeholder != 'dem' and '20m' not in placeholder:
100100
nodatavalue = nodatavalues[placeholder]
101101
n_channels = pyotb.get_nbchannels(source)
102102
mask_expr += "||"
103103
mask_expr += "&&".join(["im{}b{}=={}".format(k + 1, b, nodatavalue) for b in range(1, 1 + n_channels)])
104-
bm.set_parameters(il=[source])
104+
bm_params.update({'il': [source]})
105105
k += 1
106106

107-
infer.set_parameters({get_key("rfieldx"): src_rfield,
108-
get_key("rfieldy"): src_rfield,
109-
get_key("placeholder"): placeholder})
107+
infer_params.update({get_key("rfieldx"): src_rfield,
108+
get_key("rfieldy"): src_rfield,
109+
get_key("placeholder"): placeholder})
110110

111111
# Model
112-
infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
113-
"output.names": [padded_tensor_name(out_tensor, pad)],
114-
"output.efieldx": efield, "output.efieldy": efield,
115-
"optim.tilesizex": efield, "optim.tilesizey": efield,
116-
"optim.disabletiling": 1})
117-
infer.Execute()
112+
infer_params.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
113+
"output.names": [padded_tensor_name(out_tensor, pad)],
114+
"output.efieldx": efield, "output.efieldy": efield,
115+
"optim.tilesizex": efield, "optim.tilesizey": efield,
116+
"optim.disabletiling": True})
117+
infer = pyotb.TensorflowModelServe(infer_params)
118118

119119
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
120120
if out_pixeltype == otbApplication.ImagePixelType_uint16:
@@ -127,7 +127,8 @@ def get_key(key):
127127

128128
# Mask for post processing
129129
mask_expr += "?0:255"
130-
bm.set_parameters(exp=mask_expr)
130+
bm_params.update({'exp': mask_expr})
131+
bm = pyotb.BandMath(bm_params)
131132

132133
# Closing post processing mask to remove small groups of NoData pixels
133134
closing = pyotb.App("BinaryMorphologicalOperation", bm, filter="closing", foreval=255, structype="box",
@@ -269,6 +270,7 @@ def _closest_date(x):
269270
"""
270271
return abs(s2_product.get_timestamp() - x.get_timestamp())
271272

273+
272274
input_s1_products.sort(key=_closest_date, reverse=True)
273275
input_s1_images_10m = [product.get_raster_10m() for product in input_s1_products]
274276
# creating a mosaic with the N closest S1 images

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
setuptools.setup(
3030
name="decloud",
31-
version="0.1",
31+
version="1.3",
3232
author="Remi Cresson, Nicolas Narçon, Benjamin Commandre",
3333
author_email="remi.cresson@inrae.fr",
3434
description="Deep learning based reconstruction of optical time series using SAR imagery",

‎tests/decloud_unittest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import subprocess
55
import unittest
66
import filecmp
7-
import gdal
7+
from osgeo import gdal
88
import otbApplication as otb
99
from abc import ABC
1010
from decloud.core.system import get_env_var, basename

0 commit comments

Comments
 (0)