From f243345778fac271652508fe7118bc027ebe905d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@irstea.fr>
Date: Mon, 6 Jun 2022 09:51:59 +0200
Subject: [PATCH 01/77] CI: bump otbtf version
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7f713d76..a0efc1a8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
variables:
- OTBTF_VERSION: 3.2.1
+ OTBTF_VERSION: 3.3
OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
OTBTF_SRC: /src/otbtf # Local OTBTF source directory
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
--
GitLab
From 5ba46eb1d7ad9d75be28fc1a2a58a0bc9975395b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@irstea.fr>
Date: Mon, 6 Jun 2022 09:52:14 +0200
Subject: [PATCH 02/77] COMP: bump OTB version
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index da634cea..9c905bd1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,7 +85,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
### OTB
ARG GUI=false
-ARG OTB=7.4.0
+ARG OTB=8.0.1
ARG OTBTESTS=false
RUN mkdir /src/otb
--
GitLab
From f337926649434e1fe1d72ebf1b54204ae5d5c0a2 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@irstea.fr>
Date: Mon, 6 Jun 2022 09:55:29 +0200
Subject: [PATCH 03/77] CI: bump otbtf version
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a0efc1a8..7f713d76 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
variables:
- OTBTF_VERSION: 3.3
+ OTBTF_VERSION: 3.2.1
OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
OTBTF_SRC: /src/otbtf # Local OTBTF source directory
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
--
GitLab
From dfdddd71627d96b15807533fac5a1a18df7ccff5 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@irstea.fr>
Date: Mon, 6 Jun 2022 15:44:01 +0200
Subject: [PATCH 04/77] COMP: move IsNoData() inside otb::wrapper::
---
app/otbPatchesSelection.cxx | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/app/otbPatchesSelection.cxx b/app/otbPatchesSelection.cxx
index 68d76221..3437849b 100644
--- a/app/otbPatchesSelection.cxx
+++ b/app/otbPatchesSelection.cxx
@@ -35,6 +35,12 @@
#include <random>
#include <limits>
+namespace otb
+{
+
+namespace Wrapper
+{
+
// Functor to retrieve nodata
template<class TPixel, class OutputPixel>
class IsNoData
@@ -62,12 +68,6 @@ private:
typename TPixel::ValueType m_NoDataValue;
};
-namespace otb
-{
-
-namespace Wrapper
-{
-
class PatchesSelection : public Application
{
public:
--
GitLab
From ee10301fce2aa43b7830715eccaecbde6673728f Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Mon, 18 Jul 2022 11:43:23 +0200
Subject: [PATCH 05/77] INIT: add the first code for training and model
---
otbtf/examples/tensorflow_v2x/training.py | 13 ++
otbtf/model.py | 141 ++++++++++++++++++++++
2 files changed, 154 insertions(+)
create mode 100644 otbtf/examples/tensorflow_v2x/training.py
create mode 100644 otbtf/model.py
diff --git a/otbtf/examples/tensorflow_v2x/training.py b/otbtf/examples/tensorflow_v2x/training.py
new file mode 100644
index 00000000..cb434159
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/training.py
@@ -0,0 +1,13 @@
+from otbtf.model import ModelBase
+
+# Model creation=
+class MyModel(ModelBase):
+ pass
+ # TODO
+
+
+# Training
+model = MyModel()
+model.fit()
+
+# TODO: show with and without strategy
diff --git a/otbtf/model.py b/otbtf/model.py
new file mode 100644
index 00000000..cfcd1b5b
--- /dev/null
+++ b/otbtf/model.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+""" Base class for models"""
+import abc
+import tensorflow as tf
+from tensorflow import keras
+from otbtf.utils import _is_chief
+
+PADS = [16, 32, 64, 96, 128, 256]
+
+def padded_tensor_name(tensor_name, pad):
+ """
+ A name for the padded tensor
+ :param tensor_name: tensor name
+ :param pad: pad value
+ :return: name
+ """
+ return "{}_pad{}".format(tensor_name, pad)
+
+
+def normalize(key, placeholder):
+ """
+ Normalize an input placeholder, knowing its key
+ :param key: placeholder key
+ :param placeholder: placeholder
+ :return: normalized placeholder
+ """
+ if key == 'pan':
+ return placeholder * (1 / 10000)
+ elif key == 'xs':
+ return placeholder * (1 / 10000)
+ elif key == "tt":
+ return placeholder
+ else:
+ return placeholder
+
+
+class ModelBase(abc.ABC):
+ """
+ Base class for all models
+ """
+
+ @abc.abstractmethod
+ def __init__(self, dataset_input_keys, model_output_keys, dataset_shapes, target_cropping=None):
+ """
+ Model base class
+
+ :param dataset_input_keys: list of dataset keys used for the training
+ :param model_output_keys: list of the model outputs keys
+ :param dataset_shapes: a dict() of shapes
+ :param target_cropping: Optional. Number of pixels to be removed on each side of the target
+ """
+ self.dataset_input_keys = dataset_input_keys
+ self.model_output_keys = model_output_keys
+ self.dataset_shapes = dataset_shapes
+ self.model = None
+ self.target_cropping = target_cropping
+
+ def __getattr__(self, name):
+ """This method is called when the default attribute access fails. We choose to try to access the attribute of
+ self.model. Thus, any method of keras.Model() can be used transparently, e.g. model.summary() or model.fit()"""
+ if not self.model:
+ raise Exception("model is None. Call create_network() before using it!")
+ return getattr(self.model, name)
+
+ def get_inputs(self):
+ """
+ This method returns the dict of inputs
+ """
+ # Create Keras inputs
+ model_inputs = {}
+ for key in self.dataset_input_keys:
+ shape = self.dataset_shapes[key]
+ if shape[0] is None or (len(shape) > 3): # Remove the potential batch dimension, because keras.Input() doesn't want the batch dimension
+ shape = shape[1:]
+ # Here we modify the x and y dims of >2D tensors to enable any image size at input
+ if len(shape) > 2:
+ shape[0] = None
+ shape[1] = None
+ placeholder = keras.Input(shape=shape, name=key)
+ print(key, shape)
+ model_inputs.update({key: placeholder})
+ return model_inputs
+
+ @abc.abstractmethod
+ def get_outputs(self, normalized_inputs):
+ """
+ Implementation of the model
+ :param normalized_inputs: normalized inputs
+ :return: a dict of outputs tensors of the model
+ """
+ pass
+
+ def create_network(self):
+ """
+ This method returns the Keras model. This needs to be called **inside** the strategy.scope()
+ :return: the keras model
+ """
+
+ # Get the model inputs
+ model_inputs = self.get_inputs()
+
+ # Normalize the inputs
+ normalized_inputs = {key: normalize(key, input) for key, input in model_inputs.items()}
+
+ # Build the model
+ outputs = self.get_outputs(normalized_inputs)
+
+ # Add extra outputs
+ extra_outputs = {}
+ for out_key, out_tensor in outputs.items():
+ for pad in PADS:
+ extra_output_key = padded_tensor_name(out_key, pad)
+ extra_output_name = padded_tensor_name(out_tensor._keras_history.layer.name, pad)
+ extra_output = tf.keras.layers.Cropping2D(cropping=pad, name=extra_output_name)(out_tensor)
+ extra_outputs[extra_output_key] = extra_output
+ outputs.update(extra_outputs)
+
+ # Return the keras model
+ self.model = keras.Model(inputs=model_inputs, outputs=outputs, name=self.__class__.__name__)
+
+ def summary(self, strategy=None):
+ """
+ Wraps the summary printing of the model. When multiworker strategy, only prints if the worker is chief
+ """
+ if not strategy or _is_chief(strategy):
+ self.model.summary(line_length=150)
+
+ def plot(self, output_path, strategy=None):
+ """
+ Enables to save a figure representing the architecture of the network.
+ //!\\ only works if create_network() has been called beforehand
+ Needs pydot and graphviz to work (`pip install pydot` and https://graphviz.gitlab.io/download/)
+ """
+ # When multiworker strategy, only plot if the worker is chief
+ if not strategy or _is_chief(strategy):
+ # Build a simplified model, without normalization nor extra outputs.
+ # This model is only used for plotting the architecture thanks to `keras.utils.plot_model`
+ inputs = self.get_inputs() # inputs without normalization
+ outputs = self.get_outputs(inputs) # raw model outputs
+ model_simplified = keras.Model(inputs=inputs, outputs=outputs, name=self.__class__.__name__ + '_simplified')
+ keras.utils.plot_model(model_simplified, output_path)
--
GitLab
From 9d09ee27c5cdf34e2d922b9739d07b1ede4bb05b Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Mon, 18 Jul 2022 14:06:01 +0200
Subject: [PATCH 06/77] ENH: remove batch dim when creating TFRecords
---
otbtf/model.py | 2 +-
otbtf/tfrecords.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index cfcd1b5b..2f7cc1f2 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -70,7 +70,7 @@ class ModelBase(abc.ABC):
model_inputs = {}
for key in self.dataset_input_keys:
shape = self.dataset_shapes[key]
- if shape[0] is None or (len(shape) > 3): # Remove the potential batch dimension, because keras.Input() doesn't want the batch dimension
+ if shape[0] is None or (len(shape) > 3): # for backward comp (OTBTF<3.2.2), remove the potential batch dim
shape = shape[1:]
# Here we modify the x and y dims of >2D tensors to enable any image size at input
if len(shape) > 2:
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index b2aae0b2..3f741bc0 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -70,7 +70,7 @@ class TFRecords:
if not drop_remainder and dataset.size % n_samples_per_shard > 0:
nb_shards += 1
- output_shapes = {key: (None,) + output_shape for key, output_shape in dataset.output_shapes.items()}
+ output_shapes = {key: output_shape for key, output_shape in dataset.output_shapes.items()}
self.save(output_shapes, self.output_shape_file)
output_types = {key: output_type.name for key, output_type in dataset.output_types.items()}
--
GitLab
From 5bcc22cda33db7433d66f5d63ae6d9f6a4bc07c2 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 11:47:24 +0200
Subject: [PATCH 07/77] COMP: use SuperBuild GDAL
---
Dockerfile | 4 ++--
app/otbPatchesSelection.cxx | 11 ++++++-----
tools/docker/build-deps-cli.txt | 7 -------
tools/docker/build-flags-otb.txt | 4 ++--
4 files changed, 10 insertions(+), 16 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index da634cea..990c55f5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,7 +85,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
### OTB
ARG GUI=false
-ARG OTB=7.4.0
+ARG OTB=8.0.1
ARG OTBTESTS=false
RUN mkdir /src/otb
@@ -149,7 +149,7 @@ COPY --from=builder /src /src
# System-wide ENV
ENV PATH="/opt/otbtf/bin:$PATH"
ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
-ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
+ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
# Default user, directory and command (bash is the entrypoint when using 'docker create')
diff --git a/app/otbPatchesSelection.cxx b/app/otbPatchesSelection.cxx
index 68d76221..73b0b0de 100644
--- a/app/otbPatchesSelection.cxx
+++ b/app/otbPatchesSelection.cxx
@@ -35,6 +35,12 @@
#include <random>
#include <limits>
+namespace otb
+{
+
+namespace Wrapper
+{
+
// Functor to retrieve nodata
template<class TPixel, class OutputPixel>
class IsNoData
@@ -62,11 +68,6 @@ private:
typename TPixel::ValueType m_NoDataValue;
};
-namespace otb
-{
-
-namespace Wrapper
-{
class PatchesSelection : public Application
{
diff --git a/tools/docker/build-deps-cli.txt b/tools/docker/build-deps-cli.txt
index 5d699cb1..ffd72911 100644
--- a/tools/docker/build-deps-cli.txt
+++ b/tools/docker/build-deps-cli.txt
@@ -25,8 +25,6 @@ wget
zip
bison
-gdal-bin
-python3-gdal
libboost-date-time-dev
libboost-filesystem-dev
libboost-graph-dev
@@ -36,8 +34,6 @@ libboost-thread-dev
libcurl4-gnutls-dev
libexpat1-dev
libfftw3-dev
-libgdal-dev
-libgeotiff-dev
libgsl-dev
libinsighttoolkit4-dev
libkml-dev
@@ -45,9 +41,6 @@ libmuparser-dev
libmuparserx-dev
libopencv-core-dev
libopencv-ml-dev
-libopenthreads-dev
-libossim-dev
-libpng-dev
libsvm-dev
libtinyxml-dev
zlib1g-dev
diff --git a/tools/docker/build-flags-otb.txt b/tools/docker/build-flags-otb.txt
index 2c3e0fea..56b0434c 100644
--- a/tools/docker/build-flags-otb.txt
+++ b/tools/docker/build-flags-otb.txt
@@ -3,9 +3,9 @@
-DUSE_SYSTEM_EXPAT=ON
-DUSE_SYSTEM_FFTW=ON
-DUSE_SYSTEM_FREETYPE=ON
--DUSE_SYSTEM_GDAL=ON
+-DUSE_SYSTEM_GDAL=OFF
-DUSE_SYSTEM_GEOS=ON
--DUSE_SYSTEM_GEOTIFF=ON
+-DUSE_SYSTEM_GEOTIFF=OFF
-DUSE_SYSTEM_GLEW=ON
-DUSE_SYSTEM_GLFW=ON
-DUSE_SYSTEM_GLUT=ON
--
GitLab
From 02a067011bff1d8febc130a73c6fa2a36a7f955c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 13:25:20 +0200
Subject: [PATCH 08/77] DOC: a bit of doc
---
otbtf/tfrecords.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 3f741bc0..b19c1ce5 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -153,8 +153,16 @@ class TFRecords:
False is advisable when evaluating metrics so that all samples are used
:param shuffle_buffer_size: if None, shuffle is not used. Else, blocks of shuffle_buffer_size
elements are shuffled using uniform random.
- :param preprocessing_fn: Optional. A preprocessing function that takes input, target as args and returns
- a tuple (input_preprocessed, target_preprocessed)
+ :param preprocessing_fn: Optional. A preprocessing function that takes (input, target) as args and returns
+ a tuple (input_preprocessed, target_preprocessed). Typically, target_preprocessed
+ must be computed accordingly to (1) what the model outputs and (2) what training loss
+ needs. For instance, for a classification problem, the model will likely output the
+ softmax, or activation neurons, for each class, and the cross entropy loss requires
+ labels in one hot encoding. In this case, the preprocessing_fn has to transform the
+ labels values (integer ranging from [0, n_classes]) in one hot encoding (vector of 0
+ and 1 of length n_classes). The preprocessing_fn should not implement such things as
+ radiometric transformations from input to input_preprocessed, because those are
+ performed inside the model itself (see `model.normalize()`).
:param kwargs: some keywords arguments for preprocessing_fn
"""
options = tf.data.Options()
--
GitLab
From 699bb1582f795d60ebdc1c32b71bbbf8a215f19d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 13:54:41 +0200
Subject: [PATCH 09/77] WIP: example
---
.../tensorflow_v2x/fcnn/create_tfrecords.py | 0
.../examples/tensorflow_v2x/fcnn/sampling.py | 0
.../examples/tensorflow_v2x/fcnn/training.py | 159 ++++++++++++++++++
otbtf/examples/tensorflow_v2x/training.py | 13 --
4 files changed, 159 insertions(+), 13 deletions(-)
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/sampling.py
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/training.py
delete mode 100644 otbtf/examples/tensorflow_v2x/training.py
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
new file mode 100644
index 00000000..e69de29b
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/sampling.py b/otbtf/examples/tensorflow_v2x/fcnn/sampling.py
new file mode 100644
index 00000000..e69de29b
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/training.py b/otbtf/examples/tensorflow_v2x/fcnn/training.py
new file mode 100644
index 00000000..fdcc003c
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/fcnn/training.py
@@ -0,0 +1,159 @@
+from otbtf.model import ModelBase
+import tensorflow as tf
+import tensorflow.keras.layers as layers
+import argparse
+import pathlib
+from otbtf import TFRecords
+
+# Application parameters
+parser = argparse.ArgumentParser(description="Train a FCNN model")
+parser.add_argument("-p", "--patches_dir", required=True, help="Directory of TFRecords dirs: train, valid(, test)")
+parser.add_argument("-m", "--model_dir", required=True, help="Path to save model")
+parser.add_argument("-e", "--number_epochs", type=int, default=100, help="Number of epochs")
+parser.add_argument("-b", "--batch_size", type=int, default=8, help="Batch size")
+parser.add_argument("-r", "--learning_rate", type=float, default=0.00001, help="Learning rate")
+
+
+class FCNNModel(ModelBase):
+ """
+ A Simple Fully Convolutional model
+ """
+
+ def get_outputs(self, normalized_inputs):
+ """
+ This small model produces an output which has the same physical spacing as the input.
+ The model generates [1 x 1 x N_CLASSES] output pixel for [32 x 32 x <nb channels>] input pixels.
+
+ #Conv | depth | kernel size | out. size (in x/y dims)
+ ------ | ----------- | ----------- | ------------------------
+ 1 | N_NEURONS | 5 | 28
+ 2 | 2*N_NEURONS | 4 | 25
+ 3 | 4*N_NEURONS | 4 | 22
+ 4 | 4*N_NEURONS | 4 | 19
+ 5 | 4*N_NEURONS | 4 | 16
+ 6 | 4*N_NEURONS | 4 | 13
+ 7 | 4*N_NEURONS | 4 | 10
+ 8 | 4*N_NEURONS | 4 | 7
+ 9 | 4*N_NEURONS | 4 | 4
+ 10 | 4*N_NEURONS | 4 | 1
+ 11 | N_CLASSES | 1 | 1
+
+ :return: activation values
+ """
+
+ # Model constants
+ N_NEURONS = 16
+ N_CLASSES = 6
+
+ # Convolutions
+ depths = [N_NEURONS, 2 * N_NEURONS] + 8 * [4 * N_NEURONS]
+ ksizes = [5] + 9 * [4]
+ net = normalized_inputs["input_xs"]
+ for i, (d, k) in enumerate(zip(depths, ksizes)):
+ conv = layers.Conv2D(filters=d, kernel_size=k, activation="relu", name=f"conv{i}")
+ bn = tf.keras.layers.BatchNormalization()
+ net = bn(conv(net))
+
+ # Classifier
+ lastconv = layers.Conv2D(filters=N_CLASSES, kernel_size=1, name="conv_class")
+ return lastconv(net) # out size: 1x1xN_CLASSES
+
+
+def preprocessing_fn(inputs, targets):
+ """
+ Preprocessing function for the training dataset
+
+ This function returns an output tuple (processed_inputs, processed_targets) ready to feed the model
+ """
+ return inputs, {"label": tf.one_hot(tf.squeeze(targets["label"], axis=-1), depth=2)}
+
+if __name__ == "__main__":
+ params = parser.parse_args()
+
+ # Patches directories must contain 'train' and 'valid' dirs, 'test' is not required
+ patches = pathlib.Path(params.patches_dir)
+ ds_test = None
+ for d in patches.iterdir():
+ if "train" in d.name.lower() :
+ ds_train = TFRecords(str(d)).read(shuffle_buffer_size=1000)
+ elif "valid" in d.name.lower():
+ ds_valid = TFRecords(str(d)).read()
+ elif "test" in d.name.lower():
+ ds_test = TFRecords(str(d)).read()
+
+ strategy = tf.distribute.MirroredStrategy()
+ with strategy.scope():
+
+ # ### Input PXS image
+ # in_pxs = keras.Input(shape=[None, None, 4], name="pxs") # 4 is the number of channels in the pxs
+ #
+ # ### Create network
+ # # Normalize input roughly in the [0, 1] range
+ # if div:
+ # in_pxs = layers.Lambda(lambda t: t / div)(in_pxs)
+ #
+ # if model_type == "cnn":
+ # pseudo_proba = cnn_patch32_out6m(in_pxs)
+ # elif model_type == "fcnn":
+ # pseudo_proba = fcnn_fullres(in_pxs)
+ # elif model_type == "resnet":
+ # pseudo_proba = resnet_model1(in_pxs, sliced=True, patch_size=patch_size)
+ # else:
+ # raise NotImplementedError
+ #
+ # ### Callbacks
+ # callbacks = []
+ # # TensorBoard
+ # if logs:
+ # callbacks.append(keras.callbacks.TensorBoard(log_dir=logs + f"/{expe_name}"))
+ # # Save best checkpoint
+ # if save_best:
+ # ckpt_name = model_dir + f"/best_{save_best}.ckpt"
+ # callbacks.append(keras.callbacks.ModelCheckpoint(ckpt_name, save_best_only=True, monitor=save_best))
+ # # Rate scheduler with exponential decay after n epochs
+ # if schedule:
+ # def scheduler(epoch, lr, after=2):
+ # return lr if epoch < after else lr * tf.math.exp(-0.1)
+ # callbacks.append(tf.keras.callbacks.LearningRateScheduler(scheduler))
+ # # Early stop if loss does not increase after n consecutive epochs
+ # if early_stop:
+ # callbacks.append(keras.callbacks.EarlyStopping(monitor="loss", mode="min", patience=5))
+ #
+ # ### Metrics
+ # metrics = [
+ # tf.keras.metrics.BinaryAccuracy(),
+ # keras.metrics.Precision(),
+ # keras.metrics.Recall()
+ # ]
+ # # Metrics for multiclass, not sure about this workaround https://github.com/tensorflow/addons/issues/746
+ # # import tensorflow_addons as tfa
+ # # tfa.metrics.F1Score(num_classes=1, name="f1_score", average='micro', threshold=0.5)
+ # # tfa.metrics.CohenKappa(num_classes=2, name="cohenn_kappa")
+ #
+ # ### Compile the model
+ # model = keras.Model(inputs={"pxs": in_pxs}, outputs={"label": pseudo_proba})
+ # model.compile(
+ # loss=keras.losses.BinaryCrossentropy(),
+ # optimizer=keras.optimizers.Adam(learning_rate=rate),
+ # metrics=metrics
+ # )
+ # # Print network
+ # model.summary(line_length=120)
+ # Path(model_dir).mkdir(exist_ok=True)
+ # keras.utils.plot_model(model, model_dir + "/model.png")
+ #
+ # ### Train
+ # model.fit(
+ # ds_train,
+ # epochs=epochs,
+ # validation_data=ds_valid,
+ # callbacks=callbacks,
+ # verbose=1
+ # )
+ # ### TODO : Test ?
+ # #if ds_test is not None:
+ # # model.evaluate(ds_test, batch_size=batch)
+ # # Save full model
+ # model.save(model_dir)
+ #
+ # return 0
diff --git a/otbtf/examples/tensorflow_v2x/training.py b/otbtf/examples/tensorflow_v2x/training.py
deleted file mode 100644
index cb434159..00000000
--- a/otbtf/examples/tensorflow_v2x/training.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from otbtf.model import ModelBase
-
-# Model creation=
-class MyModel(ModelBase):
- pass
- # TODO
-
-
-# Training
-model = MyModel()
-model.fit()
-
-# TODO: show with and without strategy
--
GitLab
From 9a2b87684e0764c620eec17484ea906708db342a Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Tue, 19 Jul 2022 14:58:02 +0200
Subject: [PATCH 10/77] ENH: change 'pad' to 'crop' ENH: add the
inference_cropping as argument of model __init__ instead of static variable
ENH: add the normalize function as argument of model __init__
---
.../examples/tensorflow_v2x/fcnn/training.py | 20 ++++++
otbtf/model.py | 66 +++++++++----------
2 files changed, 50 insertions(+), 36 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/training.py b/otbtf/examples/tensorflow_v2x/fcnn/training.py
index fdcc003c..cc693355 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/training.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/training.py
@@ -67,6 +67,26 @@ def preprocessing_fn(inputs, targets):
"""
return inputs, {"label": tf.one_hot(tf.squeeze(targets["label"], axis=-1), depth=2)}
+
+# TODO: bien expliquer la différence entre preprocessing_fn (utilisé pour transformer les targets des TFRecords)
+# et normalize_fn (utilisé pour pouvoir réaliser l'inférence "directement" sur les images)
+def normalize(key, placeholder):
+ """
+ Normalize an input placeholder, knowing its key
+ :param key: placeholder key
+ :param placeholder: placeholder
+ :return: normalized placeholder
+ """
+ if key == 'pan':
+ return placeholder * (1 / 10000)
+ elif key == 'xs':
+ return placeholder * (1 / 10000)
+ elif key == "tt":
+ return placeholder
+ else:
+ return placeholder
+
+
if __name__ == "__main__":
params = parser.parse_args()
diff --git a/otbtf/model.py b/otbtf/model.py
index 2f7cc1f2..85424986 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -5,34 +5,6 @@ import tensorflow as tf
from tensorflow import keras
from otbtf.utils import _is_chief
-PADS = [16, 32, 64, 96, 128, 256]
-
-def padded_tensor_name(tensor_name, pad):
- """
- A name for the padded tensor
- :param tensor_name: tensor name
- :param pad: pad value
- :return: name
- """
- return "{}_pad{}".format(tensor_name, pad)
-
-
-def normalize(key, placeholder):
- """
- Normalize an input placeholder, knowing its key
- :param key: placeholder key
- :param placeholder: placeholder
- :return: normalized placeholder
- """
- if key == 'pan':
- return placeholder * (1 / 10000)
- elif key == 'xs':
- return placeholder * (1 / 10000)
- elif key == "tt":
- return placeholder
- else:
- return placeholder
-
class ModelBase(abc.ABC):
"""
@@ -40,20 +12,29 @@ class ModelBase(abc.ABC):
"""
@abc.abstractmethod
- def __init__(self, dataset_input_keys, model_output_keys, dataset_shapes, target_cropping=None):
+ def __init__(self, dataset_input_keys, model_output_keys, dataset_shapes, target_cropping=None,
+ inference_cropping=[16, 32, 64, 96, 128], normalize_fn=None):
"""
Model base class
:param dataset_input_keys: list of dataset keys used for the training
:param model_output_keys: list of the model outputs keys
:param dataset_shapes: a dict() of shapes
- :param target_cropping: Optional. Number of pixels to be removed on each side of the target
+ :param target_cropping: Optional. Number of pixels to be removed on each side of the target. This is used when
+ training the model and can mitigate the effects of convolution
+ :param inference_cropping: list of number of pixels to be removed on each side of the output during inference.
+ This list creates some additional outputs in the model, not used during training,
+ only during inference. Default [16, 32, 64, 96, 128]
+ :param normalize_fn: a normalization function that can be added inside the Keras model. The function must accept
+ 2 arguments `key` and `tensor`. Optional
"""
self.dataset_input_keys = dataset_input_keys
self.model_output_keys = model_output_keys
self.dataset_shapes = dataset_shapes
self.model = None
self.target_cropping = target_cropping
+ self.inference_cropping = inference_cropping
+ self.normalize_fn = normalize_fn
def __getattr__(self, name):
"""This method is called when the default attribute access fails. We choose to try to access the attribute of
@@ -100,18 +81,21 @@ class ModelBase(abc.ABC):
model_inputs = self.get_inputs()
# Normalize the inputs
- normalized_inputs = {key: normalize(key, input) for key, input in model_inputs.items()}
+ if self.normalize_fn:
+ normalized_inputs = {key: self.normalize_fn(key, input) for key, input in model_inputs.items()}
+ else:
+ normalized_inputs = model_inputs
# Build the model
outputs = self.get_outputs(normalized_inputs)
- # Add extra outputs
+ # Add extra outputs for inference
extra_outputs = {}
for out_key, out_tensor in outputs.items():
- for pad in PADS:
- extra_output_key = padded_tensor_name(out_key, pad)
- extra_output_name = padded_tensor_name(out_tensor._keras_history.layer.name, pad)
- extra_output = tf.keras.layers.Cropping2D(cropping=pad, name=extra_output_name)(out_tensor)
+ for crop in self.inference_cropping:
+ extra_output_key = cropped_tensor_name(out_key, crop)
+ extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
+ extra_output = tf.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
extra_outputs[extra_output_key] = extra_output
outputs.update(extra_outputs)
@@ -139,3 +123,13 @@ class ModelBase(abc.ABC):
outputs = self.get_outputs(inputs) # raw model outputs
model_simplified = keras.Model(inputs=inputs, outputs=outputs, name=self.__class__.__name__ + '_simplified')
keras.utils.plot_model(model_simplified, output_path)
+
+
+def cropped_tensor_name(tensor_name, crop):
+ """
+ A name for the padded tensor
+ :param tensor_name: tensor name
+ :param pad: pad value
+ :return: name
+ """
+ return "{}_crop{}".format(tensor_name, crop)
--
GitLab
From 8ca72acf97c4734e2863a2d07bdfae6b1c6eff8e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 15:02:58 +0200
Subject: [PATCH 11/77] WIP: refactoring model class
---
.../examples/tensorflow_v2x/fcnn/training.py | 148 ++++++------------
otbtf/model.py | 3 +
otbtf/tfrecords.py | 2 +-
3 files changed, 50 insertions(+), 103 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/training.py b/otbtf/examples/tensorflow_v2x/fcnn/training.py
index fdcc003c..4addc11a 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/training.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/training.py
@@ -4,6 +4,7 @@ import tensorflow.keras.layers as layers
import argparse
import pathlib
from otbtf import TFRecords
+import os
# Application parameters
parser = argparse.ArgumentParser(description="Train a FCNN model")
@@ -12,11 +13,12 @@ parser.add_argument("-m", "--model_dir", required=True, help="Path to save model
parser.add_argument("-e", "--number_epochs", type=int, default=100, help="Number of epochs")
parser.add_argument("-b", "--batch_size", type=int, default=8, help="Batch size")
parser.add_argument("-r", "--learning_rate", type=float, default=0.00001, help="Learning rate")
-
+parser.add_argument('--dataset_mode', default='tfrecords', const='tfrecords', nargs='?',
+ choices=['tfrecords', 'patches_images'])
class FCNNModel(ModelBase):
"""
- A Simple Fully Convolutional model
+ A Simple Fully Convolutional U-Net like model
"""
def get_outputs(self, normalized_inputs):
@@ -24,49 +26,42 @@ class FCNNModel(ModelBase):
This small model produces an output which has the same physical spacing as the input.
The model generates [1 x 1 x N_CLASSES] output pixel for [32 x 32 x <nb channels>] input pixels.
- #Conv | depth | kernel size | out. size (in x/y dims)
- ------ | ----------- | ----------- | ------------------------
- 1 | N_NEURONS | 5 | 28
- 2 | 2*N_NEURONS | 4 | 25
- 3 | 4*N_NEURONS | 4 | 22
- 4 | 4*N_NEURONS | 4 | 19
- 5 | 4*N_NEURONS | 4 | 16
- 6 | 4*N_NEURONS | 4 | 13
- 7 | 4*N_NEURONS | 4 | 10
- 8 | 4*N_NEURONS | 4 | 7
- 9 | 4*N_NEURONS | 4 | 4
- 10 | 4*N_NEURONS | 4 | 1
- 11 | N_CLASSES | 1 | 1
-
+ :param normalized_inputs: dict of normalized inputs`
:return: activation values
"""
# Model constants
- N_NEURONS = 16
N_CLASSES = 6
- # Convolutions
- depths = [N_NEURONS, 2 * N_NEURONS] + 8 * [4 * N_NEURONS]
- ksizes = [5] + 9 * [4]
+ # Model input
net = normalized_inputs["input_xs"]
- for i, (d, k) in enumerate(zip(depths, ksizes)):
- conv = layers.Conv2D(filters=d, kernel_size=k, activation="relu", name=f"conv{i}")
- bn = tf.keras.layers.BatchNormalization()
- net = bn(conv(net))
- # Classifier
- lastconv = layers.Conv2D(filters=N_CLASSES, kernel_size=1, name="conv_class")
- return lastconv(net) # out size: 1x1xN_CLASSES
+ # Encoder
+ convs_depth = {"conv1": 16, "conv2": 32, "conv3": 64, "conv4": 64}
+ for name, depth in convs_depth.items():
+ conv = layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)
+ net = conv(net)
+
+ # Decoder
+ tconvs_depths = {"tconv1": 64, "tconv2": 32, "tconv3": 16, "tconv4": N_CLASSES}
+ for name, depth in tconvs_depths.items():
+ tconv = layers.Conv2DTranspose(filters=depth, kernel_size=3, activation="relu", name=name)
+ net = tconv(net)
+
+ return net
def preprocessing_fn(inputs, targets):
"""
Preprocessing function for the training dataset
- This function returns an output tuple (processed_inputs, processed_targets) ready to feed the model
+ :param inputs: dict for inputs
+ :param targets: dict for targets
+ :return: an output tuple (processed_inputs, processed_targets) ready to feed the model
"""
return inputs, {"label": tf.one_hot(tf.squeeze(targets["label"], axis=-1), depth=2)}
+
if __name__ == "__main__":
params = parser.parse_args()
@@ -74,7 +69,7 @@ if __name__ == "__main__":
patches = pathlib.Path(params.patches_dir)
ds_test = None
for d in patches.iterdir():
- if "train" in d.name.lower() :
+ if "train" in d.name.lower():
ds_train = TFRecords(str(d)).read(shuffle_buffer_size=1000)
elif "valid" in d.name.lower():
ds_valid = TFRecords(str(d)).read()
@@ -84,76 +79,25 @@ if __name__ == "__main__":
strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
- # ### Input PXS image
- # in_pxs = keras.Input(shape=[None, None, 4], name="pxs") # 4 is the number of channels in the pxs
- #
- # ### Create network
- # # Normalize input roughly in the [0, 1] range
- # if div:
- # in_pxs = layers.Lambda(lambda t: t / div)(in_pxs)
- #
- # if model_type == "cnn":
- # pseudo_proba = cnn_patch32_out6m(in_pxs)
- # elif model_type == "fcnn":
- # pseudo_proba = fcnn_fullres(in_pxs)
- # elif model_type == "resnet":
- # pseudo_proba = resnet_model1(in_pxs, sliced=True, patch_size=patch_size)
- # else:
- # raise NotImplementedError
- #
- # ### Callbacks
- # callbacks = []
- # # TensorBoard
- # if logs:
- # callbacks.append(keras.callbacks.TensorBoard(log_dir=logs + f"/{expe_name}"))
- # # Save best checkpoint
- # if save_best:
- # ckpt_name = model_dir + f"/best_{save_best}.ckpt"
- # callbacks.append(keras.callbacks.ModelCheckpoint(ckpt_name, save_best_only=True, monitor=save_best))
- # # Rate scheduler with exponential decay after n epochs
- # if schedule:
- # def scheduler(epoch, lr, after=2):
- # return lr if epoch < after else lr * tf.math.exp(-0.1)
- # callbacks.append(tf.keras.callbacks.LearningRateScheduler(scheduler))
- # # Early stop if loss does not increase after n consecutive epochs
- # if early_stop:
- # callbacks.append(keras.callbacks.EarlyStopping(monitor="loss", mode="min", patience=5))
- #
- # ### Metrics
- # metrics = [
- # tf.keras.metrics.BinaryAccuracy(),
- # keras.metrics.Precision(),
- # keras.metrics.Recall()
- # ]
- # # Metrics for multiclass, not sure about this workaround https://github.com/tensorflow/addons/issues/746
- # # import tensorflow_addons as tfa
- # # tfa.metrics.F1Score(num_classes=1, name="f1_score", average='micro', threshold=0.5)
- # # tfa.metrics.CohenKappa(num_classes=2, name="cohenn_kappa")
- #
- # ### Compile the model
- # model = keras.Model(inputs={"pxs": in_pxs}, outputs={"label": pseudo_proba})
- # model.compile(
- # loss=keras.losses.BinaryCrossentropy(),
- # optimizer=keras.optimizers.Adam(learning_rate=rate),
- # metrics=metrics
- # )
- # # Print network
- # model.summary(line_length=120)
- # Path(model_dir).mkdir(exist_ok=True)
- # keras.utils.plot_model(model, model_dir + "/model.png")
- #
- # ### Train
- # model.fit(
- # ds_train,
- # epochs=epochs,
- # validation_data=ds_valid,
- # callbacks=callbacks,
- # verbose=1
- # )
- # ### TODO : Test ?
- # #if ds_test is not None:
- # # model.evaluate(ds_test, batch_size=batch)
- # # Save full model
- # model.save(model_dir)
- #
- # return 0
+ # Create and compile the model
+ model = FCNNModel()
+ model.compile(loss=tf.keras.losses.BinaryCrossentropy(),
+ optimizer=tf.keras.optimizers.Adam(learning_rate=params.learning_rate),
+ metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
+
+ # Summarize the model (in CLI)
+ model.summary(line_length=120)
+
+ # Summarize the model (in figure.png)
+ pathlib.Path(params.model_dir).mkdir(exist_ok=True)
+ tf.keras.utils.plot_model(model, os.path.join(params.model_dir, "figure.png"))
+
+ # Train
+ model.fit(ds_train, epochs=params.number_epochs, validation_data=ds_valid)
+
+ # Evaluate against test data
+ if ds_test is not None:
+ model.evaluate(ds_test, batch_size=params.batch_size)
+
+ # Save trained model
+ model.save(params.model_dir)
diff --git a/otbtf/model.py b/otbtf/model.py
index 2f7cc1f2..4ec44aa5 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -7,6 +7,7 @@ from otbtf.utils import _is_chief
PADS = [16, 32, 64, 96, 128, 256]
+
def padded_tensor_name(tensor_name, pad):
"""
A name for the padded tensor
@@ -131,6 +132,8 @@ class ModelBase(abc.ABC):
//!\\ only works if create_network() has been called beforehand
Needs pydot and graphviz to work (`pip install pydot` and https://graphviz.gitlab.io/download/)
"""
+ assert self.model, "Plot() only works if create_network() has been called beforehand"
+
# When multiworker strategy, only plot if the worker is chief
if not strategy or _is_chief(strategy):
# Build a simplified model, without normalization nor extra outputs.
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index b19c1ce5..28a72de4 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -162,7 +162,7 @@ class TFRecords:
labels values (integer ranging from [0, n_classes]) in one hot encoding (vector of 0
and 1 of length n_classes). The preprocessing_fn should not implement such things as
radiometric transformations from input to input_preprocessed, because those are
- performed inside the model itself (see `model.normalize()`).
+ performed inside the model itself (see `otbtf.ModelBase.normalize()`).
:param kwargs: some keywords arguments for preprocessing_fn
"""
options = tf.data.Options()
--
GitLab
From aa44aa935e6b619f769fe181c3e59f5761e9dec0 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Tue, 19 Jul 2022 15:29:29 +0200
Subject: [PATCH 12/77] ENH: create the network if non existing
---
otbtf/model.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 85424986..5f239975 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-
""" Base class for models"""
import abc
+import logging
+
import tensorflow as tf
from tensorflow import keras
from otbtf.utils import _is_chief
@@ -40,7 +42,11 @@ class ModelBase(abc.ABC):
"""This method is called when the default attribute access fails. We choose to try to access the attribute of
self.model. Thus, any method of keras.Model() can be used transparently, e.g. model.summary() or model.fit()"""
if not self.model:
- raise Exception("model is None. Call create_network() before using it!")
+ logging.warning("model is None. You should call `create_network()` before using it!")
+ logging.warning("Creating the neural network. Note that training could fail if using keras distribution "
+ "strategy such as MirroredStrategy. Best practice is to call `create_network()` inside "
+ "`with strategy.scope():`")
+ self.create_network()
return getattr(self.model, name)
def get_inputs(self):
--
GitLab
From a1d36e60caf415dde4df79c34a54c548c9b355f2 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Tue, 19 Jul 2022 15:49:37 +0200
Subject: [PATCH 13/77] FIX: get the key of the result of the normalization
function
---
otbtf/model.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 288e1348..80d459a9 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -15,7 +15,7 @@ class ModelBase(abc.ABC):
@abc.abstractmethod
def __init__(self, dataset_input_keys, model_output_keys, dataset_shapes, target_cropping=None,
- inference_cropping=[16, 32, 64, 96, 128], normalize_fn=None):
+ inference_cropping=None, normalize_fn=None):
"""
Model base class
@@ -27,14 +27,16 @@ class ModelBase(abc.ABC):
:param inference_cropping: list of number of pixels to be removed on each side of the output during inference.
This list creates some additional outputs in the model, not used during training,
only during inference. Default [16, 32, 64, 96, 128]
- :param normalize_fn: a normalization function that can be added inside the Keras model. The function must accept
- 2 arguments `key` and `tensor`. Optional
+ :param normalize_fn: a normalization function that can be added inside the Keras model. This function takes a
+ dict of inputs and returns a dict of normalized inputs. Optional
"""
self.dataset_input_keys = dataset_input_keys
self.model_output_keys = model_output_keys
self.dataset_shapes = dataset_shapes
self.model = None
self.target_cropping = target_cropping
+ if inference_cropping is None:
+ inference_cropping = [16, 32, 64, 96, 128]
self.inference_cropping = inference_cropping
self.normalize_fn = normalize_fn
@@ -51,7 +53,7 @@ class ModelBase(abc.ABC):
def get_inputs(self):
"""
- This method returns the dict of inputs
+ This method returns the dict of keras.Input
"""
# Create Keras inputs
model_inputs = {}
@@ -69,10 +71,10 @@ class ModelBase(abc.ABC):
return model_inputs
@abc.abstractmethod
- def get_outputs(self, normalized_inputs):
+ def get_outputs(self, inputs):
"""
Implementation of the model
- :param normalized_inputs: normalized inputs
+ :param inputs: inputs, either keras.Input or normalized_inputs
:return: a dict of outputs tensors of the model
"""
pass
@@ -86,8 +88,8 @@ class ModelBase(abc.ABC):
# Get the model inputs
model_inputs = self.get_inputs()
- # Normalize the inputs
- normalized_inputs = {key: self.normalize_fn[key](inp) for key, inp in
+ # Normalize the inputs. If some input keys are not handled by normalized_fn, these inputs are not normalized
+ normalized_inputs = {key: self.normalize_fn(inp)[key] if key in self.normalize_fn(inp) else inp for key, inp in
model_inputs.items()} if self.normalize_fn else model_inputs
# Build the model
--
GitLab
From 626bc859af2e49138e8c41e437ed5d07122f5994 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 16:19:18 +0200
Subject: [PATCH 14/77] WIP: example
---
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 81 +++++++++++++++++++
.../examples/tensorflow_v2x/fcnn/training.py | 24 ++----
2 files changed, 89 insertions(+), 16 deletions(-)
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/helper.py
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
new file mode 100644
index 00000000..5e0f7c37
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -0,0 +1,81 @@
+from otbtf import TFRecords
+import pathlib
+
+
+def get_datasets(dataset_format, dataset_dir, batch_size, target_keys):
+ """
+ Function to use either TFRecords or patches images
+
+ :param dataset_format: dataset format. Either ("tfrecords" or "patches_images")
+ :param dataset_dir": dataset root directory. Must contain 2 (3) subdirectories (train, valid(, test))
+ """
+ assert dataset_format == "tfrecords" or dataset_format == "patches_images"
+
+ # Patches directories must contain 'train' and 'valid' dirs, 'test' is not required
+ patches = pathlib.Path(dataset_dir)
+ datasets = {}
+ for d in patches.iterdir():
+ dir = d.name
+ tag = dir.lower()
+ assert tag in ["train", "valid", "test"], "Subfolders must be named train, valid (and test)"
+ if dataset_format == "tfrecords":
+ # When the dataset format is TFRecords, we expect that the files are stored in the following way, with
+ # m, n, and k denoting respectively the number of TFRecords files in the training, validation, and test
+ # datasets:
+ #
+ # /dataset_dir
+ # /train
+ # 1.records
+ # 2.records
+ # ...
+ # m.records
+ # /valid
+ # 1.records
+ # 2.records
+ # ...
+ # n.records
+ # /test
+ # 1.records
+ # 2.records
+ # ...
+ # k.records
+ #
+ tfrecords = TFRecords(dir)
+ datasets[tag] = tfrecords.read(batch_size=batch_size, target_keys=target_keys,
+ shuffle_buffer_size=1000) if tag == "train" else tfrecords.read(
+ batch_size=batch_size, target_keys=target_keys)
+ else:
+ # When the dataset format is patches_images, we expect that the files are stored in the following way, with
+ # M, N and K denoting respectively the number of patches-images in the training, validation, and test
+ # datasets:
+ #
+ # /dataset_dir
+ # /train
+ # /image_1
+ # ..._xs.tif
+ # ..._labels.tif
+ # /image_2
+ # ..._xs.tif
+ # ..._labels.tif
+ # ...
+ # /image_M
+ # ..._xs.tif
+ # ..._labels.tif
+ # /valid
+ # /image_1
+ # ..._xs.tif
+ # ..._labels.tif
+ # ...
+ # /image_N
+ # ..._xs.tif
+ # ..._labels.tif
+ # /test
+ # /image_1
+ # ..._xs.tif
+ # ..._labels.tif
+ # ...
+ # /image_K
+ # ..._xs.tif
+ # ..._labels.tif
+ for subd in d.iterdir():
+
\ No newline at end of file
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/training.py b/otbtf/examples/tensorflow_v2x/fcnn/training.py
index 1b90b2de..2255e187 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/training.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/training.py
@@ -3,18 +3,18 @@ import tensorflow as tf
import tensorflow.keras.layers as layers
import argparse
import pathlib
-from otbtf import TFRecords
import os
+import helper
# Application parameters
parser = argparse.ArgumentParser(description="Train a FCNN model")
-parser.add_argument("-p", "--patches_dir", required=True, help="Directory of TFRecords dirs: train, valid(, test)")
-parser.add_argument("-m", "--model_dir", required=True, help="Path to save model")
-parser.add_argument("-e", "--number_epochs", type=int, default=100, help="Number of epochs")
+parser.add_argument("-p", "--dataset_dir", required=True, help="Directory of subdirs: train, valid(, test)")
+parser.add_argument("-f", "--dataset_format", default="tfrecords", const="tfrecords", nargs="?",
+ choices=["tfrecords", "patches_images"], help="Format of the dataset (TFRecords or Patches images")
parser.add_argument("-b", "--batch_size", type=int, default=8, help="Batch size")
parser.add_argument("-r", "--learning_rate", type=float, default=0.00001, help="Learning rate")
-parser.add_argument('--dataset_mode', default='tfrecords', const='tfrecords', nargs='?',
- choices=['tfrecords', 'patches_images'])
+parser.add_argument("-e", "--number_epochs", type=int, default=100, help="Number of epochs")
+parser.add_argument("-m", "--model_dir", required=True, help="Path to save model")
class FCNNModel(ModelBase):
@@ -83,16 +83,8 @@ def normalize_fn(inputs):
if __name__ == "__main__":
params = parser.parse_args()
- # Patches directories must contain 'train' and 'valid' dirs, 'test' is not required
- patches = pathlib.Path(params.patches_dir)
- ds_test = None
- for d in patches.iterdir():
- if "train" in d.name.lower():
- ds_train = TFRecords(str(d)).read(shuffle_buffer_size=1000)
- elif "valid" in d.name.lower():
- ds_valid = TFRecords(str(d)).read()
- elif "test" in d.name.lower():
- ds_test = TFRecords(str(d)).read()
+ # Get datasets
+ ds_train, ds_valid, ds_test = helper.get_datasets(params.dataset_format, params.dataset_dir)
strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
--
GitLab
From 226808ed9fc059f1de5465dbdaf97b0addd5dc86 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 16:53:36 +0200
Subject: [PATCH 15/77] WIP: example
---
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 32 ++++++++++++++-----
.../examples/tensorflow_v2x/fcnn/training.py | 2 +-
2 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index 5e0f7c37..949ce0f6 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -1,22 +1,23 @@
-from otbtf import TFRecords
+from otbtf import TFRecords, dataset
import pathlib
def get_datasets(dataset_format, dataset_dir, batch_size, target_keys):
"""
- Function to use either TFRecords or patches images
+ Function to use either TFRecords or patches images.
+ Take a look in the comments below to see how files must be stored.
:param dataset_format: dataset format. Either ("tfrecords" or "patches_images")
:param dataset_dir": dataset root directory. Must contain 2 (3) subdirectories (train, valid(, test))
"""
- assert dataset_format == "tfrecords" or dataset_format == "patches_images"
# Patches directories must contain 'train' and 'valid' dirs, 'test' is not required
patches = pathlib.Path(dataset_dir)
datasets = {}
for d in patches.iterdir():
- dir = d.name
- tag = dir.lower()
+ if not d.is_dir():
+ continue
+ tag = d.name.lower()
assert tag in ["train", "valid", "test"], "Subfolders must be named train, valid (and test)"
if dataset_format == "tfrecords":
# When the dataset format is TFRecords, we expect that the files are stored in the following way, with
@@ -40,11 +41,11 @@ def get_datasets(dataset_format, dataset_dir, batch_size, target_keys):
# ...
# k.records
#
- tfrecords = TFRecords(dir)
+ tfrecords = TFRecords(d)
datasets[tag] = tfrecords.read(batch_size=batch_size, target_keys=target_keys,
shuffle_buffer_size=1000) if tag == "train" else tfrecords.read(
batch_size=batch_size, target_keys=target_keys)
- else:
+ elif dataset_format == "patches_images":
# When the dataset format is patches_images, we expect that the files are stored in the following way, with
# M, N and K denoting respectively the number of patches-images in the training, validation, and test
# datasets:
@@ -77,5 +78,20 @@ def get_datasets(dataset_format, dataset_dir, batch_size, target_keys):
# /image_K
# ..._xs.tif
# ..._labels.tif
+ filenames_dict = {"input_xs": [],
+ "labels": []}
for subd in d.iterdir():
-
\ No newline at end of file
+ if not subd.is_dir():
+ continue
+ for filename in subd.iterdir():
+ if filename.lower().endswith("_xs.tif"):
+ filenames_dict["input_xs"].append(filename)
+ if filename.lower().endswith("_labels.tif"):
+ filenames_dict["labels"].append(filename)
+
+ # You can turn use_streaming=True to lower the memory footprint (patches are read on-the-fly on disk)
+ datasets[tag] = dataset.DatasetFromPatchesImages(filenames_dict=filenames_dict)
+ else:
+ raise ValueError("dataset_format must be \"tfrecords\" or \"patches_images\"")
+
+ return datasets
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/training.py b/otbtf/examples/tensorflow_v2x/fcnn/training.py
index 2255e187..c7d1b2e1 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/training.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/training.py
@@ -86,7 +86,7 @@ if __name__ == "__main__":
# Get datasets
ds_train, ds_valid, ds_test = helper.get_datasets(params.dataset_format, params.dataset_dir)
- strategy = tf.distribute.MirroredStrategy()
+ strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
with strategy.scope():
# Create and compile the model
--
GitLab
From ea2e42f157ca940cf6b83b37cdc5787db019ded8 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 17:02:33 +0200
Subject: [PATCH 16/77] WIP: example
---
.../tensorflow_v2x/fcnn/create_tfrecords.py | 44 +++++++++++++++++++
.../examples/tensorflow_v2x/fcnn/training.py | 2 +-
2 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
index e69de29b..c1fc3209 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
@@ -0,0 +1,44 @@
+import argparse
+from pathlib import Path
+from otbtf import DatasetFromPatchesImages
+
+# Application parameters
+parser = argparse.ArgumentParser(description="Converts patches-images into TFRecords")
+parser.add_argument("--xs", required=True, nargs="+", default=[], help="A list of patches-images for the XS image")
+parser.add_argument("--labels", required=True, nargs="+", default=[], help="A list of patches-images for the labels")
+parser.add_argument("--outdir", required=True, help="Output dir for TFRecords files")
+params = parser.parse_args()
+
+
+def check_files_order(patches, labels):
+ """
+ Here we check that the (input_xs, labels) patches are well sorted
+ """
+ assert len(patches) == len(labels)
+
+ def get_basename(n):
+ return "_".join([n.split("_")][:-1])
+
+ for p, l in zip(patches, labels):
+ assert get_basename(p) == get_basename(l)
+
+
+if __name__ == "__main__":
+
+ # Sort patches and labels
+ patches = sorted(params.patches)
+ labels = sorted(params.labels)
+
+ # Check patches and labels are correctly sorted
+ check_files_order(patches, labels)
+
+ # Create output directory
+ outdir = Path(params.outdir)
+ if not outdir.exists():
+ outdir.mkdir(exist_ok=True)
+
+ # Create dataset from the filename dict
+ dataset = DatasetFromPatchesImages(filenames_dict={"input_xs": patches, "labels": labels})
+
+ # Convert the dataset into TFRecords
+ dataset.to_tfrecords(output_dir=params.outdir)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/training.py b/otbtf/examples/tensorflow_v2x/fcnn/training.py
index c7d1b2e1..5bac52d1 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/training.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/training.py
@@ -62,7 +62,7 @@ def preprocessing_fn(inputs, targets):
:param targets: dict for targets
:return: an output tuple (processed_inputs, processed_targets)
"""
- return inputs, {"label": tf.one_hot(tf.squeeze(targets["label"], axis=-1), depth=2)}
+ return inputs, {"labels": tf.one_hot(tf.squeeze(targets["labels"], axis=-1), depth=2)}
def normalize_fn(inputs):
--
GitLab
From 69d66112f950183db7a8a651cbd28d0e6bbd8927 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 17:02:44 +0200
Subject: [PATCH 17/77] WIP: example
---
otbtf/examples/tensorflow_v2x/fcnn/sampling.py | 0
1 file changed, 0 insertions(+), 0 deletions(-)
delete mode 100644 otbtf/examples/tensorflow_v2x/fcnn/sampling.py
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/sampling.py b/otbtf/examples/tensorflow_v2x/fcnn/sampling.py
deleted file mode 100644
index e69de29b..00000000
--
GitLab
From f1a23e5a09ccc6b4f927c0d7c2f98468cd650929 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 19 Jul 2022 22:28:11 +0200
Subject: [PATCH 18/77] WIP: example
---
otbtf/examples/tensorflow_v2x/fcnn/README.md | 67 ++++++++++
.../tensorflow_v2x/fcnn/create_tfrecords.py | 20 +--
.../fcnn/{training.py => fcnn_model.py} | 29 ++---
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 116 ++++--------------
.../fcnn/train_from_patches-images.py | 84 +++++++++++++
.../fcnn/train_from_tfrecords.py | 56 +++++++++
6 files changed, 251 insertions(+), 121 deletions(-)
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/README.md
rename otbtf/examples/tensorflow_v2x/fcnn/{training.py => fcnn_model.py} (75%)
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
create mode 100644 otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/README.md b/otbtf/examples/tensorflow_v2x/fcnn/README.md
new file mode 100644
index 00000000..fe0280c1
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/fcnn/README.md
@@ -0,0 +1,67 @@
+This example show how to train a small fully convolutional model using the
+OTBTF python API. In particular, the example show how a model can be trained
+(1) from **patches-images**, or (2) from **TFRecords** files.
+
+# Files
+
+- `fcnn_model.py` implements a small fully convolutional U-Net like model,
+with the preprocessing and normalization functions
+- `train_from_patches-images.py` shows how to train the model from a list of
+patches-images
+- `train_from_tfrecords.py` shows how to train the model from TFRecords files
+- `create_tfrecords.py` shows how to convert patch-images into TFRecords files
+- `helper.py` contains a few helping functions
+-
+# Patches-images vs TFRecords based datasets
+
+TensorFlow datasets are the most practical way to feed a network data during
+training steps.
+In particular, they are very useful to train models with data parallelism using
+multiple workers (i.e. multiple GPU devices).
+Since OTBTF 3, two kind of approaches are available to deliver the patches:
+- Create TF datasets from **patches-images**: the first approach implemented in
+OTBTF, relying on geospatial raster formats supported by GDAL. Patches are simply
+stacked in rows. patches-images are friendly because they can be visualized
+like any other image. However this approach is **not very optimized**, since it
+generates a lot of I/O and stresses the filesystem when iterating randomly over
+patches.
+- Create TF datasets from **TFRecords** files. The principle is that a number of
+patches are stored in TFRecords files (google protubuf serialized data). This
+approach provides the best performances, since it generates less I/Os since
+multiple patches are read simultaneously together. It is the recommended approach
+to work on high end gear. It requires an additional step of converting the
+patches-images into TFRecords files.
+
+
+# A quick overview
+
+## Patches-images based datasets
+
+**Patches-images** are generated from the `PatchesExtraction` application of OTBTF.
+They consist in extracted patches stacked in rows into geospatial rasters.
+The `otbtf.DatasetFromPatchesImages` provides access to **patches-images** as a
+TF dataset. It inherits from the `otbtf.Dataset` class, which can be a base class
+to develop other raster based datasets.
+The `use_streaming` option can be used to read the patches on-the-fly
+on the filesystem. However, this can cause I/O bottleneck when one training step
+is shorter that fetching one batch of data. Typically, this is very common with
+small networks trained over large amount of data using multiple GPUs, causing the
+filesystem read operation being the weak point (and the GPUs wait for the batches
+to be ready). The class offers other functionalities, for instance changing the
+iterator class with a custom one (can inherit from `otbtf.dataset.IteratorBase`)
+which is, by default, an `otbtf.dataset.RandomIterator`. This could enable to
+control how the patches are walked, from the multiple patches-images of the
+dataset.
+
+## TFRecords batches datasets
+
+**TFRecord** based datasets are implemented in the `otbtf.tfrecords` module.
+They basically deliver patches from the TFRecords files, which can be created
+with the `to_tfrecords()` method of the `otbtf.Dataset` based classes.
+Depending on the filesystem characteristics and the computational cost of one
+training step, it can be good to select the number of samples per TFRecords file.
+Another tweak is the shuffling: since one TFRecord file contains multiple patches,
+the way TFRecords files are accessed (sometimes, we need them to be randomly
+accessed), and the way patches are accessed (within a buffer, of size set with the
+`shuffle_buffer_size`), is crucial.
+
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
index c1fc3209..ebd72adb 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
@@ -1,5 +1,10 @@
+"""
+This example shows how to convert patches-images (like the ones generated from the `PatchesExtraction`)
+into TFRecords files.
+"""
import argparse
from pathlib import Path
+import helper
from otbtf import DatasetFromPatchesImages
# Application parameters
@@ -10,19 +15,6 @@ parser.add_argument("--outdir", required=True, help="Output dir for TFRecords fi
params = parser.parse_args()
-def check_files_order(patches, labels):
- """
- Here we check that the (input_xs, labels) patches are well sorted
- """
- assert len(patches) == len(labels)
-
- def get_basename(n):
- return "_".join([n.split("_")][:-1])
-
- for p, l in zip(patches, labels):
- assert get_basename(p) == get_basename(l)
-
-
if __name__ == "__main__":
# Sort patches and labels
@@ -30,7 +22,7 @@ if __name__ == "__main__":
labels = sorted(params.labels)
# Check patches and labels are correctly sorted
- check_files_order(patches, labels)
+ helper.check_files_order(patches, labels)
# Create output directory
outdir = Path(params.outdir)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/training.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
similarity index 75%
rename from otbtf/examples/tensorflow_v2x/fcnn/training.py
rename to otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 5bac52d1..1ce18340 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/training.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -1,20 +1,11 @@
+"""
+Implementation of a small U-Net like model
+"""
from otbtf.model import ModelBase
import tensorflow as tf
import tensorflow.keras.layers as layers
-import argparse
import pathlib
import os
-import helper
-
-# Application parameters
-parser = argparse.ArgumentParser(description="Train a FCNN model")
-parser.add_argument("-p", "--dataset_dir", required=True, help="Directory of subdirs: train, valid(, test)")
-parser.add_argument("-f", "--dataset_format", default="tfrecords", const="tfrecords", nargs="?",
- choices=["tfrecords", "patches_images"], help="Format of the dataset (TFRecords or Patches images")
-parser.add_argument("-b", "--batch_size", type=int, default=8, help="Batch size")
-parser.add_argument("-r", "--learning_rate", type=float, default=0.00001, help="Learning rate")
-parser.add_argument("-e", "--number_epochs", type=int, default=100, help="Number of epochs")
-parser.add_argument("-m", "--model_dir", required=True, help="Path to save model")
class FCNNModel(ModelBase):
@@ -80,16 +71,18 @@ def normalize_fn(inputs):
return {"input_xs": inputs["input_xs"] * 0.0001}
-if __name__ == "__main__":
- params = parser.parse_args()
+def train(params, ds_train, ds_valid, ds_test):
+ """
+ Create, train, and save the model.
- # Get datasets
- ds_train, ds_valid, ds_test = helper.get_datasets(params.dataset_format, params.dataset_dir)
+ :param params: contains batch_size, learning_rate, nb_epochs, and model_dir
+ """
strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
with strategy.scope():
# Create and compile the model
+ # Note that the normalize_fn will now be a part of the model
model = FCNNModel(normalize_fn=normalize_fn)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=params.learning_rate),
@@ -103,11 +96,11 @@ if __name__ == "__main__":
tf.keras.utils.plot_model(model, os.path.join(params.model_dir, "figure.png"))
# Train
- model.fit(ds_train, epochs=params.number_epochs, validation_data=ds_valid)
+ model.fit(ds_train, epochs=params.nb_epochs, validation_data=ds_valid)
# Evaluate against test data
if ds_test is not None:
model.evaluate(ds_test, batch_size=params.batch_size)
- # Save trained model
+ # Save trained model as SavedModel
model.save(params.model_dir)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index 949ce0f6..da0ed04f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -1,97 +1,35 @@
-from otbtf import TFRecords, dataset
-import pathlib
+"""
+A set of helpers for the examples
+"""
+import argparse
-def get_datasets(dataset_format, dataset_dir, batch_size, target_keys):
+def base_parser():
"""
- Function to use either TFRecords or patches images.
- Take a look in the comments below to see how files must be stored.
+ Create a parser with the base parameters
- :param dataset_format: dataset format. Either ("tfrecords" or "patches_images")
- :param dataset_dir": dataset root directory. Must contain 2 (3) subdirectories (train, valid(, test))
+ :return: argparse.ArgumentParser instance
"""
+ parser = argparse.ArgumentParser(description="Train a FCNN model")
+ parser.add_argument("-b", "--batch_size", type=int, default=8, help="Batch size")
+ parser.add_argument("-r", "--learning_rate", type=float, default=0.00001, help="Learning rate")
+ parser.add_argument("-e", "--nb_epochs", type=int, default=100, help="Number of epochs")
+ parser.add_argument("-m", "--model_dir", required=True, help="Path to save model")
+ return parser
- # Patches directories must contain 'train' and 'valid' dirs, 'test' is not required
- patches = pathlib.Path(dataset_dir)
- datasets = {}
- for d in patches.iterdir():
- if not d.is_dir():
- continue
- tag = d.name.lower()
- assert tag in ["train", "valid", "test"], "Subfolders must be named train, valid (and test)"
- if dataset_format == "tfrecords":
- # When the dataset format is TFRecords, we expect that the files are stored in the following way, with
- # m, n, and k denoting respectively the number of TFRecords files in the training, validation, and test
- # datasets:
- #
- # /dataset_dir
- # /train
- # 1.records
- # 2.records
- # ...
- # m.records
- # /valid
- # 1.records
- # 2.records
- # ...
- # n.records
- # /test
- # 1.records
- # 2.records
- # ...
- # k.records
- #
- tfrecords = TFRecords(d)
- datasets[tag] = tfrecords.read(batch_size=batch_size, target_keys=target_keys,
- shuffle_buffer_size=1000) if tag == "train" else tfrecords.read(
- batch_size=batch_size, target_keys=target_keys)
- elif dataset_format == "patches_images":
- # When the dataset format is patches_images, we expect that the files are stored in the following way, with
- # M, N and K denoting respectively the number of patches-images in the training, validation, and test
- # datasets:
- #
- # /dataset_dir
- # /train
- # /image_1
- # ..._xs.tif
- # ..._labels.tif
- # /image_2
- # ..._xs.tif
- # ..._labels.tif
- # ...
- # /image_M
- # ..._xs.tif
- # ..._labels.tif
- # /valid
- # /image_1
- # ..._xs.tif
- # ..._labels.tif
- # ...
- # /image_N
- # ..._xs.tif
- # ..._labels.tif
- # /test
- # /image_1
- # ..._xs.tif
- # ..._labels.tif
- # ...
- # /image_K
- # ..._xs.tif
- # ..._labels.tif
- filenames_dict = {"input_xs": [],
- "labels": []}
- for subd in d.iterdir():
- if not subd.is_dir():
- continue
- for filename in subd.iterdir():
- if filename.lower().endswith("_xs.tif"):
- filenames_dict["input_xs"].append(filename)
- if filename.lower().endswith("_labels.tif"):
- filenames_dict["labels"].append(filename)
- # You can turn use_streaming=True to lower the memory footprint (patches are read on-the-fly on disk)
- datasets[tag] = dataset.DatasetFromPatchesImages(filenames_dict=filenames_dict)
- else:
- raise ValueError("dataset_format must be \"tfrecords\" or \"patches_images\"")
+def check_files_order(files1, files2):
+ """
+ Here we check that the two input lists of str are correctly sorted.
+ Except for the last, splits of files1[i] and files2[i] from the "_" character, must be equal.
+
+ :param files1: list of filenames (str)
+ :param files2: list of filenames (str)
+ """
+ assert len(files1) == len(files2)
+
+ def get_basename(n):
+ return "_".join([n.split("_")][:-1])
- return datasets
+ for p, l in zip(files1, files2):
+ assert get_basename(p) == get_basename(l)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
new file mode 100644
index 00000000..26728473
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -0,0 +1,84 @@
+"""
+This example shows how to use the otbtf python API to train a deep net from patches-images.
+
+We expect that the files are stored in the following way, with M, N and K denoting respectively
+the number of patches-images in the training, validation, and test datasets:
+
+/dataset_dir
+ /train
+ /image_1
+ ..._xs.tif
+ ..._labels.tif
+ /image_2
+ ..._xs.tif
+ ..._labels.tif
+ ...
+ /image_M
+ ..._xs.tif
+ ..._labels.tif
+ /valid
+ /image_1
+ ..._xs.tif
+ ..._labels.tif
+ ...
+ /image_N
+ ..._xs.tif
+ ..._labels.tif
+ /test
+ /image_1
+ ..._xs.tif
+ ..._labels.tif
+ ...
+ /image_K
+ ..._xs.tif
+ ..._labels.tif
+
+"""
+import helper
+from otbtf import DatasetFromPatchesImages
+import fcnn_model
+
+parser = helper.base_parser()
+parser.add_argument("--train_xs", required=True, nargs="+", default=[],
+ help="A list of patches-images for the XS image (training dataset)")
+parser.add_argument("--train_labels", required=True, nargs="+", default=[],
+ help="A list of patches-images for the labels (training dataset)")
+parser.add_argument("--valid_xs", required=True, nargs="+", default=[],
+ help="A list of patches-images for the XS image (validation dataset)")
+parser.add_argument("--valid_labels", required=True, nargs="+", default=[],
+ help="A list of patches-images for the labels (validation dataset)")
+parser.add_argument("--test_xs", required=False, nargs="+", default=[],
+ help="A list of patches-images for the XS image (test dataset)")
+parser.add_argument("--test_labels", required=False, nargs="+", default=[],
+ help="A list of patches-images for the labels (test dataset)")
+
+
+def create_dataset(xs_filenames, labels_filenames, batch_size):
+ """
+ Create an otbtf.DatasetFromPatchesImages
+ """
+ # Sort patches and labels
+ xs_filenames.sort()
+ labels_filenames.sort()
+
+ # Check patches and labels are correctly sorted
+ helper.check_files_order(xs_filenames, labels_filenames)
+
+ # Create dataset from the filename dict
+ # You can add the use_streaming option here, is you want to lower the memory budget.
+ # However, this can slow down your process since the patches are read on-the-fly on the filesystem.
+ # Good when one batch computation is slower than one batch gathering.
+ ds = DatasetFromPatchesImages(filenames_dict={"input_xs": xs_filenames, "labels": labels_filenames})
+ return ds, ds.get_tf_dataset(batch_size=batch_size)
+
+
+if __name__ == "__main__":
+ params = parser.parse_args()
+
+ _, ds_train = create_dataset(params.train_xs, params.train_labels, params.batchsize)
+ _, ds_valid = create_dataset(params.valid_xs, params.valid_labels, params.batchsize)
+ _, ds_test = create_dataset(params.valid_xs, params.valid_labels, params.batchsize) \
+ if params.test_xs and params.test_labels else None
+
+ # Train the model
+ fcnn_model.train(params, ds_train, ds_valid, ds_test)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
new file mode 100644
index 00000000..8b476d2a
--- /dev/null
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -0,0 +1,56 @@
+"""
+This example shows how to use the otbtf python API to train a deep net from TFRecords.
+
+We expect that the files are stored in the following way, with m, n, and k denoting respectively
+the number of TFRecords files in the training, validation, and test datasets:
+
+/dataset_dir
+ /train
+ 1.records
+ 2.records
+ ...
+ m.records
+ /valid
+ 1.records
+ 2.records
+ ...
+ n.records
+ /test
+ 1.records
+ 2.records
+ ...
+ k.records
+
+"""
+import helper
+import os
+from otbtf import TFRecords
+import fcnn_model
+
+parser = helper.base_parser()
+parser.add_argument("-p", "--tfrecords_dir", required=True,
+ help="Directory of subdirs containing TFRecords files: train, valid(, test)")
+
+if __name__ == "__main__":
+ params = parser.parse_args()
+
+ # Patches directories must contain 'train' and 'valid' dirs ('test' is not required)
+ train_dir = os.path.join(params.tfrecords_dir, "train")
+ valid_dir = os.path.join(params.tfrecords_dir, "valid")
+ test_dir = os.path.join(params.tfrecords_dir, "test")
+
+ # Training dataset. Must be shuffled!
+ assert os.path.isdir(train_dir)
+ ds_train = TFRecords(train_dir).read(batch_size=params.batch_size, target_keys=["label"],
+ shuffle_buffer_size=1000)
+
+ # Validation dataset
+ assert os.path.isdir(valid_dir)
+ ds_valid = TFRecords(valid_dir).read(batch_size=params.batch_size, target_keys=["label"])
+
+ # Test dataset (optional)
+ ds_test = TFRecords(test_dir).read(batch_size=params.batch_size, target_keys=["label"]) if os.path.isdir(
+ test_dir) else None
+
+ # Train the model
+ fcnn_model.train(params, ds_train, ds_valid, ds_test)
--
GitLab
From 80bbdf3934fa06322743bbfa695fb23a903e7ac1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 10:49:34 +0200
Subject: [PATCH 19/77] ADD: fix exception message
---
otbtf/dataset.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index 00275481..bef69c9e 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -470,7 +470,7 @@ class Dataset:
:return: The TF dataset
"""
if batch_size <= 2 * self.miner_buffer.max_length:
- logging.warning("Batch size is {} but dataset buffer has {} elements. Consider using a larger dataset "
+ logging.warning("Batch size is %s but dataset buffer has %s elements. Consider using a larger dataset "
"buffer to avoid I/O bottleneck", batch_size, self.miner_buffer.max_length)
return self.tf_dataset.batch(batch_size, drop_remainder=drop_remainder)
--
GitLab
From b55144b871e32788b33163d23cf87d2abe7de7ee Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:01:52 +0200
Subject: [PATCH 20/77] REFAC: output_shape --> output_shapes
---
otbtf/tfrecords.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 28a72de4..d92edf37 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -41,8 +41,8 @@ class TFRecords:
self.dirpath = path
os.makedirs(self.dirpath, exist_ok=True)
self.output_types_file = os.path.join(self.dirpath, "output_types.json")
- self.output_shape_file = os.path.join(self.dirpath, "output_shape.json")
- self.output_shape = self.load(self.output_shape_file) if os.path.exists(self.output_shape_file) else None
+ self.output_shapes_file = os.path.join(self.dirpath, "output_shapes.json")
+ self.output_shapes = self.load(self.output_shapes_file) if os.path.exists(self.output_shapes_file) else None
self.output_types = self.load(self.output_types_file) if os.path.exists(self.output_types_file) else None
@staticmethod
@@ -71,7 +71,7 @@ class TFRecords:
nb_shards += 1
output_shapes = {key: output_shape for key, output_shape in dataset.output_shapes.items()}
- self.save(output_shapes, self.output_shape_file)
+ self.save(output_shapes, self.output_shapes_file)
output_types = {key: output_type.name for key, output_type in dataset.output_types.items()}
self.save(output_types, self.output_types_file)
--
GitLab
From 0e8d697b0ac1398bba8bce19bc7809e00c81c163 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:02:13 +0200
Subject: [PATCH 21/77] ADD: _is_chief() method
---
otbtf/utils.py | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/otbtf/utils.py b/otbtf/utils.py
index 069638a5..0cc3b4a4 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -63,3 +63,27 @@ def read_as_np_arr(gdal_ds, as_patches=True, dtype=None):
buffer = buffer.astype(dtype)
return buffer
+
+
+def _is_chief(strategy):
+ """
+ Tell if the current worker is the chief.
+
+ :param strategy: strategy
+ :return: True if the current worker is the chief, False else
+ """
+ # Note: there are two possible `TF_CONFIG` configuration.
+ # 1) In addition to `worker` tasks, a `chief` task type is use;
+ # in this case, this function should be modified to
+ # `return task_type == 'chief'`.
+ # 2) Only `worker` task type is used; in this case, worker 0 is
+ # regarded as the chief. The implementation demonstrated here
+ # is for this case.
+ # For the purpose of this Colab section, the `task_type is None` case
+ # is added because it is effectively run with only a single worker.
+
+ if strategy.cluster_resolver: # this means MultiWorkerMirroredStrategy
+ task_type, task_id = strategy.cluster_resolver.task_type, strategy.cluster_resolver.task_id
+ return (task_type == 'chief') or (task_type == 'worker' and task_id == 0) or task_type is None
+ else: # strategy with only one worker
+ return True
\ No newline at end of file
--
GitLab
From c15e7cde836bd624e4b4b26527284f9c5df6b8b0 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:02:46 +0200
Subject: [PATCH 22/77] REFAC: simplify parser opts
---
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index da0ed04f..86226c26 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -11,10 +11,10 @@ def base_parser():
:return: argparse.ArgumentParser instance
"""
parser = argparse.ArgumentParser(description="Train a FCNN model")
- parser.add_argument("-b", "--batch_size", type=int, default=8, help="Batch size")
- parser.add_argument("-r", "--learning_rate", type=float, default=0.00001, help="Learning rate")
- parser.add_argument("-e", "--nb_epochs", type=int, default=100, help="Number of epochs")
- parser.add_argument("-m", "--model_dir", required=True, help="Path to save model")
+ parser.add_argument("--batch_size", type=int, default=8, help="Batch size")
+ parser.add_argument("--learning_rate", type=float, default=0.00001, help="Learning rate")
+ parser.add_argument("--nb_epochs", type=int, default=100, help="Number of epochs")
+ parser.add_argument("--model_dir", required=True, help="Path to save model")
return parser
--
GitLab
From 365239ae7bcf9a050eec5bcdbab7406cd611dd5c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:03:27 +0200
Subject: [PATCH 23/77] DOC: wip
---
otbtf/examples/tensorflow_v2x/fcnn/README.md | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/README.md b/otbtf/examples/tensorflow_v2x/fcnn/README.md
index fe0280c1..e6cfce78 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/README.md
+++ b/otbtf/examples/tensorflow_v2x/fcnn/README.md
@@ -11,7 +11,7 @@ patches-images
- `train_from_tfrecords.py` shows how to train the model from TFRecords files
- `create_tfrecords.py` shows how to convert patch-images into TFRecords files
- `helper.py` contains a few helping functions
--
+
# Patches-images vs TFRecords based datasets
TensorFlow datasets are the most practical way to feed a network data during
@@ -32,9 +32,6 @@ multiple patches are read simultaneously together. It is the recommended approac
to work on high end gear. It requires an additional step of converting the
patches-images into TFRecords files.
-
-# A quick overview
-
## Patches-images based datasets
**Patches-images** are generated from the `PatchesExtraction` application of OTBTF.
--
GitLab
From ce1d286d2ea6b5230406d0c42817e98f65a52671 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:03:50 +0200
Subject: [PATCH 24/77] WIP: model working with both examples
---
.../examples/tensorflow_v2x/fcnn/fcnn_model.py | 10 ++++++----
.../fcnn/train_from_patches-images.py | 18 +++++++++++-------
.../fcnn/train_from_tfrecords.py | 7 ++++---
otbtf/model.py | 3 +--
4 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 1ce18340..c4e8cf7a 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -71,7 +71,7 @@ def normalize_fn(inputs):
return {"input_xs": inputs["input_xs"] * 0.0001}
-def train(params, ds_train, ds_valid, ds_test):
+def train(params, ds_train, ds_valid, ds_test, output_shape):
"""
Create, train, and save the model.
@@ -80,10 +80,12 @@ def train(params, ds_train, ds_valid, ds_test):
strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
with strategy.scope():
-
# Create and compile the model
- # Note that the normalize_fn will now be a part of the model
- model = FCNNModel(normalize_fn=normalize_fn)
+
+ model = FCNNModel(dataset_input_keys=["input_xs"],
+ model_output_keys=["labels"],
+ dataset_shapes=output_shape,
+ normalize_fn=normalize_fn) # Note that the normalize_fn is now part of the model
model.compile(loss=tf.keras.losses.BinaryCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=params.learning_rate),
metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 26728473..64319829 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -53,7 +53,7 @@ parser.add_argument("--test_labels", required=False, nargs="+", default=[],
help="A list of patches-images for the labels (test dataset)")
-def create_dataset(xs_filenames, labels_filenames, batch_size):
+def create_dataset(xs_filenames, labels_filenames):
"""
Create an otbtf.DatasetFromPatchesImages
"""
@@ -69,16 +69,20 @@ def create_dataset(xs_filenames, labels_filenames, batch_size):
# However, this can slow down your process since the patches are read on-the-fly on the filesystem.
# Good when one batch computation is slower than one batch gathering.
ds = DatasetFromPatchesImages(filenames_dict={"input_xs": xs_filenames, "labels": labels_filenames})
- return ds, ds.get_tf_dataset(batch_size=batch_size)
+ print(ds)
+ tf_ds = ds.get_tf_dataset(batch_size=params.batch_size)
+ print(tf_ds)
+ return ds, tf_ds
if __name__ == "__main__":
params = parser.parse_args()
- _, ds_train = create_dataset(params.train_xs, params.train_labels, params.batchsize)
- _, ds_valid = create_dataset(params.valid_xs, params.valid_labels, params.batchsize)
- _, ds_test = create_dataset(params.valid_xs, params.valid_labels, params.batchsize) \
- if params.test_xs and params.test_labels else None
+ ds, ds_train = create_dataset(params.train_xs, params.train_labels)
+ _, ds_valid = create_dataset(params.valid_xs, params.valid_labels)
+ ds_test = None
+ if params.test_xs and params.test_labels:
+ _, ds_test = create_dataset(params.test_xs, params.test_labels)
# Train the model
- fcnn_model.train(params, ds_train, ds_valid, ds_test)
+ fcnn_model.train(params, ds_train, ds_valid, ds_test, ds.output_shapes)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 8b476d2a..34344a3f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -28,7 +28,7 @@ from otbtf import TFRecords
import fcnn_model
parser = helper.base_parser()
-parser.add_argument("-p", "--tfrecords_dir", required=True,
+parser.add_argument("--tfrecords_dir", required=True,
help="Directory of subdirs containing TFRecords files: train, valid(, test)")
if __name__ == "__main__":
@@ -41,7 +41,8 @@ if __name__ == "__main__":
# Training dataset. Must be shuffled!
assert os.path.isdir(train_dir)
- ds_train = TFRecords(train_dir).read(batch_size=params.batch_size, target_keys=["label"],
+ ds = TFRecords(train_dir)
+ ds_train = ds.read(batch_size=params.batch_size, target_keys=["label"],
shuffle_buffer_size=1000)
# Validation dataset
@@ -53,4 +54,4 @@ if __name__ == "__main__":
test_dir) else None
# Train the model
- fcnn_model.train(params, ds_train, ds_valid, ds_test)
+ fcnn_model.train(params, ds_train, ds_valid, ds_test, ds.output_shapes)
diff --git a/otbtf/model.py b/otbtf/model.py
index 80d459a9..35ad2290 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -13,7 +13,6 @@ class ModelBase(abc.ABC):
Base class for all models
"""
- @abc.abstractmethod
def __init__(self, dataset_input_keys, model_output_keys, dataset_shapes, target_cropping=None,
inference_cropping=None, normalize_fn=None):
"""
@@ -77,7 +76,7 @@ class ModelBase(abc.ABC):
:param inputs: inputs, either keras.Input or normalized_inputs
:return: a dict of outputs tensors of the model
"""
- pass
+ raise NotImplemented("This method has to be implemented. Here you code the model :)")
def create_network(self):
"""
--
GitLab
From a63dd1a919241e7d9b5246352915f2e1dc49fd28 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:16:48 +0200
Subject: [PATCH 25/77] FIX: normalize_fn
---
otbtf/model.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 35ad2290..2e1ce3ae 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -58,14 +58,15 @@ class ModelBase(abc.ABC):
model_inputs = {}
for key in self.dataset_input_keys:
shape = self.dataset_shapes[key]
+ new_shape = list(shape)
if shape[0] is None or (len(shape) > 3): # for backward comp (OTBTF<3.2.2), remove the potential batch dim
- shape = shape[1:]
+ new_shape = shape[1:]
# Here we modify the x and y dims of >2D tensors to enable any image size at input
- if len(shape) > 2:
- shape[0] = None
- shape[1] = None
- placeholder = keras.Input(shape=shape, name=key)
- print(key, shape)
+ if len(new_shape) > 2:
+ new_shape[0] = None
+ new_shape[1] = None
+ placeholder = keras.Input(shape=new_shape, name=key)
+ logging.info("New shape for input %s: %s", key, new_shape)
model_inputs.update({key: placeholder})
return model_inputs
@@ -88,8 +89,8 @@ class ModelBase(abc.ABC):
model_inputs = self.get_inputs()
# Normalize the inputs. If some input keys are not handled by normalized_fn, these inputs are not normalized
- normalized_inputs = {key: self.normalize_fn(inp)[key] if key in self.normalize_fn(inp) else inp for key, inp in
- model_inputs.items()} if self.normalize_fn else model_inputs
+ normalized_inputs = model_inputs.copy()
+ normalized_inputs.update(self.normalize_fn(model_inputs))
# Build the model
outputs = self.get_outputs(normalized_inputs)
--
GitLab
From a603d3f2b58a207ba2c449d61bd07a94454c1c9f Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:20:53 +0200
Subject: [PATCH 26/77] WIP: model working with both examples
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 10 +++-------
.../tensorflow_v2x/fcnn/train_from_patches-images.py | 2 --
2 files changed, 3 insertions(+), 9 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index c4e8cf7a..1ff84543 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -40,7 +40,7 @@ class FCNNModel(ModelBase):
tconv = layers.Conv2DTranspose(filters=depth, kernel_size=3, activation="relu", name=name)
net = tconv(net)
- return net
+ return {"estimated": net}
def preprocessing_fn(inputs, targets):
@@ -53,7 +53,7 @@ def preprocessing_fn(inputs, targets):
:param targets: dict for targets
:return: an output tuple (processed_inputs, processed_targets)
"""
- return inputs, {"labels": tf.one_hot(tf.squeeze(targets["labels"], axis=-1), depth=2)}
+ return inputs, {"estimated": tf.one_hot(tf.squeeze(targets["labels"], axis=-1), depth=2)}
def normalize_fn(inputs):
@@ -91,11 +91,7 @@ def train(params, ds_train, ds_valid, ds_test, output_shape):
metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
# Summarize the model (in CLI)
- model.summary(line_length=120)
-
- # Summarize the model (in figure.png)
- pathlib.Path(params.model_dir).mkdir(exist_ok=True)
- tf.keras.utils.plot_model(model, os.path.join(params.model_dir, "figure.png"))
+ model.summary()
# Train
model.fit(ds_train, epochs=params.nb_epochs, validation_data=ds_valid)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 64319829..154c253b 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -69,9 +69,7 @@ def create_dataset(xs_filenames, labels_filenames):
# However, this can slow down your process since the patches are read on-the-fly on the filesystem.
# Good when one batch computation is slower than one batch gathering.
ds = DatasetFromPatchesImages(filenames_dict={"input_xs": xs_filenames, "labels": labels_filenames})
- print(ds)
tf_ds = ds.get_tf_dataset(batch_size=params.batch_size)
- print(tf_ds)
return ds, tf_ds
--
GitLab
From 11ecc90ff8e475651d19caf3a1877b4155d0e5ef Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:56:10 +0200
Subject: [PATCH 27/77] WIP: model working with both examples
---
.../examples/tensorflow_v2x/fcnn/fcnn_model.py | 13 ++++++-------
otbtf/model.py | 18 +++++++++---------
2 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 1ff84543..fdc54e13 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -4,8 +4,6 @@ Implementation of a small U-Net like model
from otbtf.model import ModelBase
import tensorflow as tf
import tensorflow.keras.layers as layers
-import pathlib
-import os
class FCNNModel(ModelBase):
@@ -40,7 +38,7 @@ class FCNNModel(ModelBase):
tconv = layers.Conv2DTranspose(filters=depth, kernel_size=3, activation="relu", name=name)
net = tconv(net)
- return {"estimated": net}
+ return {"labels": tf.keras.activations.softmax(net)}
def preprocessing_fn(inputs, targets):
@@ -53,7 +51,8 @@ def preprocessing_fn(inputs, targets):
:param targets: dict for targets
:return: an output tuple (processed_inputs, processed_targets)
"""
- return inputs, {"estimated": tf.one_hot(tf.squeeze(targets["labels"], axis=-1), depth=2)}
+ return {"input_xs": tf.keras.layers.Cropping2D(cropping=32)(inputs["input_xs"])}, \
+ {"labels": tf.one_hot(tf.squeeze(targets["labels"], axis=-1), depth=2)}
def normalize_fn(inputs):
@@ -71,7 +70,7 @@ def normalize_fn(inputs):
return {"input_xs": inputs["input_xs"] * 0.0001}
-def train(params, ds_train, ds_valid, ds_test, output_shape):
+def train(params, ds_train, ds_valid, ds_test, output_shapes):
"""
Create, train, and save the model.
@@ -84,9 +83,9 @@ def train(params, ds_train, ds_valid, ds_test, output_shape):
model = FCNNModel(dataset_input_keys=["input_xs"],
model_output_keys=["labels"],
- dataset_shapes=output_shape,
+ dataset_shapes=output_shapes,
normalize_fn=normalize_fn) # Note that the normalize_fn is now part of the model
- model.compile(loss=tf.keras.losses.BinaryCrossentropy(),
+ model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=params.learning_rate),
metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
diff --git a/otbtf/model.py b/otbtf/model.py
index 2e1ce3ae..9e28d88f 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -95,15 +95,15 @@ class ModelBase(abc.ABC):
# Build the model
outputs = self.get_outputs(normalized_inputs)
- # Add extra outputs for inference
- extra_outputs = {}
- for out_key, out_tensor in outputs.items():
- for crop in self.inference_cropping:
- extra_output_key = cropped_tensor_name(out_key, crop)
- extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
- extra_output = tf.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
- extra_outputs[extra_output_key] = extra_output
- outputs.update(extra_outputs)
+ # # Add extra outputs for inference
+ # extra_outputs = {}
+ # for out_key, out_tensor in outputs.items():
+ # for crop in self.inference_cropping:
+ # extra_output_key = cropped_tensor_name(out_key, crop)
+ # extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
+ # extra_output = tf.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
+ # extra_outputs[extra_output_key] = extra_output
+ # outputs.update(extra_outputs)
# Return the keras model
self.model = keras.Model(inputs=model_inputs, outputs=outputs, name=self.__class__.__name__)
--
GitLab
From d7553ea5cec1dd23650502935dd234e46da90287 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 11:56:50 +0200
Subject: [PATCH 28/77] WIP: model working with both examples
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index fdc54e13..31993269 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -51,6 +51,7 @@ def preprocessing_fn(inputs, targets):
:param targets: dict for targets
:return: an output tuple (processed_inputs, processed_targets)
"""
+ # TODO: delete the cropping, just for testing with my data
return {"input_xs": tf.keras.layers.Cropping2D(cropping=32)(inputs["input_xs"])}, \
{"labels": tf.one_hot(tf.squeeze(targets["labels"], axis=-1), depth=2)}
--
GitLab
From 277b9116b504acf2fe795c4d7a43d58645774c7a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 12:33:15 +0200
Subject: [PATCH 29/77] WIP: model working with both examples
---
.../tensorflow_v2x/fcnn/fcnn_model.py | 28 +++++++++++--------
.../fcnn/train_from_patches-images.py | 12 ++++++--
.../fcnn/train_from_tfrecords.py | 2 +-
3 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 31993269..dfacb309 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -5,6 +5,8 @@ from otbtf.model import ModelBase
import tensorflow as tf
import tensorflow.keras.layers as layers
+N_CLASSES = 6
+
class FCNNModel(ModelBase):
"""
@@ -20,9 +22,6 @@ class FCNNModel(ModelBase):
:return: activation values
"""
- # Model constants
- N_CLASSES = 6
-
# Model input
net = normalized_inputs["input_xs"]
@@ -38,22 +37,27 @@ class FCNNModel(ModelBase):
tconv = layers.Conv2DTranspose(filters=depth, kernel_size=3, activation="relu", name=name)
net = tconv(net)
- return {"labels": tf.keras.activations.softmax(net)}
+ # final layers
+ net = tf.keras.activations.softmax(net)
+ net = tf.keras.layers.Cropping2D(cropping=32)(net)
+
+ return {"predictions": net}
-def preprocessing_fn(inputs, targets):
+def preprocessing_fn(examples):
"""
Preprocessing function for the training dataset.
This function is only used at training time, to put the data in the expected format.
- DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see `otbtf.ModelBase.normalize_fn` for that)
+ DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see `otbtf.ModelBase.normalize_fn` for that).
+ Note that this function is not called here, but in the code that prepares the datasets.
- :param inputs: dict for inputs
- :param targets: dict for targets
- :return: an output tuple (processed_inputs, processed_targets)
+ :param examples: dict for examples (i.e. inputs and targets stored in a single dict)
+ :return: preprocessed examples
"""
- # TODO: delete the cropping, just for testing with my data
- return {"input_xs": tf.keras.layers.Cropping2D(cropping=32)(inputs["input_xs"])}, \
- {"labels": tf.one_hot(tf.squeeze(targets["labels"], axis=-1), depth=2)}
+ def _to_categorical(x):
+ return tf.one_hot(tf.squeeze(x, axis=-1), depth=N_CLASSES)
+ return {"input_xs": examples["input_xs"],
+ "predictions": _to_categorical(examples["labels"])}
def normalize_fn(inputs):
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 154c253b..05b67c9f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -53,7 +53,7 @@ parser.add_argument("--test_labels", required=False, nargs="+", default=[],
help="A list of patches-images for the labels (test dataset)")
-def create_dataset(xs_filenames, labels_filenames):
+def create_dataset(xs_filenames, labels_filenames, targets_keys=["labels"]):
"""
Create an otbtf.DatasetFromPatchesImages
"""
@@ -70,7 +70,15 @@ def create_dataset(xs_filenames, labels_filenames):
# Good when one batch computation is slower than one batch gathering.
ds = DatasetFromPatchesImages(filenames_dict={"input_xs": xs_filenames, "labels": labels_filenames})
tf_ds = ds.get_tf_dataset(batch_size=params.batch_size)
- return ds, tf_ds
+
+ def _split_inp_target(all_inp):
+ # Differentiating inputs and outputs
+ all_inp_prep = fcnn_model.preprocessing_fn(all_inp)
+ inputs = {key: value for (key, value) in all_inp_prep.items() if key not in targets_keys}
+ targets = {key: value for (key, value) in all_inp_prep.items() if key in targets_keys}
+ return inputs, targets
+
+ return ds, tf_ds.map(_split_inp_target)
if __name__ == "__main__":
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 34344a3f..2c4d97ea 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
# Training dataset. Must be shuffled!
assert os.path.isdir(train_dir)
- ds = TFRecords(train_dir)
+ ds = TFRecords(train_dir, preprocessing_fn=fcnn_model.preprocessing_fn)
ds_train = ds.read(batch_size=params.batch_size, target_keys=["label"],
shuffle_buffer_size=1000)
--
GitLab
From 5b9af71bdd4660467999543bbe2fca7d2b20010b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 12:39:17 +0200
Subject: [PATCH 30/77] WIP: model working with both examples
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index dfacb309..9762e07f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -82,14 +82,17 @@ def train(params, ds_train, ds_valid, ds_test, output_shapes):
:param params: contains batch_size, learning_rate, nb_epochs, and model_dir
"""
- strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
+ # Model
+ model = FCNNModel(dataset_input_keys=["input_xs"],
+ model_output_keys=["labels"],
+ dataset_shapes=output_shapes,
+ normalize_fn=normalize_fn) # Note that the normalize_fn is now part of the model
+
+ # strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
+ strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0")
with strategy.scope():
# Create and compile the model
-
- model = FCNNModel(dataset_input_keys=["input_xs"],
- model_output_keys=["labels"],
- dataset_shapes=output_shapes,
- normalize_fn=normalize_fn) # Note that the normalize_fn is now part of the model
+ model.create_network()
model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=params.learning_rate),
metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
--
GitLab
From d7f0a584ac4cff0a9e206f386d422dd7bca3427a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 12:52:53 +0200
Subject: [PATCH 31/77] REFAC: simplify preprocessing_fn() method... now
input/output a single dict. Good idea?
---
otbtf/tfrecords.py | 32 +++++++++++++++-----------------
1 file changed, 15 insertions(+), 17 deletions(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index d92edf37..63f07592 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -121,8 +121,7 @@ class TFRecords:
:param example: Example object to parse
:param features_types: List of types for each feature
:param target_keys: list of keys of the targets
- :param preprocessing_fn: Optional. A preprocessing function that takes input, target as args and returns
- a tuple (input_preprocessed, target_preprocessed)
+ :param preprocessing_fn: Optional. A preprocessing function that process the input example
:param kwargs: some keywords arguments for preprocessing_fn
"""
read_features = {key: tf.io.FixedLenFeature([], dtype=tf.string) for key in features_types}
@@ -132,11 +131,9 @@ class TFRecords:
example_parsed[key] = tf.io.parse_tensor(example_parsed[key], out_type=features_types[key])
# Differentiating inputs and outputs
- input_parsed = {key: value for (key, value) in example_parsed.items() if key not in target_keys}
- target_parsed = {key: value for (key, value) in example_parsed.items() if key in target_keys}
-
- if preprocessing_fn:
- input_parsed, target_parsed = preprocessing_fn(input_parsed, target_parsed, **kwargs)
+ example_parsed_prep = preprocessing_fn(example_parsed, **kwargs) if preprocessing_fn else example_parsed
+ input_parsed = {key: value for (key, value) in example_parsed_prep.items() if key not in target_keys}
+ target_parsed = {key: value for (key, value) in example_parsed_prep.items() if key in target_keys}
return input_parsed, target_parsed
@@ -153,16 +150,17 @@ class TFRecords:
False is advisable when evaluating metrics so that all samples are used
:param shuffle_buffer_size: if None, shuffle is not used. Else, blocks of shuffle_buffer_size
elements are shuffled using uniform random.
- :param preprocessing_fn: Optional. A preprocessing function that takes (input, target) as args and returns
- a tuple (input_preprocessed, target_preprocessed). Typically, target_preprocessed
- must be computed accordingly to (1) what the model outputs and (2) what training loss
- needs. For instance, for a classification problem, the model will likely output the
- softmax, or activation neurons, for each class, and the cross entropy loss requires
- labels in one hot encoding. In this case, the preprocessing_fn has to transform the
- labels values (integer ranging from [0, n_classes]) in one hot encoding (vector of 0
- and 1 of length n_classes). The preprocessing_fn should not implement such things as
- radiometric transformations from input to input_preprocessed, because those are
- performed inside the model itself (see `otbtf.ModelBase.normalize()`).
+ :param preprocessing_fn: Optional. A preprocessing function that takes input examples as args and returns the
+ preprocessed input examples. Typically, examples are composed of model inputs and
+ targets. Model inputs and model targets must be computed accordingly to (1) what the
+ model outputs and (2) what training loss needs. For instance, for a classification
+ problem, the model will likely output the softmax, or activation neurons, for each
+ class, and the cross entropy loss requires labels in one hot encoding. In this case,
+ the preprocessing_fn has to transform the labels values (integer ranging from
+ [0, n_classes]) in one hot encoding (vector of 0 and 1 of length n_classes). The
+ preprocessing_fn should not implement such things as radiometric transformations from
+ input to input_preprocessed, because those are performed inside the model itself
+ (see `otbtf.ModelBase.normalize()`).
:param kwargs: some keywords arguments for preprocessing_fn
"""
options = tf.data.Options()
--
GitLab
From 1b1b385e63d4aeed63167c524f8c6052fe0f1281 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 12:53:36 +0200
Subject: [PATCH 32/77] REFAC: remove unused model_output_keys
---
otbtf/model.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 9e28d88f..c7e47380 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -13,13 +13,12 @@ class ModelBase(abc.ABC):
Base class for all models
"""
- def __init__(self, dataset_input_keys, model_output_keys, dataset_shapes, target_cropping=None,
+ def __init__(self, dataset_input_keys, dataset_shapes, target_cropping=None,
inference_cropping=None, normalize_fn=None):
"""
Model base class
:param dataset_input_keys: list of dataset keys used for the training
- :param model_output_keys: list of the model outputs keys
:param dataset_shapes: a dict() of shapes
:param target_cropping: Optional. Number of pixels to be removed on each side of the target. This is used when
training the model and can mitigate the effects of convolution
@@ -30,7 +29,6 @@ class ModelBase(abc.ABC):
dict of inputs and returns a dict of normalized inputs. Optional
"""
self.dataset_input_keys = dataset_input_keys
- self.model_output_keys = model_output_keys
self.dataset_shapes = dataset_shapes
self.model = None
self.target_cropping = target_cropping
@@ -87,13 +85,16 @@ class ModelBase(abc.ABC):
# Get the model inputs
model_inputs = self.get_inputs()
+ logging.info(f"Model inputs: {model_inputs}")
# Normalize the inputs. If some input keys are not handled by normalized_fn, these inputs are not normalized
normalized_inputs = model_inputs.copy()
normalized_inputs.update(self.normalize_fn(model_inputs))
+ logging.info(f"Normalized model inputs: {normalized_inputs}")
# Build the model
outputs = self.get_outputs(normalized_inputs)
+ logging.info(f"Model outputs: {outputs}")
# # Add extra outputs for inference
# extra_outputs = {}
--
GitLab
From 593be85f0e82b1717733e870ccfd399cf1457655 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 12:53:53 +0200
Subject: [PATCH 33/77] REFAC: remove unused model_output_keys
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 9762e07f..a4a4055f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -84,7 +84,6 @@ def train(params, ds_train, ds_valid, ds_test, output_shapes):
# Model
model = FCNNModel(dataset_input_keys=["input_xs"],
- model_output_keys=["labels"],
dataset_shapes=output_shapes,
normalize_fn=normalize_fn) # Note that the normalize_fn is now part of the model
--
GitLab
From b2c319c197b7939c99f109d126831e5f10537dbf Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 12:54:28 +0200
Subject: [PATCH 34/77] FIX: target name
---
otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 05b67c9f..47b3afef 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -53,7 +53,7 @@ parser.add_argument("--test_labels", required=False, nargs="+", default=[],
help="A list of patches-images for the labels (test dataset)")
-def create_dataset(xs_filenames, labels_filenames, targets_keys=["labels"]):
+def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"]):
"""
Create an otbtf.DatasetFromPatchesImages
"""
--
GitLab
From dd65fdb416afdeed20452a9f00c99393699ee55d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 13:25:19 +0200
Subject: [PATCH 35/77] FIX: logging
---
otbtf/dataset.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index bef69c9e..2fa93498 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -273,7 +273,7 @@ class PatchesImagesReader(PatchesReaderBase):
"mean": rsize * _sums[src_key],
"std": np.sqrt(rsize * _sqsums[src_key] - np.square(rsize * _sums[src_key]))
} for src_key in self.gdal_ds}
- logging.info("Stats: {}", stats)
+ logging.info("Stats: %s", stats)
return stats
def get_size(self):
@@ -362,8 +362,8 @@ class Dataset:
self.output_shapes[src_key] = np_arr.shape
self.output_types[src_key] = tf.dtypes.as_dtype(np_arr.dtype)
- logging.info("output_types: {}", self.output_types)
- logging.info("output_shapes: {}", self.output_shapes)
+ logging.info("output_types: %s", self.output_types)
+ logging.info("output_shapes: %s", self.output_shapes)
# buffers
if self.size <= buffer_length:
--
GitLab
From 2993432b2c3db296c59666df2b37d2f5155178fa Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 13:27:42 +0200
Subject: [PATCH 36/77] REFAC: simplify model
---
.../tensorflow_v2x/fcnn/fcnn_model.py | 15 ++++++++-----
.../fcnn/train_from_patches-images.py | 4 ++--
.../fcnn/train_from_tfrecords.py | 16 ++++++++------
otbtf/model.py | 22 +++++++++----------
4 files changed, 32 insertions(+), 25 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index a4a4055f..03586b71 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -4,7 +4,10 @@ Implementation of a small U-Net like model
from otbtf.model import ModelBase
import tensorflow as tf
import tensorflow.keras.layers as layers
+import logging
+
+logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
N_CLASSES = 6
@@ -75,17 +78,19 @@ def normalize_fn(inputs):
return {"input_xs": inputs["input_xs"] * 0.0001}
-def train(params, ds_train, ds_valid, ds_test, output_shapes):
+def train(params, ds_train, ds_valid, ds_test):
"""
Create, train, and save the model.
:param params: contains batch_size, learning_rate, nb_epochs, and model_dir
+ :param ds_train: training dataset
+ :param ds_valid: validation dataset
+ :param ds_test: testing dataset
"""
- # Model
- model = FCNNModel(dataset_input_keys=["input_xs"],
- dataset_shapes=output_shapes,
- normalize_fn=normalize_fn) # Note that the normalize_fn is now part of the model
+ # Model instantiation
+ # Note that the normalize_fn is now part of the model
+ model = FCNNModel(dataset_element_spec=ds_train.element_spec, normalize_fn=normalize_fn)
# strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0")
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 47b3afef..5e4f9017 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -84,11 +84,11 @@ def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"])
if __name__ == "__main__":
params = parser.parse_args()
- ds, ds_train = create_dataset(params.train_xs, params.train_labels)
+ _, ds_train = create_dataset(params.train_xs, params.train_labels)
_, ds_valid = create_dataset(params.valid_xs, params.valid_labels)
ds_test = None
if params.test_xs and params.test_labels:
_, ds_test = create_dataset(params.test_xs, params.test_labels)
# Train the model
- fcnn_model.train(params, ds_train, ds_valid, ds_test, ds.output_shapes)
+ fcnn_model.train(params, ds_train, ds_valid, ds_test)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 2c4d97ea..9cd23ef6 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -39,19 +39,21 @@ if __name__ == "__main__":
valid_dir = os.path.join(params.tfrecords_dir, "valid")
test_dir = os.path.join(params.tfrecords_dir, "test")
+ def _tfrecords(directory):
+ return TFRecords(directory, preprocessing_fn=fcnn_model.preprocessing_fn)
+
# Training dataset. Must be shuffled!
assert os.path.isdir(train_dir)
- ds = TFRecords(train_dir, preprocessing_fn=fcnn_model.preprocessing_fn)
- ds_train = ds.read(batch_size=params.batch_size, target_keys=["label"],
- shuffle_buffer_size=1000)
+ ds_train = _tfrecords(train_dir).read(batch_size=params.batch_size, target_keys=["label"], shuffle_buffer_size=1000)
# Validation dataset
assert os.path.isdir(valid_dir)
- ds_valid = TFRecords(valid_dir).read(batch_size=params.batch_size, target_keys=["label"])
+ ds_valid = _tfrecords(valid_dir).read(batch_size=params.batch_size, target_keys=["label"])
# Test dataset (optional)
- ds_test = TFRecords(test_dir).read(batch_size=params.batch_size, target_keys=["label"]) if os.path.isdir(
- test_dir) else None
+ ds_test = None
+ if os.path.isdir(test_dir):
+ ds_test = _tfrecords(test_dir).read(batch_size=params.batch_size, target_keys=["label"]) if os.path.isdir(test_dir)
# Train the model
- fcnn_model.train(params, ds_train, ds_valid, ds_test, ds.output_shapes)
+ fcnn_model.train(params, ds_train, ds_valid, ds_test)
diff --git a/otbtf/model.py b/otbtf/model.py
index c7e47380..c0079033 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -2,8 +2,6 @@
""" Base class for models"""
import abc
import logging
-
-import tensorflow as tf
from tensorflow import keras
from otbtf.utils import _is_chief
@@ -13,13 +11,13 @@ class ModelBase(abc.ABC):
Base class for all models
"""
- def __init__(self, dataset_input_keys, dataset_shapes, target_cropping=None,
+ def __init__(self, dataset_element_spec, target_cropping=None,
inference_cropping=None, normalize_fn=None):
"""
Model base class
- :param dataset_input_keys: list of dataset keys used for the training
- :param dataset_shapes: a dict() of shapes
+ :param dataset_element_spec: the dataset elements specification (shape, dtype, etc). Can be retrieved from the
+ dataset instance simply with `ds.element_spec`
:param target_cropping: Optional. Number of pixels to be removed on each side of the target. This is used when
training the model and can mitigate the effects of convolution
:param inference_cropping: list of number of pixels to be removed on each side of the output during inference.
@@ -28,8 +26,12 @@ class ModelBase(abc.ABC):
:param normalize_fn: a normalization function that can be added inside the Keras model. This function takes a
dict of inputs and returns a dict of normalized inputs. Optional
"""
- self.dataset_input_keys = dataset_input_keys
- self.dataset_shapes = dataset_shapes
+ dataset_input_element_spec = dataset_element_spec[0]
+ logging.info("Dataset input element spec: %s", dataset_input_element_spec)
+ self.dataset_input_keys = list(dataset_input_element_spec)
+ logging.info("Found dataset input keys: %s", self.dataset_input_keys)
+ self.inputs_shapes = {key: dataset_input_element_spec[key].shape[1:] for key in self.dataset_input_keys}
+ logging.info("Inputs shapes: %s", self.inputs_shapes)
self.model = None
self.target_cropping = target_cropping
if inference_cropping is None:
@@ -55,10 +57,8 @@ class ModelBase(abc.ABC):
# Create Keras inputs
model_inputs = {}
for key in self.dataset_input_keys:
- shape = self.dataset_shapes[key]
- new_shape = list(shape)
- if shape[0] is None or (len(shape) > 3): # for backward comp (OTBTF<3.2.2), remove the potential batch dim
- new_shape = shape[1:]
+ new_shape = list(self.inputs_shapes[key])
+ logging.info("Original shape for input %s: %s", key, new_shape)
# Here we modify the x and y dims of >2D tensors to enable any image size at input
if len(new_shape) > 2:
new_shape[0] = None
--
GitLab
From f2c18101fb89ca1c7883f5b53a57d7b9241820dc Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 13:50:38 +0200
Subject: [PATCH 37/77] REFAC: simplify things
---
.../tensorflow_v2x/fcnn/fcnn_model.py | 11 ++++----
.../fcnn/train_from_tfrecords.py | 7 +++---
otbtf/model.py | 25 ++++++++-----------
3 files changed, 20 insertions(+), 23 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 03586b71..8188b8b3 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -88,15 +88,14 @@ def train(params, ds_train, ds_valid, ds_test):
:param ds_test: testing dataset
"""
- # Model instantiation
- # Note that the normalize_fn is now part of the model
- model = FCNNModel(dataset_element_spec=ds_train.element_spec, normalize_fn=normalize_fn)
-
# strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0")
with strategy.scope():
- # Create and compile the model
- model.create_network()
+ # Model instantiation
+ # Note that the normalize_fn is now part of the model
+ model = FCNNModel(dataset_element_spec=ds_train.element_spec, normalize_fn=normalize_fn)
+
+ # Compile the model
model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=params.learning_rate),
metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 9cd23ef6..44de20c5 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -39,9 +39,11 @@ if __name__ == "__main__":
valid_dir = os.path.join(params.tfrecords_dir, "valid")
test_dir = os.path.join(params.tfrecords_dir, "test")
+
def _tfrecords(directory):
return TFRecords(directory, preprocessing_fn=fcnn_model.preprocessing_fn)
+
# Training dataset. Must be shuffled!
assert os.path.isdir(train_dir)
ds_train = _tfrecords(train_dir).read(batch_size=params.batch_size, target_keys=["label"], shuffle_buffer_size=1000)
@@ -51,9 +53,8 @@ if __name__ == "__main__":
ds_valid = _tfrecords(valid_dir).read(batch_size=params.batch_size, target_keys=["label"])
# Test dataset (optional)
- ds_test = None
- if os.path.isdir(test_dir):
- ds_test = _tfrecords(test_dir).read(batch_size=params.batch_size, target_keys=["label"]) if os.path.isdir(test_dir)
+ ds_test = _tfrecords(test_dir).read(batch_size=params.batch_size, target_keys=["label"]) if os.path.isdir(
+ test_dir) else None
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
diff --git a/otbtf/model.py b/otbtf/model.py
index c0079033..fe74c981 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -11,43 +11,40 @@ class ModelBase(abc.ABC):
Base class for all models
"""
- def __init__(self, dataset_element_spec, target_cropping=None,
- inference_cropping=None, normalize_fn=None):
+ def __init__(self, dataset_element_spec, inference_cropping=None, normalize_fn=None):
"""
- Model base class
+ Model initializer, must be called **inside** the strategy.scope().
:param dataset_element_spec: the dataset elements specification (shape, dtype, etc). Can be retrieved from the
dataset instance simply with `ds.element_spec`
- :param target_cropping: Optional. Number of pixels to be removed on each side of the target. This is used when
- training the model and can mitigate the effects of convolution
:param inference_cropping: list of number of pixels to be removed on each side of the output during inference.
This list creates some additional outputs in the model, not used during training,
only during inference. Default [16, 32, 64, 96, 128]
:param normalize_fn: a normalization function that can be added inside the Keras model. This function takes a
dict of inputs and returns a dict of normalized inputs. Optional
"""
+ # Retrieve dataset inputs shapes
dataset_input_element_spec = dataset_element_spec[0]
logging.info("Dataset input element spec: %s", dataset_input_element_spec)
+
self.dataset_input_keys = list(dataset_input_element_spec)
logging.info("Found dataset input keys: %s", self.dataset_input_keys)
+
self.inputs_shapes = {key: dataset_input_element_spec[key].shape[1:] for key in self.dataset_input_keys}
logging.info("Inputs shapes: %s", self.inputs_shapes)
- self.model = None
- self.target_cropping = target_cropping
+
+ # Setup cropping, normalization function
if inference_cropping is None:
inference_cropping = [16, 32, 64, 96, 128]
self.inference_cropping = inference_cropping
self.normalize_fn = normalize_fn
+ # Create model
+ self.model = self.create_network()
+
def __getattr__(self, name):
"""This method is called when the default attribute access fails. We choose to try to access the attribute of
self.model. Thus, any method of keras.Model() can be used transparently, e.g. model.summary() or model.fit()"""
- if not self.model:
- logging.warning("model is None. You should call `create_network()` before using it!")
- logging.warning("Creating the neural network. Note that training could fail if using keras distribution "
- "strategy such as MirroredStrategy. Best practice is to call `create_network()` inside "
- "`with strategy.scope():`")
- self.create_network()
return getattr(self.model, name)
def get_inputs(self):
@@ -107,7 +104,7 @@ class ModelBase(abc.ABC):
# outputs.update(extra_outputs)
# Return the keras model
- self.model = keras.Model(inputs=model_inputs, outputs=outputs, name=self.__class__.__name__)
+ return keras.Model(inputs=model_inputs, outputs=outputs, name=self.__class__.__name__)
def summary(self, strategy=None):
"""
--
GitLab
From 1ac71e3621ccaeebaea293e958d9e1c17bd8bcc3 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 14:07:24 +0200
Subject: [PATCH 38/77] REFAC: simplify things
---
.../tensorflow_v2x/fcnn/fcnn_model.py | 38 +++++++-------
.../fcnn/train_from_patches-images.py | 2 +-
.../fcnn/train_from_tfrecords.py | 2 +-
otbtf/model.py | 51 +++++++++++--------
4 files changed, 50 insertions(+), 43 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 8188b8b3..d8ab4aa5 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -16,6 +16,20 @@ class FCNNModel(ModelBase):
A Simple Fully Convolutional U-Net like model
"""
+ def normalize_input(inputs):
+ """
+ The model will use this function internally to normalize its inputs, before applying the `get_outputs()`
+ function that actually builds the operations graph (convolutions, etc).
+ This function will hence work at training time and inference time.
+
+ In this example, we assume that we have an input 12 bits multispectral image with values ranging from
+ [0, 10000], that we process using a simple stretch to roughly match the [0, 1] range.
+
+ :param inputs: dict of inputs
+ :return: dict of normalized inputs, ready to be used from the `get_outputs()` function of the model
+ """
+ return {"input_xs": inputs["input_xs"] * 0.0001}
+
def get_outputs(self, normalized_inputs):
"""
This small model produces an output which has the same physical spacing as the input.
@@ -42,12 +56,12 @@ class FCNNModel(ModelBase):
# final layers
net = tf.keras.activations.softmax(net)
- net = tf.keras.layers.Cropping2D(cropping=32)(net)
+ net = tf.keras.layers.Cropping2D(cropping=32, name="predictions_softmax_tensor")(net)
return {"predictions": net}
-def preprocessing_fn(examples):
+def dataset_preprocessing_fn(examples):
"""
Preprocessing function for the training dataset.
This function is only used at training time, to put the data in the expected format.
@@ -63,21 +77,6 @@ def preprocessing_fn(examples):
"predictions": _to_categorical(examples["labels"])}
-def normalize_fn(inputs):
- """
- The model will use this function internally to normalize its input, before applying the `get_outputs()` function
- that actually builds the operations graph (convolutions, etc).
- This function will hence work at training time and inference time.
-
- In this example, we assume that we have an input 12 bits multispectral image with values ranging from [0, 10 000],
- that we process using a simple stretch to roughly match the [0, 1] range.
-
- :param inputs: dict of inputs
- :return: dict of normalized inputs, ready to be used from the `get_outputs()` function of the model
- """
- return {"input_xs": inputs["input_xs"] * 0.0001}
-
-
def train(params, ds_train, ds_valid, ds_test):
"""
Create, train, and save the model.
@@ -91,9 +90,8 @@ def train(params, ds_train, ds_valid, ds_test):
# strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0")
with strategy.scope():
- # Model instantiation
- # Note that the normalize_fn is now part of the model
- model = FCNNModel(dataset_element_spec=ds_train.element_spec, normalize_fn=normalize_fn)
+ # Model instantiation. Note that the normalize_fn is now part of the model
+ model = FCNNModel(dataset_element_spec=ds_train.element_spec)
# Compile the model
model.compile(loss=tf.keras.losses.CategoricalCrossentropy(),
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 5e4f9017..275a8099 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -73,7 +73,7 @@ def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"])
def _split_inp_target(all_inp):
# Differentiating inputs and outputs
- all_inp_prep = fcnn_model.preprocessing_fn(all_inp)
+ all_inp_prep = fcnn_model.dataset_preprocessing_fn(all_inp)
inputs = {key: value for (key, value) in all_inp_prep.items() if key not in targets_keys}
targets = {key: value for (key, value) in all_inp_prep.items() if key in targets_keys}
return inputs, targets
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 44de20c5..944b071f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
def _tfrecords(directory):
- return TFRecords(directory, preprocessing_fn=fcnn_model.preprocessing_fn)
+ return TFRecords(directory, preprocessing_fn=fcnn_model.dataset_preprocessing_fn)
# Training dataset. Must be shuffled!
diff --git a/otbtf/model.py b/otbtf/model.py
index fe74c981..d65ee73a 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -2,7 +2,7 @@
""" Base class for models"""
import abc
import logging
-from tensorflow import keras
+import tensorflow
from otbtf.utils import _is_chief
@@ -11,7 +11,7 @@ class ModelBase(abc.ABC):
Base class for all models
"""
- def __init__(self, dataset_element_spec, inference_cropping=None, normalize_fn=None):
+ def __init__(self, dataset_element_spec, inference_cropping=None):
"""
Model initializer, must be called **inside** the strategy.scope().
@@ -20,8 +20,6 @@ class ModelBase(abc.ABC):
:param inference_cropping: list of number of pixels to be removed on each side of the output during inference.
This list creates some additional outputs in the model, not used during training,
only during inference. Default [16, 32, 64, 96, 128]
- :param normalize_fn: a normalization function that can be added inside the Keras model. This function takes a
- dict of inputs and returns a dict of normalized inputs. Optional
"""
# Retrieve dataset inputs shapes
dataset_input_element_spec = dataset_element_spec[0]
@@ -37,7 +35,6 @@ class ModelBase(abc.ABC):
if inference_cropping is None:
inference_cropping = [16, 32, 64, 96, 128]
self.inference_cropping = inference_cropping
- self.normalize_fn = normalize_fn
# Create model
self.model = self.create_network()
@@ -60,7 +57,7 @@ class ModelBase(abc.ABC):
if len(new_shape) > 2:
new_shape[0] = None
new_shape[1] = None
- placeholder = keras.Input(shape=new_shape, name=key)
+ placeholder = tensorflow.keras.Input(shape=new_shape, name=key)
logging.info("New shape for input %s: %s", key, new_shape)
model_inputs.update({key: placeholder})
return model_inputs
@@ -74,9 +71,22 @@ class ModelBase(abc.ABC):
"""
raise NotImplemented("This method has to be implemented. Here you code the model :)")
+ @staticmethod
+ def normalize_inputs(inputs):
+ """
+ A normalization function that can be added inside the Keras model. This function takes the dict of inputs and
+ returns a dict of normalized inputs. Can be reimplemented depending on the needs.
+
+ :param inputs: inputs, either keras.Input or normalized_inputs
+ :return: a dict of outputs tensors of the model
+ """
+ return inputs
+
def create_network(self):
"""
- This method returns the Keras model. This needs to be called **inside** the strategy.scope()
+ This method returns the Keras model. This needs to be called **inside** the strategy.scope().
+ Can be reimplemented depending on the needs.
+
:return: the keras model
"""
@@ -84,27 +94,27 @@ class ModelBase(abc.ABC):
model_inputs = self.get_inputs()
logging.info(f"Model inputs: {model_inputs}")
- # Normalize the inputs. If some input keys are not handled by normalized_fn, these inputs are not normalized
- normalized_inputs = model_inputs.copy()
- normalized_inputs.update(self.normalize_fn(model_inputs))
+ # Normalize the inputs
+ normalized_inputs = self.normalize_inputs(model_inputs)
logging.info(f"Normalized model inputs: {normalized_inputs}")
# Build the model
outputs = self.get_outputs(normalized_inputs)
logging.info(f"Model outputs: {outputs}")
- # # Add extra outputs for inference
- # extra_outputs = {}
- # for out_key, out_tensor in outputs.items():
- # for crop in self.inference_cropping:
- # extra_output_key = cropped_tensor_name(out_key, crop)
- # extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
- # extra_output = tf.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
- # extra_outputs[extra_output_key] = extra_output
- # outputs.update(extra_outputs)
+ # Add extra outputs for inference
+ extra_outputs = {}
+ for out_key, out_tensor in outputs.items():
+ for crop in self.inference_cropping:
+ extra_output_key = cropped_tensor_name(out_key, crop)
+ extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
+ #extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
+ extra_output = tensorflow.identity(out_tensor[:, crop:crop, crop:crop, :], name=extra_output_name)
+ extra_outputs[extra_output_key] = extra_output
+ outputs.update(extra_outputs)
# Return the keras model
- return keras.Model(inputs=model_inputs, outputs=outputs, name=self.__class__.__name__)
+ return tensorflow.keras.Model(inputs=model_inputs, outputs=outputs, name=self.__class__.__name__)
def summary(self, strategy=None):
"""
@@ -116,7 +126,6 @@ class ModelBase(abc.ABC):
def plot(self, output_path, strategy=None):
"""
Enables to save a figure representing the architecture of the network.
- //!\\ only works if create_network() has been called beforehand
Needs pydot and graphviz to work (`pip install pydot` and https://graphviz.gitlab.io/download/)
"""
assert self.model, "Plot() only works if create_network() has been called beforehand"
--
GitLab
From 4d16c441d95669e5cee6b260f648cecca29ba5c8 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 17:48:13 +0200
Subject: [PATCH 39/77] FIX: ensure shape of TFRecords.read() dataset + add
some parameters in read()
---
otbtf/tfrecords.py | 25 +++++++++++++------------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 63f07592..d3d0a25c 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -101,7 +101,6 @@ class TFRecords:
:param data: Data to save json format
:param filepath: Output file name
"""
-
with open(filepath, 'w') as file:
json.dump(data, file, indent=4)
@@ -114,21 +113,21 @@ class TFRecords:
with open(filepath, 'r') as file:
return json.load(file)
- @staticmethod
- def parse_tfrecord(example, features_types, target_keys, preprocessing_fn=None, **kwargs):
+ def parse_tfrecord(self, example, target_keys, preprocessing_fn=None, **kwargs):
"""
Parse example object to sample dict.
:param example: Example object to parse
- :param features_types: List of types for each feature
:param target_keys: list of keys of the targets
:param preprocessing_fn: Optional. A preprocessing function that process the input example
:param kwargs: some keywords arguments for preprocessing_fn
"""
- read_features = {key: tf.io.FixedLenFeature([], dtype=tf.string) for key in features_types}
+ read_features = {key: tf.io.FixedLenFeature([], dtype=tf.string) for key in self.output_types}
example_parsed = tf.io.parse_single_example(example, read_features)
- for key in read_features.keys():
- example_parsed[key] = tf.io.parse_tensor(example_parsed[key], out_type=features_types[key])
+ for key, out_type in self.output_types.items():
+ example_parsed[key] = tf.io.parse_tensor(example_parsed[key], out_type=out_type)
+ for key, shape in self.output_shapes.items():
+ example_parsed[key] = tf.ensure_shape(example_parsed[key], shape)
# Differentiating inputs and outputs
example_parsed_prep = preprocessing_fn(example_parsed, **kwargs) if preprocessing_fn else example_parsed
@@ -138,7 +137,8 @@ class TFRecords:
return input_parsed, target_parsed
def read(self, batch_size, target_keys, n_workers=1, drop_remainder=True, shuffle_buffer_size=None,
- preprocessing_fn=None, **kwargs):
+ preprocessing_fn=None, shard_policy=tf.data.experimental.AutoShardPolicy.AUTO,
+ prefetch_buffer_size=tf.data.experimental.AUTOTUNE, **kwargs):
"""
Read all tfrecord files matching with pattern and convert data to tensorflow dataset.
:param batch_size: Size of tensorflow batch
@@ -161,14 +161,15 @@ class TFRecords:
preprocessing_fn should not implement such things as radiometric transformations from
input to input_preprocessed, because those are performed inside the model itself
(see `otbtf.ModelBase.normalize()`).
+ :param shard_policy: sharding policy
+ :param prefetch_buffer_size: prefetch buffer size
:param kwargs: some keywords arguments for preprocessing_fn
"""
options = tf.data.Options()
if shuffle_buffer_size:
options.experimental_deterministic = False # disable order, increase speed
- options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.AUTO # for multiworker
- parse = partial(self.parse_tfrecord, features_types=self.output_types, target_keys=target_keys,
- preprocessing_fn=preprocessing_fn, **kwargs)
+ options.experimental_distribute.auto_shard_policy = shard_policy # for multiworker
+ parse = partial(self.parse_tfrecord, target_keys=target_keys, preprocessing_fn=preprocessing_fn, **kwargs)
# TODO: to be investigated :
# 1/ num_parallel_reads useful ? I/O bottleneck of not ?
@@ -189,6 +190,6 @@ class TFRecords:
if shuffle_buffer_size:
dataset = dataset.shuffle(buffer_size=shuffle_buffer_size)
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
- dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
+ dataset = dataset.prefetch(buffer_size=prefetch_buffer_size)
return dataset
--
GitLab
From 162b8803682a890ad52ca9323fbb69daecbdc6d4 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 17:48:44 +0200
Subject: [PATCH 40/77] FIX: crop
---
otbtf/model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index d65ee73a..5b9e16c4 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -109,7 +109,7 @@ class ModelBase(abc.ABC):
extra_output_key = cropped_tensor_name(out_key, crop)
extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
#extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
- extra_output = tensorflow.identity(out_tensor[:, crop:crop, crop:crop, :], name=extra_output_name)
+ extra_output = tensorflow.identity(out_tensor[:, crop:-crop, crop:-crop, :], name=extra_output_name)
extra_outputs[extra_output_key] = extra_output
outputs.update(extra_outputs)
--
GitLab
From 2bd4c3d13e872c79ebd66817ddb18d0b468c20ff Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 17:49:04 +0200
Subject: [PATCH 41/77] FIX: create_tfrecords example
---
otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
index ebd72adb..0d10c886 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
@@ -18,7 +18,7 @@ params = parser.parse_args()
if __name__ == "__main__":
# Sort patches and labels
- patches = sorted(params.patches)
+ patches = sorted(params.xs)
labels = sorted(params.labels)
# Check patches and labels are correctly sorted
@@ -33,4 +33,4 @@ if __name__ == "__main__":
dataset = DatasetFromPatchesImages(filenames_dict={"input_xs": patches, "labels": labels})
# Convert the dataset into TFRecords
- dataset.to_tfrecords(output_dir=params.outdir)
+ dataset.to_tfrecords(output_dir=params.outdir, drop_remainder=False)
--
GitLab
From 1855dea9b47349546c5c353156e1e892cedf8476 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 17:49:44 +0200
Subject: [PATCH 42/77] REFAC: training
---
.../fcnn/train_from_patches-images.py | 33 -------------------
.../fcnn/train_from_tfrecords.py | 14 ++++----
2 files changed, 6 insertions(+), 41 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 275a8099..93eb2bcb 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -1,38 +1,5 @@
"""
This example shows how to use the otbtf python API to train a deep net from patches-images.
-
-We expect that the files are stored in the following way, with M, N and K denoting respectively
-the number of patches-images in the training, validation, and test datasets:
-
-/dataset_dir
- /train
- /image_1
- ..._xs.tif
- ..._labels.tif
- /image_2
- ..._xs.tif
- ..._labels.tif
- ...
- /image_M
- ..._xs.tif
- ..._labels.tif
- /valid
- /image_1
- ..._xs.tif
- ..._labels.tif
- ...
- /image_N
- ..._xs.tif
- ..._labels.tif
- /test
- /image_1
- ..._xs.tif
- ..._labels.tif
- ...
- /image_K
- ..._xs.tif
- ..._labels.tif
-
"""
import helper
from otbtf import DatasetFromPatchesImages
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index 944b071f..e355d324 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -39,22 +39,20 @@ if __name__ == "__main__":
valid_dir = os.path.join(params.tfrecords_dir, "valid")
test_dir = os.path.join(params.tfrecords_dir, "test")
-
- def _tfrecords(directory):
- return TFRecords(directory, preprocessing_fn=fcnn_model.dataset_preprocessing_fn)
-
+ kwargs = {"batch_size": params.batch_size,
+ "target_keys": ["predictions"],
+ "preprocessing_fn": fcnn_model.dataset_preprocessing_fn}
# Training dataset. Must be shuffled!
assert os.path.isdir(train_dir)
- ds_train = _tfrecords(train_dir).read(batch_size=params.batch_size, target_keys=["label"], shuffle_buffer_size=1000)
+ ds_train = TFRecords(train_dir).read(shuffle_buffer_size=1000, **kwargs)
# Validation dataset
assert os.path.isdir(valid_dir)
- ds_valid = _tfrecords(valid_dir).read(batch_size=params.batch_size, target_keys=["label"])
+ ds_valid = TFRecords(valid_dir).read(**kwargs)
# Test dataset (optional)
- ds_test = _tfrecords(test_dir).read(batch_size=params.batch_size, target_keys=["label"]) if os.path.isdir(
- test_dir) else None
+ ds_test = TFRecords(test_dir).read(**kwargs) if os.path.isdir(test_dir) else None
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
--
GitLab
From 855dff7c31c84e6f000d5c9c1b2253af93679db2 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 19:21:19 +0200
Subject: [PATCH 43/77] FIX: keras module
---
otbtf/model.py | 15 ++++++++-------
otbtf/utils.py | 4 ++--
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 5b9e16c4..1b3dff41 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -69,7 +69,7 @@ class ModelBase(abc.ABC):
:param inputs: inputs, either keras.Input or normalized_inputs
:return: a dict of outputs tensors of the model
"""
- raise NotImplemented("This method has to be implemented. Here you code the model :)")
+ raise NotImplementedError("This method has to be implemented. Here you code the model :)")
@staticmethod
def normalize_inputs(inputs):
@@ -92,15 +92,15 @@ class ModelBase(abc.ABC):
# Get the model inputs
model_inputs = self.get_inputs()
- logging.info(f"Model inputs: {model_inputs}")
+ logging.info("Model inputs: %s", model_inputs)
# Normalize the inputs
normalized_inputs = self.normalize_inputs(model_inputs)
- logging.info(f"Normalized model inputs: {normalized_inputs}")
+ logging.info(f"Normalized model inputs: %", normalized_inputs)
# Build the model
outputs = self.get_outputs(normalized_inputs)
- logging.info(f"Model outputs: {outputs}")
+ logging.info(f"Model outputs: %s", outputs)
# Add extra outputs for inference
extra_outputs = {}
@@ -108,7 +108,7 @@ class ModelBase(abc.ABC):
for crop in self.inference_cropping:
extra_output_key = cropped_tensor_name(out_key, crop)
extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
- #extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
+ # extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
extra_output = tensorflow.identity(out_tensor[:, crop:-crop, crop:-crop, :], name=extra_output_name)
extra_outputs[extra_output_key] = extra_output
outputs.update(extra_outputs)
@@ -136,8 +136,9 @@ class ModelBase(abc.ABC):
# This model is only used for plotting the architecture thanks to `keras.utils.plot_model`
inputs = self.get_inputs() # inputs without normalization
outputs = self.get_outputs(inputs) # raw model outputs
- model_simplified = keras.Model(inputs=inputs, outputs=outputs, name=self.__class__.__name__ + '_simplified')
- keras.utils.plot_model(model_simplified, output_path)
+ model_simplified = tensorflow.keras.Model(inputs=inputs, outputs=outputs,
+ name=self.__class__.__name__ + '_simplified')
+ tensorflow.keras.utils.plot_model(model_simplified, output_path)
def cropped_tensor_name(tensor_name, crop):
diff --git a/otbtf/utils.py b/otbtf/utils.py
index 0cc3b4a4..989db35e 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -85,5 +85,5 @@ def _is_chief(strategy):
if strategy.cluster_resolver: # this means MultiWorkerMirroredStrategy
task_type, task_id = strategy.cluster_resolver.task_type, strategy.cluster_resolver.task_id
return (task_type == 'chief') or (task_type == 'worker' and task_id == 0) or task_type is None
- else: # strategy with only one worker
- return True
\ No newline at end of file
+ # strategy with only one worker
+ return True
--
GitLab
From 58d40f0759397ab69e6a001be0233bda35e19f2d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 21:01:02 +0200
Subject: [PATCH 44/77] WIP: a few fixes
---
.../tensorflow_v2x/fcnn/fcnn_model.py | 36 ++++++++++---------
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 2 +-
otbtf/model.py | 9 ++---
3 files changed, 26 insertions(+), 21 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index d8ab4aa5..9a2740b1 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -6,7 +6,6 @@ import tensorflow as tf
import tensorflow.keras.layers as layers
import logging
-
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
N_CLASSES = 6
@@ -16,7 +15,7 @@ class FCNNModel(ModelBase):
A Simple Fully Convolutional U-Net like model
"""
- def normalize_input(inputs):
+ def normalize_inputs(self, inputs):
"""
The model will use this function internally to normalize its inputs, before applying the `get_outputs()`
function that actually builds the operations graph (convolutions, etc).
@@ -40,22 +39,26 @@ class FCNNModel(ModelBase):
"""
# Model input
- net = normalized_inputs["input_xs"]
+ norm_inp = normalized_inputs["input_xs"]
# Encoder
- convs_depth = {"conv1": 16, "conv2": 32, "conv3": 64, "conv4": 64}
- for name, depth in convs_depth.items():
- conv = layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)
- net = conv(net)
+ def _conv(inp, depth, name):
+ return layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)(inp)
+
+ def _tconv(inp, depth, name, activation="relu"):
+ return layers.Conv2DTranspose(filters=depth, kernel_size=3, activation=activation, name=name)(inp)
- # Decoder
- tconvs_depths = {"tconv1": 64, "tconv2": 32, "tconv3": 16, "tconv4": N_CLASSES}
- for name, depth in tconvs_depths.items():
- tconv = layers.Conv2DTranspose(filters=depth, kernel_size=3, activation="relu", name=name)
- net = tconv(net)
+ out_conv1 = _conv(norm_inp, 16, "conv1")
+ out_conv2 = _conv(out_conv1, 32, "conv2")
+ out_conv3 = _conv(out_conv2, 64, "conv3")
+ out_conv4 = _conv(out_conv3, 64, "conv4")
+ out_tconv1 = _tconv(out_conv4, 64, "tconv1") + out_conv3
+ out_tconv2 = _tconv(out_tconv1, 32, "tconv2") + out_conv2
+ out_tconv3 = _tconv(out_tconv2, 16, "tconv3") + out_conv1
+ out_tconv4 = _tconv(out_tconv3, N_CLASSES, "classifier", None)
# final layers
- net = tf.keras.activations.softmax(net)
+ net = tf.keras.activations.softmax(out_tconv4)
net = tf.keras.layers.Cropping2D(cropping=32, name="predictions_softmax_tensor")(net)
return {"predictions": net}
@@ -65,14 +68,16 @@ def dataset_preprocessing_fn(examples):
"""
Preprocessing function for the training dataset.
This function is only used at training time, to put the data in the expected format.
- DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see `otbtf.ModelBase.normalize_fn` for that).
+ DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see `otbtf.ModelBase.normalize_inputs` for that).
Note that this function is not called here, but in the code that prepares the datasets.
:param examples: dict for examples (i.e. inputs and targets stored in a single dict)
:return: preprocessed examples
"""
+
def _to_categorical(x):
return tf.one_hot(tf.squeeze(x, axis=-1), depth=N_CLASSES)
+
return {"input_xs": examples["input_xs"],
"predictions": _to_categorical(examples["labels"])}
@@ -87,8 +92,7 @@ def train(params, ds_train, ds_valid, ds_test):
:param ds_test: testing dataset
"""
- # strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
- strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0")
+ strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
with strategy.scope():
# Model instantiation. Note that the normalize_fn is now part of the model
model = FCNNModel(dataset_element_spec=ds_train.element_spec)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index 86226c26..7426de0f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -12,7 +12,7 @@ def base_parser():
"""
parser = argparse.ArgumentParser(description="Train a FCNN model")
parser.add_argument("--batch_size", type=int, default=8, help="Batch size")
- parser.add_argument("--learning_rate", type=float, default=0.00001, help="Learning rate")
+ parser.add_argument("--learning_rate", type=float, default=0.0001, help="Learning rate")
parser.add_argument("--nb_epochs", type=int, default=100, help="Number of epochs")
parser.add_argument("--model_dir", required=True, help="Path to save model")
return parser
diff --git a/otbtf/model.py b/otbtf/model.py
index 1b3dff41..becd26a8 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -71,8 +71,7 @@ class ModelBase(abc.ABC):
"""
raise NotImplementedError("This method has to be implemented. Here you code the model :)")
- @staticmethod
- def normalize_inputs(inputs):
+ def normalize_inputs(self, inputs):
"""
A normalization function that can be added inside the Keras model. This function takes the dict of inputs and
returns a dict of normalized inputs. Can be reimplemented depending on the needs.
@@ -80,6 +79,8 @@ class ModelBase(abc.ABC):
:param inputs: inputs, either keras.Input or normalized_inputs
:return: a dict of outputs tensors of the model
"""
+ logging.warning("normalize_input() undefined. No normalization of the model inputs will be performed. "
+ "You can implement the function in your model class")
return inputs
def create_network(self):
@@ -96,11 +97,11 @@ class ModelBase(abc.ABC):
# Normalize the inputs
normalized_inputs = self.normalize_inputs(model_inputs)
- logging.info(f"Normalized model inputs: %", normalized_inputs)
+ logging.info("Normalized model inputs: %s", normalized_inputs)
# Build the model
outputs = self.get_outputs(normalized_inputs)
- logging.info(f"Model outputs: %s", outputs)
+ logging.info("Model outputs: %s", outputs)
# Add extra outputs for inference
extra_outputs = {}
--
GitLab
From 57400b883716c7efd3e3bf4ddc06ecd94d59ed2f Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 23:33:24 +0200
Subject: [PATCH 45/77] FIX: cropping
---
otbtf/model.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index becd26a8..d5d89914 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -32,9 +32,8 @@ class ModelBase(abc.ABC):
logging.info("Inputs shapes: %s", self.inputs_shapes)
# Setup cropping, normalization function
- if inference_cropping is None:
- inference_cropping = [16, 32, 64, 96, 128]
- self.inference_cropping = inference_cropping
+ self.inference_cropping = [16, 32, 64, 96, 128] if not inference_cropping else inference_cropping
+ logging.info("Inference cropping values: %s", self.inference_cropping)
# Create model
self.model = self.create_network()
@@ -109,9 +108,14 @@ class ModelBase(abc.ABC):
for crop in self.inference_cropping:
extra_output_key = cropped_tensor_name(out_key, crop)
extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
+ logging.info("Adding extra output for tensor %s with crop %s (%s)", out_key, crop, extra_output_name)
+ # Does not work anymore when crop > patch size:
# extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
- extra_output = tensorflow.identity(out_tensor[:, crop:-crop, crop:-crop, :], name=extra_output_name)
- extra_outputs[extra_output_key] = extra_output
+ # Works when crop > patch size but we lose tensors names:
+ # extra_output = tensorflow.identity(out_tensor[:, crop:-crop, crop:-crop, :], name=extra_output_name)
+ slice = out_tensor[:, crop:-crop, crop:-crop, :]
+ identity = tensorflow.keras.layers.Activation('linear', name=extra_output_name)
+ extra_outputs[extra_output_key] = identity(slice)
outputs.update(extra_outputs)
# Return the keras model
--
GitLab
From 6464a901967a149adaf2ae1367df24fbc773d150 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 23:35:21 +0200
Subject: [PATCH 46/77] DOC: explain steps
---
otbtf/tfrecords.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index d3d0a25c..9799766e 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -124,13 +124,18 @@ class TFRecords:
read_features = {key: tf.io.FixedLenFeature([], dtype=tf.string) for key in self.output_types}
example_parsed = tf.io.parse_single_example(example, read_features)
+ # Tensor with right data type
for key, out_type in self.output_types.items():
example_parsed[key] = tf.io.parse_tensor(example_parsed[key], out_type=out_type)
+
+ # Ensure shape
for key, shape in self.output_shapes.items():
example_parsed[key] = tf.ensure_shape(example_parsed[key], shape)
- # Differentiating inputs and outputs
+ # Preprocessing
example_parsed_prep = preprocessing_fn(example_parsed, **kwargs) if preprocessing_fn else example_parsed
+
+ # Differentiating inputs and targets
input_parsed = {key: value for (key, value) in example_parsed_prep.items() if key not in target_keys}
target_parsed = {key: value for (key, value) in example_parsed_prep.items() if key in target_keys}
--
GitLab
From 28b450b143c0a37c2c7c95c83bd1059a2d14b41e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 20 Jul 2022 23:37:48 +0200
Subject: [PATCH 47/77] DOC: explain
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 9a2740b1..a19797d5 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -38,10 +38,8 @@ class FCNNModel(ModelBase):
:return: activation values
"""
- # Model input
norm_inp = normalized_inputs["input_xs"]
- # Encoder
def _conv(inp, depth, name):
return layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)(inp)
@@ -95,6 +93,7 @@ def train(params, ds_train, ds_valid, ds_test):
strategy = tf.distribute.MirroredStrategy() # For single or multi-GPUs
with strategy.scope():
# Model instantiation. Note that the normalize_fn is now part of the model
+ # It is mandatory to instantiate the model inside the strategy scope.
model = FCNNModel(dataset_element_spec=ds_train.element_spec)
# Compile the model
--
GitLab
From c532d56a61bc98e02b0c11661cd8c4fcfa935493 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 10:05:02 +0200
Subject: [PATCH 48/77] ENH: enhance Dataset.dataset so it can be used more
easilly with keras
---
otbtf/dataset.py | 30 +++++++++-
.../tensorflow_v2x/fcnn/fcnn_model.py | 35 ++++++++----
.../fcnn/train_from_patches-images.py | 18 ++----
otbtf/model.py | 57 +++++++++++++------
otbtf/tfrecords.py | 2 +-
5 files changed, 99 insertions(+), 43 deletions(-)
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index 2fa93498..b7ca2025 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -462,17 +462,41 @@ class Dataset:
for _ in range(self.size):
yield self.read_one_sample()
- def get_tf_dataset(self, batch_size, drop_remainder=True):
+ def get_tf_dataset(self, batch_size, drop_remainder=True, preprocessing_fn=None, targets_keys=None):
"""
Returns a TF dataset, ready to be used with the provided batch size
:param batch_size: the batch size
:param drop_remainder: drop incomplete batches
+ :param preprocessing_fn: Optional. A preprocessing function that takes input examples as args and returns the
+ preprocessed input examples. Typically, examples are composed of model inputs and
+ targets. Model inputs and model targets must be computed accordingly to (1) what the
+ model outputs and (2) what training loss needs. For instance, for a classification
+ problem, the model will likely output the softmax, or activation neurons, for each
+ class, and the cross entropy loss requires labels in one hot encoding. In this case,
+ the preprocessing_fn has to transform the labels values (integer ranging from
+ [0, n_classes]) in one hot encoding (vector of 0 and 1 of length n_classes). The
+ preprocessing_fn should not implement such things as radiometric transformations from
+ input to input_preprocessed, because those are performed inside the model itself
+ (see `otbtf.ModelBase.normalize_inputs()`).
+ :param targets_keys: Optional. When provided, the dataset returns a tuple of dicts (inputs_dict, target_dict) so
+ it can be straightforwardly used with keras models objects.
:return: The TF dataset
"""
- if batch_size <= 2 * self.miner_buffer.max_length:
+ if 2 * batch_size >= self.miner_buffer.max_length:
logging.warning("Batch size is %s but dataset buffer has %s elements. Consider using a larger dataset "
"buffer to avoid I/O bottleneck", batch_size, self.miner_buffer.max_length)
- return self.tf_dataset.batch(batch_size, drop_remainder=drop_remainder)
+ tf_ds = self.tf_dataset.map(preprocessing_fn) if preprocessing_fn else self.tf_dataset
+
+ if targets_keys:
+ def _split_input_and_target(example):
+ # Differentiating inputs and outputs for keras
+ inputs = {key: value for (key, value) in example.items() if key not in targets_keys}
+ targets = {key: value for (key, value) in example.items() if key in targets_keys}
+ return inputs, targets
+
+ tf_ds = tf_ds.map(_split_input_and_target)
+
+ return tf_ds.batch(batch_size, drop_remainder=drop_remainder)
def get_total_wait_in_seconds(self):
"""
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index a19797d5..0557df66 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -3,7 +3,6 @@ Implementation of a small U-Net like model
"""
from otbtf.model import ModelBase
import tensorflow as tf
-import tensorflow.keras.layers as layers
import logging
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
@@ -17,6 +16,8 @@ class FCNNModel(ModelBase):
def normalize_inputs(self, inputs):
"""
+ Inherits from `ModelBase`
+
The model will use this function internally to normalize its inputs, before applying the `get_outputs()`
function that actually builds the operations graph (convolutions, etc).
This function will hence work at training time and inference time.
@@ -31,6 +32,8 @@ class FCNNModel(ModelBase):
def get_outputs(self, normalized_inputs):
"""
+ Inherits from `ModelBase`
+
This small model produces an output which has the same physical spacing as the input.
The model generates [1 x 1 x N_CLASSES] output pixel for [32 x 32 x <nb channels>] input pixels.
@@ -41,10 +44,10 @@ class FCNNModel(ModelBase):
norm_inp = normalized_inputs["input_xs"]
def _conv(inp, depth, name):
- return layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)(inp)
+ return tf.keras.layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)(inp)
def _tconv(inp, depth, name, activation="relu"):
- return layers.Conv2DTranspose(filters=depth, kernel_size=3, activation=activation, name=name)(inp)
+ return tf.keras.layers.Conv2DTranspose(filters=depth, kernel_size=3, activation=activation, name=name)(inp)
out_conv1 = _conv(norm_inp, 16, "conv1")
out_conv2 = _conv(out_conv1, 32, "conv2")
@@ -55,17 +58,29 @@ class FCNNModel(ModelBase):
out_tconv3 = _tconv(out_tconv2, 16, "tconv3") + out_conv1
out_tconv4 = _tconv(out_tconv3, N_CLASSES, "classifier", None)
- # final layers
- net = tf.keras.activations.softmax(out_tconv4)
- net = tf.keras.layers.Cropping2D(cropping=32, name="predictions_softmax_tensor")(net)
-
- return {"predictions": net}
+ # Generally it is a good thing to name the final layers of the network (i.e. the layers of which outputs are
+ # returned from the `MyModel.get_output()` method).
+ # Indeed this enables to retrieve them for inference time, using their name.
+ # In case your forgot to name the last layers, it is still possible to look at the model outputs using the
+ # `saved_model_cli show --dir /path/to/your/savedmodel --all` command.
+ #
+ # Do not confuse **the name of the output layers** (i.e. the "name" property of the tf.keras.layer that is used
+ # to generate an output tensor) and **the key of the output tensor**, in the dict returned from the
+ # `MyModel.get_output()` method. They are two identifiers with a different purpose:
+ # - the output layer name is used only at inference time, to identify the output tensor from which generate
+ # the output image,
+ # - the output tensor key identifies the output tensors, mainly to fit the targets to model outputs during
+ # training process, but it can also be used to access the tensors as tf/keras objects, for instance to
+ # display previews images in TensorBoard.
+ predictions = tf.keras.layers.Softmax(name="predictions_softmax_tensor")(out_tconv4)
+
+ return {"predictions": predictions}
def dataset_preprocessing_fn(examples):
"""
Preprocessing function for the training dataset.
- This function is only used at training time, to put the data in the expected format.
+ This function is only used at training time, to put the data in the expected format for the training step.
DO NOT USE THIS FUNCTION TO NORMALIZE THE INPUTS ! (see `otbtf.ModelBase.normalize_inputs` for that).
Note that this function is not called here, but in the code that prepares the datasets.
@@ -76,7 +91,7 @@ def dataset_preprocessing_fn(examples):
def _to_categorical(x):
return tf.one_hot(tf.squeeze(x, axis=-1), depth=N_CLASSES)
- return {"input_xs": examples["input_xs"],
+ return {"input_xs": examples["input_xs"][32:-32, 32:-32, :],
"predictions": _to_categorical(examples["labels"])}
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index 93eb2bcb..f534d9d6 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -36,26 +36,20 @@ def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"])
# However, this can slow down your process since the patches are read on-the-fly on the filesystem.
# Good when one batch computation is slower than one batch gathering.
ds = DatasetFromPatchesImages(filenames_dict={"input_xs": xs_filenames, "labels": labels_filenames})
- tf_ds = ds.get_tf_dataset(batch_size=params.batch_size)
+ tf_ds = ds.get_tf_dataset(batch_size=params.batch_size, preprocessing_fn=fcnn_model.dataset_preprocessing_fn,
+ targets_keys=targets_keys)
- def _split_inp_target(all_inp):
- # Differentiating inputs and outputs
- all_inp_prep = fcnn_model.dataset_preprocessing_fn(all_inp)
- inputs = {key: value for (key, value) in all_inp_prep.items() if key not in targets_keys}
- targets = {key: value for (key, value) in all_inp_prep.items() if key in targets_keys}
- return inputs, targets
-
- return ds, tf_ds.map(_split_inp_target)
+ return tf_ds
if __name__ == "__main__":
params = parser.parse_args()
- _, ds_train = create_dataset(params.train_xs, params.train_labels)
- _, ds_valid = create_dataset(params.valid_xs, params.valid_labels)
+ ds_train = create_dataset(params.train_xs, params.train_labels)
+ ds_valid = create_dataset(params.valid_xs, params.valid_labels)
ds_test = None
if params.test_xs and params.test_labels:
- _, ds_test = create_dataset(params.test_xs, params.test_labels)
+ ds_test = create_dataset(params.test_xs, params.test_labels)
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
diff --git a/otbtf/model.py b/otbtf/model.py
index d5d89914..98419276 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -62,26 +62,41 @@ class ModelBase(abc.ABC):
return model_inputs
@abc.abstractmethod
- def get_outputs(self, inputs):
+ def get_outputs(self, normalized_inputs):
"""
- Implementation of the model
- :param inputs: inputs, either keras.Input or normalized_inputs
- :return: a dict of outputs tensors of the model
+ Implementation of the model, from the normalized inputs.
+
+ :param normalized_inputs: normalized inputs, as generated from `self.normalize_inputs()`
+ :return: dict of model outputs
"""
raise NotImplementedError("This method has to be implemented. Here you code the model :)")
def normalize_inputs(self, inputs):
"""
- A normalization function that can be added inside the Keras model. This function takes the dict of inputs and
- returns a dict of normalized inputs. Can be reimplemented depending on the needs.
+ Normalize the model inputs.
+ Takes the dict of inputs and returns a dict of normalized inputs.
- :param inputs: inputs, either keras.Input or normalized_inputs
- :return: a dict of outputs tensors of the model
+ :param inputs: model inputs
+ :return: a dict of normalized model inputs
"""
logging.warning("normalize_input() undefined. No normalization of the model inputs will be performed. "
- "You can implement the function in your model class")
+ "You can implement the function in your model class if you want.")
return inputs
+ def postprocess_outputs(self, outputs, inputs=None, normalized_inputs=None):
+ """
+ Post-process the model outputs.
+ Takes the dicts of inputs and outputs, and returns a dict of post-processed outputs.
+
+ :param outputs: dict of model outputs
+ :param inputs: dict of model inputs (optional)
+ :param normalized_inputs: dict of normalized model inputs (optional)
+ :return: a dict of post-processed model outputs
+ """
+ logging.warning("postprocess_outputs() undefined. No post-processing of the model inputs will be performed. "
+ "You can implement the function in your model class if you want.")
+ return outputs
+
def create_network(self):
"""
This method returns the Keras model. This needs to be called **inside** the strategy.scope().
@@ -91,35 +106,43 @@ class ModelBase(abc.ABC):
"""
# Get the model inputs
- model_inputs = self.get_inputs()
- logging.info("Model inputs: %s", model_inputs)
+ inputs = self.get_inputs()
+ logging.info("Model inputs: %s", inputs)
# Normalize the inputs
- normalized_inputs = self.normalize_inputs(model_inputs)
+ normalized_inputs = self.normalize_inputs(inputs=inputs)
logging.info("Normalized model inputs: %s", normalized_inputs)
# Build the model
- outputs = self.get_outputs(normalized_inputs)
+ outputs = self.get_outputs(normalized_inputs=normalized_inputs)
logging.info("Model outputs: %s", outputs)
+ # Post-processing for inference
+ postprocessed_outputs = self.postprocess_outputs(outputs=outputs, inputs=inputs,
+ normalized_inputs=normalized_inputs)
+
# Add extra outputs for inference
extra_outputs = {}
- for out_key, out_tensor in outputs.items():
+ for out_key, out_tensor in postprocessed_outputs.items():
for crop in self.inference_cropping:
extra_output_key = cropped_tensor_name(out_key, crop)
extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
logging.info("Adding extra output for tensor %s with crop %s (%s)", out_key, crop, extra_output_name)
# Does not work anymore when crop > patch size:
# extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
- # Works when crop > patch size but we lose tensors names:
+ # Works when crop > patch size, but we lose tensors names:
# extra_output = tensorflow.identity(out_tensor[:, crop:-crop, crop:-crop, :], name=extra_output_name)
+ # Works when crop > patch size, but doesnt work when len(self.inference_cropping) > 1!
+ # extra_output = tensorflow.keras.layers.Lambda(x: x[:, crop:-crop, crop:-crop, :],
+ # name=extra_output_name)(out_tensor)
slice = out_tensor[:, crop:-crop, crop:-crop, :]
identity = tensorflow.keras.layers.Activation('linear', name=extra_output_name)
extra_outputs[extra_output_key] = identity(slice)
- outputs.update(extra_outputs)
+ postprocessed_outputs.update(extra_outputs)
+ outputs.update(postprocessed_outputs)
# Return the keras model
- return tensorflow.keras.Model(inputs=model_inputs, outputs=outputs, name=self.__class__.__name__)
+ return tensorflow.keras.Model(inputs=inputs, outputs=outputs, name=self.__class__.__name__)
def summary(self, strategy=None):
"""
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 9799766e..d04e9fcb 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -165,7 +165,7 @@ class TFRecords:
[0, n_classes]) in one hot encoding (vector of 0 and 1 of length n_classes). The
preprocessing_fn should not implement such things as radiometric transformations from
input to input_preprocessed, because those are performed inside the model itself
- (see `otbtf.ModelBase.normalize()`).
+ (see `otbtf.ModelBase.normalize_inputs()`).
:param shard_policy: sharding policy
:param prefetch_buffer_size: prefetch buffer size
:param kwargs: some keywords arguments for preprocessing_fn
--
GitLab
From 39f4e845331cf0fae7e162e82ab69b41814848b2 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 11:23:31 +0200
Subject: [PATCH 49/77] ENH: assert file lists not empty
---
otbtf/examples/tensorflow_v2x/fcnn/helper.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/helper.py b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
index 7426de0f..aea3a0ac 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/helper.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/helper.py
@@ -6,7 +6,7 @@ import argparse
def base_parser():
"""
- Create a parser with the base parameters
+ Create a parser with the base parameters for the training applications
:return: argparse.ArgumentParser instance
"""
@@ -26,6 +26,8 @@ def check_files_order(files1, files2):
:param files1: list of filenames (str)
:param files2: list of filenames (str)
"""
+ assert files1
+ assert files2
assert len(files1) == len(files2)
def get_basename(n):
--
GitLab
From 3df2435a731a7dd0ed699ea039925c0cc531f86c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 11:23:56 +0200
Subject: [PATCH 50/77] REFAC: simplify code, DOC: explain
---
.../fcnn/train_from_patches-images.py | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
index f534d9d6..aa07e8a5 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
@@ -22,7 +22,7 @@ parser.add_argument("--test_labels", required=False, nargs="+", default=[],
def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"]):
"""
- Create an otbtf.DatasetFromPatchesImages
+ Returns a TF dataset generated from an `otbtf.DatasetFromPatchesImages` instance
"""
# Sort patches and labels
xs_filenames.sort()
@@ -32,10 +32,15 @@ def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"])
helper.check_files_order(xs_filenames, labels_filenames)
# Create dataset from the filename dict
- # You can add the use_streaming option here, is you want to lower the memory budget.
+ # You can add the `use_streaming` option here, is you want to lower the memory budget.
# However, this can slow down your process since the patches are read on-the-fly on the filesystem.
- # Good when one batch computation is slower than one batch gathering.
+ # Good when one batch computation is slower than one batch gathering!
+ # You can also use a custom `Iterator` of your own (default is `RandomIterator`). See `otbtf.dataset.Iterator`.
ds = DatasetFromPatchesImages(filenames_dict={"input_xs": xs_filenames, "labels": labels_filenames})
+
+ # We generate the TF dataset, and we use a preprocessing option to put the labels into one hot encoding (see the
+ # `fcnn_model.dataset_preprocessing_fn` function). Also, we set the `target_keys` parameter to ask the dataset to
+ # deliver samples in the form expected by keras, i.e. a tuple of dicts (inputs_dict, target_dict).
tf_ds = ds.get_tf_dataset(batch_size=params.batch_size, preprocessing_fn=fcnn_model.dataset_preprocessing_fn,
targets_keys=targets_keys)
@@ -45,11 +50,10 @@ def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"])
if __name__ == "__main__":
params = parser.parse_args()
+ # Create TF datasets
ds_train = create_dataset(params.train_xs, params.train_labels)
ds_valid = create_dataset(params.valid_xs, params.valid_labels)
- ds_test = None
- if params.test_xs and params.test_labels:
- ds_test = create_dataset(params.test_xs, params.test_labels)
+ ds_test = create_dataset(params.test_xs, params.test_labels) if params.test_xs else None
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
--
GitLab
From 02e93ceb5904e3d2d9f4857328d6b7dde9015a00 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 11:24:29 +0200
Subject: [PATCH 51/77] DOC: explain
---
otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index e355d324..e3d4d327 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -29,7 +29,7 @@ import fcnn_model
parser = helper.base_parser()
parser.add_argument("--tfrecords_dir", required=True,
- help="Directory of subdirs containing TFRecords files: train, valid(, test)")
+ help="Directory containing train, valid(, test) folders of TFRecords files")
if __name__ == "__main__":
params = parser.parse_args()
@@ -43,7 +43,7 @@ if __name__ == "__main__":
"target_keys": ["predictions"],
"preprocessing_fn": fcnn_model.dataset_preprocessing_fn}
- # Training dataset. Must be shuffled!
+ # Training dataset. Must be shuffled
assert os.path.isdir(train_dir)
ds_train = TFRecords(train_dir).read(shuffle_buffer_size=1000, **kwargs)
--
GitLab
From db60cf067bcd8b717c24fc4c02b982a99fb381e8 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 14:38:06 +0200
Subject: [PATCH 52/77] CI: new tests for the python API
---
.gitlab-ci.yml | 12 +++++++++++-
...patches-images.py => train_from_patchesimages.py} | 0
2 files changed, 11 insertions(+), 1 deletion(-)
rename otbtf/examples/tensorflow_v2x/fcnn/{train_from_patches-images.py => train_from_patchesimages.py} (100%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7f713d76..e0cb8a34 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,6 +7,8 @@ variables:
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
ARTIFACT_TEST_DIR: $CI_PROJECT_DIR/testing
CRC_BOOK_TMP: /tmp/crc_book_tests_tmp
+ API_TEST_TMP: /tmp/api_tests_tmp
+ DATADIR: $CI_PROJECT_DIR/test/data
DOCKER_BUILDKIT: 1
DOCKER_DRIVER: overlay2
CACHE_IMAGE_BASE: $CI_REGISTRY_IMAGE:otbtf-base
@@ -140,7 +142,7 @@ crc_book:
extends: .applications_test_base
script:
- mkdir -p $CRC_BOOK_TMP
- - TMPDIR=$CRC_BOOK_TMP DATADIR=$CI_PROJECT_DIR/test/data python -m pytest --junitxml=$CI_PROJECT_DIR/report_tutorial.xml $OTBTF_SRC/test/tutorial_unittest.py
+ - TMPDIR=$CRC_BOOK_TMP python -m pytest --junitxml=$CI_PROJECT_DIR/report_tutorial.xml $OTBTF_SRC/test/tutorial_unittest.py
after_script:
- cp $CRC_BOOK_TMP/*.* $ARTIFACT_TEST_DIR/
@@ -157,6 +159,14 @@ sr4rs:
- export PYTHONPATH=$PYTHONPATH:$PWD/sr4rs
- python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_sr4rs.xml $OTBTF_SRC/test/sr4rs_unittest.py
+otbtf_api:
+ extends: .applications_test_base
+ script:
+ - mkdir $API_TEST_TMP
+ - TMPDIR=$API_TEST_TMP python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_api.xml $OTBTF_SRC/test/api_unittest.py
+ after_script:
+ - cp $API_TEST_TMP/*.* $ARTIFACT_TEST_DIR/
+
deploy_cpu-dev-testing:
stage: Update dev image
extends: .docker_build_base
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
similarity index 100%
rename from otbtf/examples/tensorflow_v2x/fcnn/train_from_patches-images.py
rename to otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
--
GitLab
From ea8494da6faa770af909b351c3cdbf783513cf6a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 14:38:27 +0200
Subject: [PATCH 53/77] ADD: new tests for the python API
---
test/api_unittest.py | 136 +++++++++++++++++++++++++++
test/data/classif_model4_softmax.tif | 3 +
test/test_utils.py | 65 +++++++++++--
test/tutorial_unittest.py | 55 +----------
4 files changed, 195 insertions(+), 64 deletions(-)
create mode 100644 test/api_unittest.py
create mode 100644 test/data/classif_model4_softmax.tif
diff --git a/test/api_unittest.py b/test/api_unittest.py
new file mode 100644
index 00000000..29582489
--- /dev/null
+++ b/test/api_unittest.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import pytest
+import unittest
+from test_utils import resolve_paths, files_exist, run_command_and_compare
+from otbtf.examples.tensorflow_v2x.fcnn.fcnn_model import INPUT_NAME, OUTPUT_SOFTMAX_NAME
+from otbtf.examples.tensorflow_v2x.fcnn import train_from_patchesimages
+from otbtf.examples.tensorflow_v2x.fcnn import train_from_tfrecords
+from otbtf.examples.tensorflow_v2x.fcnn import create_tfrecords
+from otbtf.model import cropped_tensor_name
+
+INFERENCE_MAE_TOL = 10.0 # Dummy value: we don't really care of the mae value but rather the image size etc
+
+
+class APITest(unittest.TestCase):
+
+ @pytest.mark.order(1)
+ def test_train_from_patchesimages(self):
+ params = train_from_patchesimages.parser.parse_args(['--model_dir', resolve_paths('$TMPDIR/model_from_pimg'),
+ '--nb_epochs', '1',
+ '--train_xs',
+ resolve_paths('$DATADIR/amsterdam_patches_A.tif'),
+ '--train_labels',
+ resolve_paths('$DATADIR/amsterdam_labels_A.tif'),
+ '--valid_xs',
+ resolve_paths('$DATADIR/amsterdam_patches_B.tif'),
+ '--valid_labels',
+ resolve_paths('$DATADIR/amsterdam_labels_B.tif')])
+ train_from_patchesimages.train(params=params)
+ self.assertTrue(files_exist(['$TMPDIR/model_from_pimg/keras_metadata.pb',
+ '$TMPDIR/model_from_pimg/saved_model.pb',
+ '$TMPDIR/model_from_pimg/variables/variables.data-00000-of-00001',
+ '$TMPDIR/model_from_pimg/variables/variables.index']))
+
+ @pytest.mark.order(2)
+ def test_model_inference1(self):
+ self.assertTrue(
+ run_command_and_compare(
+ command=
+ "otbcli_TensorflowModelServe "
+ "-source1.il $DATADIR/fake_spot6.jp2 "
+ "-source1.rfieldx 64 "
+ "-source1.rfieldy 64 "
+ f"-source1.placeholder {INPUT_NAME} "
+ "-model.dir $TMPDIR/model_from_pimg "
+ "-model.fullyconv on "
+ f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 16)} "
+ "-output.efieldx 32 "
+ "-output.efieldy 32 "
+ "-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
+ to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ tol=INFERENCE_MAE_TOL))
+ self.assertTrue(
+ run_command_and_compare(
+ command=
+ "otbcli_TensorflowModelServe "
+ "-source1.il $DATADIR/fake_spot6.jp2 "
+ "-source1.rfieldx 128 "
+ "-source1.rfieldy 128 "
+ f"-source1.placeholder {INPUT_NAME} "
+ "-model.dir $TMPDIR/model_from_pimg "
+ "-model.fullyconv on "
+ f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 32)} "
+ "-output.efieldx 64 "
+ "-output.efieldy 64 "
+ "-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
+ to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ tol=INFERENCE_MAE_TOL))
+
+ @pytest.mark.order(3)
+ def test_create_tfrecords(self):
+ params = create_tfrecords.parser.parse_args(['--xs', resolve_paths('$DATADIR/amsterdam_patches_A.tif'),
+ '--labels', resolve_paths('$DATADIR/amsterdam_labels_A.tif'),
+ '--outdir', resolve_paths('$TMPDIR/train')])
+ create_tfrecords.create_tfrecords(params=params)
+ self.assertTrue(files_exist(['$TMPDIR/train/output_shapes.json',
+ '$TMPDIR/train/output_types.json',
+ '$TMPDIR/train/0.records']))
+ params = create_tfrecords.parser.parse_args(['--xs', resolve_paths('$DATADIR/amsterdam_patches_B.tif'),
+ '--labels', resolve_paths('$DATADIR/amsterdam_labels_B.tif'),
+ '--outdir', resolve_paths('$TMPDIR/valid')])
+ create_tfrecords.create_tfrecords(params=params)
+ self.assertTrue(files_exist(['$TMPDIR/valid/output_shapes.json',
+ '$TMPDIR/valid/output_types.json',
+ '$TMPDIR/valid/0.records']))
+
+ @pytest.mark.order(4)
+ def test_train_from_tfrecords(self):
+ params = train_from_tfrecords.parser.parse_args(['--model_dir', resolve_paths('$TMPDIR/model_from_tfrecs'),
+ '--nb_epochs', '1',
+ '--tfrecords_dir', resolve_paths('$TMPDIR')])
+ train_from_tfrecords.train(params=params)
+ self.assertTrue(files_exist(['$TMPDIR/model_from_tfrecs/keras_metadata.pb',
+ '$TMPDIR/model_from_tfrecs/saved_model.pb',
+ '$TMPDIR/model_from_tfrecs/variables/variables.data-00000-of-00001',
+ '$TMPDIR/model_from_tfrecs/variables/variables.index']))
+
+ @pytest.mark.order(5)
+ def test_model_inference2(self):
+ self.assertTrue(
+ run_command_and_compare(
+ command=
+ "otbcli_TensorflowModelServe "
+ "-source1.il $DATADIR/fake_spot6.jp2 "
+ "-source1.rfieldx 64 "
+ "-source1.rfieldy 64 "
+ f"-source1.placeholder {INPUT_NAME} "
+ "-model.dir $TMPDIR/model_from_pimg "
+ "-model.fullyconv on "
+ f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 16)} "
+ "-output.efieldx 32 "
+ "-output.efieldy 32 "
+ "-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
+ to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ tol=INFERENCE_MAE_TOL))
+
+ self.assertTrue(
+ run_command_and_compare(
+ command=
+ "otbcli_TensorflowModelServe "
+ "-source1.il $DATADIR/fake_spot6.jp2 "
+ "-source1.rfieldx 128 "
+ "-source1.rfieldy 128 "
+ f"-source1.placeholder {INPUT_NAME} "
+ "-model.dir $TMPDIR/model_from_pimg "
+ "-model.fullyconv on "
+ f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 32)} "
+ "-output.efieldx 64 "
+ "-output.efieldy 64 "
+ "-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
+ to_compare_dict={"$DATADIR/classif_model4_softmax.tif": "$TMPDIR/classif_model4_softmax.tif"},
+ tol=INFERENCE_MAE_TOL))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/data/classif_model4_softmax.tif b/test/data/classif_model4_softmax.tif
new file mode 100644
index 00000000..6aebc388
--- /dev/null
+++ b/test/data/classif_model4_softmax.tif
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68fcaf190239fe63d11614fb9c1761a472aa8936cea2c28e9435dd813cb571e6
+size 34372
diff --git a/test/test_utils.py b/test/test_utils.py
index c07301e9..4554e28e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1,5 +1,6 @@
import otbApplication
import os
+from pathlib import Path
def get_nb_of_channels(raster):
@@ -42,15 +43,59 @@ def compare(raster1, raster2, tol=0.01):
return True
-def resolve_paths(filename, var_list):
+def resolve_paths(path):
"""
- Retrieve environment variables in paths
- :param filename: file name
- :params var_list: variable list
- :return filename with retrieved environment variables
+ Resolve a path with the environment variables
"""
- new_filename = filename
- for var in var_list:
- new_filename = new_filename.replace("${}".format(var), os.environ[var])
- print("Resolve filename...\n\tfilename: {}, \n\tnew filename: {}".format(filename, new_filename))
- return new_filename
+ return os.path.expandvars(path)
+
+
+def files_exist(file_list):
+ """
+ Check is all files exist
+ """
+ print("Checking if files exist...")
+ for file in file_list:
+ print("\t{}".format(file))
+ path = Path(resolve_paths(file))
+ if not path.is_file():
+ print("File {} does not exist!".format(file))
+ return False
+ print("\tOk")
+ return True
+
+
+def run_command(command):
+ """
+ Run a command
+ :param command: the command to run
+ """
+ full_command = resolve_paths(command)
+ print("Running command: \n\t {}".format(full_command))
+ os.system(full_command)
+
+
+def run_command_and_test_exist(command, file_list):
+ """
+ :param command: the command to run (str)
+ :param file_list: list of files to check
+ :return True or False
+ """
+ run_command(command)
+ return files_exist(file_list)
+
+
+def run_command_and_compare(command, to_compare_dict, tol=0.01):
+ """
+ :param command: the command to run (str)
+ :param to_compare_dict: a dict of {baseline1: output1, ..., baselineN: outputN}
+ :param tol: tolerance (float)
+ :return True or False
+ """
+
+ run_command(command)
+ for baseline, output in to_compare_dict.items():
+ if not compare(resolve_paths(baseline), resolve_paths(output), tol):
+ print("Baseline {} and output {} differ.".format(baseline, output))
+ return False
+ return True
diff --git a/test/tutorial_unittest.py b/test/tutorial_unittest.py
index 7934862f..af2b181c 100644
--- a/test/tutorial_unittest.py
+++ b/test/tutorial_unittest.py
@@ -2,64 +2,11 @@
# -*- coding: utf-8 -*-
import pytest
import unittest
-import os
-from pathlib import Path
-import test_utils
+from test_utils import run_command, run_command_and_test_exist, run_command_and_compare
INFERENCE_MAE_TOL = 10.0 # Dummy value: we don't really care of the mae value but rather the image size etc
-def resolve_paths(path):
- """
- Resolve a path with the environment variables
- """
- return test_utils.resolve_paths(path, var_list=["TMPDIR", "DATADIR"])
-
-
-def run_command(command):
- """
- Run a command
- :param command: the command to run
- """
- full_command = resolve_paths(command)
- print("Running command: \n\t {}".format(full_command))
- os.system(full_command)
-
-
-def run_command_and_test_exist(command, file_list):
- """
- :param command: the command to run (str)
- :param file_list: list of files to check
- :return True or False
- """
- run_command(command)
- print("Checking if files exist...")
- for file in file_list:
- print("\t{}".format(file))
- path = Path(resolve_paths(file))
- if not path.is_file():
- print("File {} does not exist!".format(file))
- return False
- print("\tOk")
- return True
-
-
-def run_command_and_compare(command, to_compare_dict, tol=0.01):
- """
- :param command: the command to run (str)
- :param to_compare_dict: a dict of {baseline1: output1, ..., baselineN: outputN}
- :param tol: tolerance (float)
- :return True or False
- """
-
- run_command(command)
- for baseline, output in to_compare_dict.items():
- if not test_utils.compare(resolve_paths(baseline), resolve_paths(output), tol):
- print("Baseline {} and output {} differ.".format(baseline, output))
- return False
- return True
-
-
class TutorialTest(unittest.TestCase):
@pytest.mark.order(1)
--
GitLab
From 3304e6d043cb1a7d1397131fb8cf8d616609e530 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 14:38:59 +0200
Subject: [PATCH 54/77] REFAC: update imports
---
.../tensorflow_v2x/fcnn/create_tfrecords.py | 17 +++++++------
.../tensorflow_v2x/fcnn/fcnn_model.py | 17 +++++++------
.../fcnn/train_from_patchesimages.py | 25 +++++++++++--------
.../fcnn/train_from_tfrecords.py | 13 ++++++----
4 files changed, 42 insertions(+), 30 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
index 0d10c886..51043ef1 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/create_tfrecords.py
@@ -4,19 +4,17 @@ into TFRecords files.
"""
import argparse
from pathlib import Path
-import helper
+from otbtf.examples.tensorflow_v2x.fcnn import helper
from otbtf import DatasetFromPatchesImages
-# Application parameters
parser = argparse.ArgumentParser(description="Converts patches-images into TFRecords")
parser.add_argument("--xs", required=True, nargs="+", default=[], help="A list of patches-images for the XS image")
-parser.add_argument("--labels", required=True, nargs="+", default=[], help="A list of patches-images for the labels")
+parser.add_argument("--labels", required=True, nargs="+", default=[],
+ help="A list of patches-images for the labels")
parser.add_argument("--outdir", required=True, help="Output dir for TFRecords files")
-params = parser.parse_args()
-if __name__ == "__main__":
-
+def create_tfrecords(params):
# Sort patches and labels
patches = sorted(params.xs)
labels = sorted(params.labels)
@@ -30,7 +28,12 @@ if __name__ == "__main__":
outdir.mkdir(exist_ok=True)
# Create dataset from the filename dict
- dataset = DatasetFromPatchesImages(filenames_dict={"input_xs": patches, "labels": labels})
+ dataset = DatasetFromPatchesImages(filenames_dict={"input_xs_patches": patches, "labels_patches": labels})
# Convert the dataset into TFRecords
dataset.to_tfrecords(output_dir=params.outdir, drop_remainder=False)
+
+
+if __name__ == "__main__":
+ params = parser.parse_args()
+ create_tfrecords(params)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 0557df66..2a73ef3f 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -7,6 +7,9 @@ import logging
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
N_CLASSES = 6
+INPUT_NAME = "input_xs" # name of the input in the `FCNNModel` instance, also name of the input node in the SavedModel
+TARGET_NAME = "predictions" # name of the output in the `FCNNModel` instance
+OUTPUT_SOFTMAX_NAME = "predictions_softmax_tensor" # name (prefix) of the output node in the SavedModel
class FCNNModel(ModelBase):
@@ -28,7 +31,7 @@ class FCNNModel(ModelBase):
:param inputs: dict of inputs
:return: dict of normalized inputs, ready to be used from the `get_outputs()` function of the model
"""
- return {"input_xs": inputs["input_xs"] * 0.0001}
+ return {INPUT_NAME: tf.cast(inputs[INPUT_NAME], tf.float32) * 0.0001}
def get_outputs(self, normalized_inputs):
"""
@@ -41,7 +44,7 @@ class FCNNModel(ModelBase):
:return: activation values
"""
- norm_inp = normalized_inputs["input_xs"]
+ norm_inp = normalized_inputs[INPUT_NAME]
def _conv(inp, depth, name):
return tf.keras.layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)(inp)
@@ -72,9 +75,9 @@ class FCNNModel(ModelBase):
# - the output tensor key identifies the output tensors, mainly to fit the targets to model outputs during
# training process, but it can also be used to access the tensors as tf/keras objects, for instance to
# display previews images in TensorBoard.
- predictions = tf.keras.layers.Softmax(name="predictions_softmax_tensor")(out_tconv4)
+ predictions = tf.keras.layers.Softmax(name=OUTPUT_SOFTMAX_NAME)(out_tconv4)
- return {"predictions": predictions}
+ return {TARGET_NAME: predictions}
def dataset_preprocessing_fn(examples):
@@ -89,10 +92,10 @@ def dataset_preprocessing_fn(examples):
"""
def _to_categorical(x):
- return tf.one_hot(tf.squeeze(x, axis=-1), depth=N_CLASSES)
+ return tf.one_hot(tf.squeeze(tf.cast(x, tf.int32), axis=-1), depth=N_CLASSES)
- return {"input_xs": examples["input_xs"][32:-32, 32:-32, :],
- "predictions": _to_categorical(examples["labels"])}
+ return {INPUT_NAME: examples["input_xs_patches"],
+ TARGET_NAME: _to_categorical(examples["labels_patches"])}
def train(params, ds_train, ds_valid, ds_test):
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
index aa07e8a5..9299c9e0 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_patchesimages.py
@@ -1,9 +1,9 @@
"""
This example shows how to use the otbtf python API to train a deep net from patches-images.
"""
-import helper
from otbtf import DatasetFromPatchesImages
-import fcnn_model
+from otbtf.examples.tensorflow_v2x.fcnn import helper
+from otbtf.examples.tensorflow_v2x.fcnn import fcnn_model
parser = helper.base_parser()
parser.add_argument("--train_xs", required=True, nargs="+", default=[],
@@ -20,7 +20,7 @@ parser.add_argument("--test_labels", required=False, nargs="+", default=[],
help="A list of patches-images for the labels (test dataset)")
-def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"]):
+def create_dataset(xs_filenames, labels_filenames, batch_size, targets_keys=[fcnn_model.TARGET_NAME]):
"""
Returns a TF dataset generated from an `otbtf.DatasetFromPatchesImages` instance
"""
@@ -36,24 +36,27 @@ def create_dataset(xs_filenames, labels_filenames, targets_keys=["predictions"])
# However, this can slow down your process since the patches are read on-the-fly on the filesystem.
# Good when one batch computation is slower than one batch gathering!
# You can also use a custom `Iterator` of your own (default is `RandomIterator`). See `otbtf.dataset.Iterator`.
- ds = DatasetFromPatchesImages(filenames_dict={"input_xs": xs_filenames, "labels": labels_filenames})
+ ds = DatasetFromPatchesImages(filenames_dict={"input_xs_patches": xs_filenames, "labels_patches": labels_filenames})
# We generate the TF dataset, and we use a preprocessing option to put the labels into one hot encoding (see the
# `fcnn_model.dataset_preprocessing_fn` function). Also, we set the `target_keys` parameter to ask the dataset to
# deliver samples in the form expected by keras, i.e. a tuple of dicts (inputs_dict, target_dict).
- tf_ds = ds.get_tf_dataset(batch_size=params.batch_size, preprocessing_fn=fcnn_model.dataset_preprocessing_fn,
+ tf_ds = ds.get_tf_dataset(batch_size=batch_size, preprocessing_fn=fcnn_model.dataset_preprocessing_fn,
targets_keys=targets_keys)
return tf_ds
-if __name__ == "__main__":
- params = parser.parse_args()
-
+def train(params):
# Create TF datasets
- ds_train = create_dataset(params.train_xs, params.train_labels)
- ds_valid = create_dataset(params.valid_xs, params.valid_labels)
- ds_test = create_dataset(params.test_xs, params.test_labels) if params.test_xs else None
+ ds_train = create_dataset(params.train_xs, params.train_labels, batch_size=params.batch_size)
+ ds_valid = create_dataset(params.valid_xs, params.valid_labels, batch_size=params.batch_size)
+ ds_test = create_dataset(params.test_xs, params.test_labels,
+ batch_size=params.batch_size) if params.test_xs else None
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
+
+
+if __name__ == "__main__":
+ train(parser.parse_args())
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
index e3d4d327..3fbfe472 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/train_from_tfrecords.py
@@ -22,25 +22,24 @@ the number of TFRecords files in the training, validation, and test datasets:
k.records
"""
-import helper
import os
from otbtf import TFRecords
-import fcnn_model
+from otbtf.examples.tensorflow_v2x.fcnn import helper
+from otbtf.examples.tensorflow_v2x.fcnn import fcnn_model
parser = helper.base_parser()
parser.add_argument("--tfrecords_dir", required=True,
help="Directory containing train, valid(, test) folders of TFRecords files")
-if __name__ == "__main__":
- params = parser.parse_args()
+def train(params):
# Patches directories must contain 'train' and 'valid' dirs ('test' is not required)
train_dir = os.path.join(params.tfrecords_dir, "train")
valid_dir = os.path.join(params.tfrecords_dir, "valid")
test_dir = os.path.join(params.tfrecords_dir, "test")
kwargs = {"batch_size": params.batch_size,
- "target_keys": ["predictions"],
+ "target_keys": [fcnn_model.TARGET_NAME],
"preprocessing_fn": fcnn_model.dataset_preprocessing_fn}
# Training dataset. Must be shuffled
@@ -56,3 +55,7 @@ if __name__ == "__main__":
# Train the model
fcnn_model.train(params, ds_train, ds_valid, ds_test)
+
+
+if __name__ == "__main__":
+ train(parser.parse_args())
--
GitLab
From 4e6d239ff0ce4821433f1161aa65791674b63897 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Thu, 21 Jul 2022 16:54:24 +0200
Subject: [PATCH 55/77] ENH: add the possibility to specify the input keys
---
otbtf/model.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 98419276..39a76ab1 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -11,12 +11,14 @@ class ModelBase(abc.ABC):
Base class for all models
"""
- def __init__(self, dataset_element_spec, inference_cropping=None):
+ def __init__(self, dataset_element_spec, input_keys=None, inference_cropping=None):
"""
Model initializer, must be called **inside** the strategy.scope().
:param dataset_element_spec: the dataset elements specification (shape, dtype, etc). Can be retrieved from the
dataset instance simply with `ds.element_spec`
+ :param input_keys: Optional. the keys of the inputs used in the model. If not specified, all inputs from the
+ dataset will be considered.
:param inference_cropping: list of number of pixels to be removed on each side of the output during inference.
This list creates some additional outputs in the model, not used during training,
only during inference. Default [16, 32, 64, 96, 128]
@@ -25,8 +27,12 @@ class ModelBase(abc.ABC):
dataset_input_element_spec = dataset_element_spec[0]
logging.info("Dataset input element spec: %s", dataset_input_element_spec)
- self.dataset_input_keys = list(dataset_input_element_spec)
- logging.info("Found dataset input keys: %s", self.dataset_input_keys)
+ if input_keys:
+ self.dataset_input_keys = input_keys
+ logging.info("Using input keys: %s", self.dataset_input_keys)
+ else:
+ self.dataset_input_keys = list(dataset_input_element_spec)
+ logging.info("Found dataset input keys: %s", self.dataset_input_keys)
self.inputs_shapes = {key: dataset_input_element_spec[key].shape[1:] for key in self.dataset_input_keys}
logging.info("Inputs shapes: %s", self.inputs_shapes)
--
GitLab
From 3c5b8e6cfbd9f4b2d8ee9a6ff6799e5f944139dd Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 21 Jul 2022 19:41:31 +0200
Subject: [PATCH 56/77] DOC: typo
---
otbtf/model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 39a76ab1..d727827a 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -138,7 +138,7 @@ class ModelBase(abc.ABC):
# extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
# Works when crop > patch size, but we lose tensors names:
# extra_output = tensorflow.identity(out_tensor[:, crop:-crop, crop:-crop, :], name=extra_output_name)
- # Works when crop > patch size, but doesnt work when len(self.inference_cropping) > 1!
+ # Works when crop > patch size, but doesn't work when len(self.inference_cropping) > 1!
# extra_output = tensorflow.keras.layers.Lambda(x: x[:, crop:-crop, crop:-crop, :],
# name=extra_output_name)(out_tensor)
slice = out_tensor[:, crop:-crop, crop:-crop, :]
--
GitLab
From e850aecb7d5c7ca2c03dc4d204b15b8faed12b54 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 25 Jul 2022 17:04:59 +0200
Subject: [PATCH 57/77] ADD: change the number of classes to 2
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 2a73ef3f..ca559ab4 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -6,7 +6,7 @@ import tensorflow as tf
import logging
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
-N_CLASSES = 6
+N_CLASSES = 2
INPUT_NAME = "input_xs" # name of the input in the `FCNNModel` instance, also name of the input node in the SavedModel
TARGET_NAME = "predictions" # name of the output in the `FCNNModel` instance
OUTPUT_SOFTMAX_NAME = "predictions_softmax_tensor" # name (prefix) of the output node in the SavedModel
--
GitLab
From d33681dd6e12a80cbc0f173cfe8692f1deae591b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 25 Jul 2022 17:13:40 +0200
Subject: [PATCH 58/77] DOC: update release notes
---
RELEASE_NOTES.txt | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 6e542468..890209fc 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,3 +1,11 @@
+Version 3.3.0 (27 jul 2022)
+----------------------------------------------------------------
+* Improves the `dataset` classes (`DatasetFromPatchesImages`, `TFRecords`) to use them easily in keras
+* Document the python API (`otbtf.dataset` and `otbtf.tfrecords`)
+* Test the python API in the CI, using the (XS, labels) patches of the Amsterdam dataset (from CRC book)
+* Upgrade OTB to version 8.0.1
+* Upgrade GDAL to version
+
Version 3.2.1 (1 jun 2022)
----------------------------------------------------------------
* Changing docker images naming convention (cpu/gpu-basic* --> cpu/gpu*, cpu/gpu* --> cpu/gpu-opt*) + only images without optimizations are pushed on dockerhub
--
GitLab
From a7cb59a6318ae8d341baab9e36b80764ab78036c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 25 Jul 2022 17:14:03 +0200
Subject: [PATCH 59/77] ADD: update baseline for python API
---
test/data/classif_model4_softmax.tif | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/test/data/classif_model4_softmax.tif b/test/data/classif_model4_softmax.tif
index 6aebc388..eb5ff03c 100644
--- a/test/data/classif_model4_softmax.tif
+++ b/test/data/classif_model4_softmax.tif
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:68fcaf190239fe63d11614fb9c1761a472aa8936cea2c28e9435dd813cb571e6
-size 34372
+oid sha256:9a2ef82781b7e42c82be069db085e7dcc6a3c5b9db84c1277153fe730fd52741
+size 9504
--
GitLab
From f301964cdbc06158287b26b232c37af5c96c1dcf Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 25 Jul 2022 22:06:05 +0200
Subject: [PATCH 60/77] DOC: update docker images
---
README.md | 10 +++--
doc/DOCKERUSE.md | 86 ++++++++++++++++++++++--------------------
tools/docker/README.md | 4 +-
3 files changed, 55 insertions(+), 45 deletions(-)
diff --git a/README.md b/README.md
index 343c4863..b69601ab 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
# OTBTF: Orfeo ToolBox meets TensorFlow
[](https://opensource.org/licenses/Apache-2.0)
-[](https://gitlab.irstea.fr/remi.cresson/otbtf/-/commits/develop)
+[](https://gitlab.irstea.fr/remi.cresson/otbtf/-/commits/develop)
This remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org) provides a generic, multi purpose deep learning framework, targeting remote sensing images processing.
It contains a set of new process objects that internally invoke [Tensorflow](https://www.tensorflow.org/), and a bunch of user-oriented applications to perform deep learning with real-world remote sensing images.
@@ -36,6 +36,10 @@ Below are some screen captures of deep learning applications performed at large
+ - Sentinel-2 reconstruction with Sentinel-1 VV/VH with the [Decloud software](https://github.com/CNES/decloud), which is based on OTBTF
+
+
+ -
- Image to image translation (Spot-7 image --> Wikimedia Map using CGAN. So unnecessary but fun!)
@@ -46,9 +50,9 @@ For now you have two options: either use the existing **docker image**, or build
### Docker
-Use the latest image from dockerhub:
+Use the latest CPU or GPU-enabled image from dockerhub:
```
-docker run mdl4eo/otbtf3.1:cpu-basic otbcli_PatchesExtraction -help
+docker run mdl4eo/otbtf:3.3.0-cpu otbcli_PatchesExtraction -help
```
Read more in the [docker use documentation](doc/DOCKERUSE.md).
diff --git a/doc/DOCKERUSE.md b/doc/DOCKERUSE.md
index 457e6e01..33da5788 100644
--- a/doc/DOCKERUSE.md
+++ b/doc/DOCKERUSE.md
@@ -5,38 +5,44 @@
Here is the list of OTBTF docker images hosted on [dockerhub](https://hub.docker.com/u/mdl4eo).
Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
-| Name | Os | TF | OTB | Description | Dev files | Compute capability |
-| --------------------------------- | ------------- | ------ | ----- | ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf1.6:cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
-| **mdl4eo/otbtf1.7:cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
-| **mdl4eo/otbtf1.7:gpu** | Ubuntu Xenial | r1.14 | 7.0.0 | GPU | yes | 5.2,6.1,7.0 |
-| **mdl4eo/otbtf2.0:cpu** | Ubuntu Xenial | r2.1 | 7.1.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf2.0:gpu** | Ubuntu Xenial | r2.1 | 7.1.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf2.4:cpu-basic** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf2.4:cpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, few optimizations | no | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf2.4:cpu-mkl** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, Intel MKL, AVX512 | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf2.4:gpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf2.5:cpu-basic** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf2.5:cpu-basic-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf2.5:cpu** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, few optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf2.5:gpu** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf2.5:gpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.0:cpu-basic** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.0:cpu-basic-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.0:gpu** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.0:gpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.1:cpu-basic** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.1:cpu-basic-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.1:gpu-basic** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.1:gpu-basic-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.1:gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.1:gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.2.1:cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.2.1:cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.2.1:gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf3.2.1:gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf3.2.1:gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf3.2.1:gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| Name | Os | TF | OTB | Description | Dev files | Compute capability |
+|------------------------------------------------------------------------------------| ------------- | ------ |-------| ---------------------- | --------- | ------------------ |
+| **mdl4eo/otbtf:1.6-cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
+| **mdl4eo/otbtf:1.7-cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
+| **mdl4eo/otbtf:1.7-gpu** | Ubuntu Xenial | r1.14 | 7.0.0 | GPU | yes | 5.2,6.1,7.0 |
+| **mdl4eo/otbtf:2.0-cpu** | Ubuntu Xenial | r2.1 | 7.1.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.0-gpu** | Ubuntu Xenial | r2.1 | 7.1.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-cpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-cpu-opt** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, few optimizations | no | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-cpu-mkl** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, Intel MKL, AVX512 | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-gpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.5-cpu** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5:cpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5-cpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, few optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5-gpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5-gpu-opt-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-cpu** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-cpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-gpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-gpu-opt-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.2.1-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.2.1-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.2.1-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.2.1-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.2.1-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.2.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-gpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
You can also find more interesting OTBTF flavored images at [LaTelescop gitlab registry](https://gitlab.com/latelescop/docker/otbtf/container_registry/).
@@ -58,7 +64,7 @@ For instance, suppose you have some data in `/mnt/my_device/` that you want to u
The following command shows you how to access the folder from the docker image.
```bash
-docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf3.2.1:cpu bash -c "ls /data"
+docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf:3.2.1-cpu bash -c "ls /data"
```
Beware of ownership issues! see the last section of this doc.
@@ -71,13 +77,13 @@ You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime*
With Docker version earlier than 19.03 :
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf3.2.1:gpu bash
+docker run --runtime=nvidia -ti mdl4eo/otbtf:3.2.1-gpu bash
```
With Docker version including and after 19.03 :
```bash
-docker run --gpus all -ti mdl4eo/otbtf3.2.1:gpu bash
+docker run --gpus all -ti mdl4eo/otbtf:3.2.1-gpu bash
```
You can find some details on the **GPU docker image** and some **docker tips and tricks** on [this blog](https://mdl4eo.irstea.fr/2019/10/15/otbtf-docker-image-with-gpu/).
@@ -90,7 +96,7 @@ Be careful though, these infos might be a bit outdated...
1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux)
2. Install [docker desktop](https://www.docker.com/products/docker-desktop)
3. Start **docker desktop** and **enable WSL2** from *Settings* > *General* then tick the box *Use the WSL2 based engine*
-3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf3.2.1:cpu`
+3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:3.2.1-cpu`
4. Open **docker desktop**, and check that the docker is running in the **Container/Apps** menu
5. From **docker desktop**, click on the icon highlighted as shown below, and use the bash terminal that should pop up!
@@ -139,12 +145,12 @@ sudo systemctl {status,enable,disable,start,stop} docker
Run a simple command in a one-shot container:
```bash
-docker run mdl4eo/otbtf3.2.1:cpu otbcli_PatchesExtraction
+docker run mdl4eo/otbtf:3.2.1-cpu otbcli_PatchesExtraction
```
You can also use the image in interactive mode with bash:
```bash
-docker run -ti mdl4eo/otbtf3.2.1:cpu bash
+docker run -ti mdl4eo/otbtf:3.2.1-cpu bash
```
### Persistent container
@@ -154,7 +160,7 @@ Beware of ownership issues, see the last section of this doc.
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \
- --name otbtf mdl4eo/otbtf3.2.1:cpu /bin/bash
+ --name otbtf mdl4eo/otbtf:3.2.1-cpu /bin/bash
```
### Interactive session
@@ -218,7 +224,7 @@ Create a named container (here with your HOME as volume), Docker will automatica
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser \
- --name otbtf mdl4eo/otbtf3.2.1:cpu /bin/bash
+ --name otbtf mdl4eo/otbtf:3.2.1-cpu /bin/bash
```
Start a background container process:
diff --git a/tools/docker/README.md b/tools/docker/README.md
index 4246b74d..dc718e68 100644
--- a/tools/docker/README.md
+++ b/tools/docker/README.md
@@ -111,7 +111,7 @@ If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g. 0.75
## Container examples
```bash
# Pull GPU image and create a new container with your home directory as volume (requires apt package nvidia-docker2 and CUDA>=11.0)
-docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf2.4:gpu
+docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf:3.3.0-gpu
# Run interactive
docker start -i otbtf-gpu
@@ -123,7 +123,7 @@ docker exec otbtf-gpu python -c 'import tensorflow as tf; print(tf.test.is_gpu_a
### Rebuild OTB with more modules
```bash
-docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf2.4:gpu-dev
+docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf:3.3.0-gpu-dev
docker start -i otbtf-gpu-dev
```
```bash
--
GitLab
From e9cc647b40b1a78066c2314ffc48ca0061924962 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 25 Jul 2022 22:06:31 +0200
Subject: [PATCH 61/77] CI: change docker images naming convention
---
.gitlab-ci.yml | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e0cb8a34..c2f01c4b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
variables:
- OTBTF_VERSION: 3.2.1
+ OTBTF_VERSION: 3.3.0
OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
OTBTF_SRC: /src/otbtf # Local OTBTF source directory
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
@@ -15,8 +15,8 @@ variables:
CACHE_IMAGE_BUILDER: $CI_REGISTRY_IMAGE:builder
BRANCH_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
DEV_IMAGE: $CI_REGISTRY_IMAGE:cpu-basic-dev-testing
- CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE/$OTBTF_VERSION
- DOCKERHUB_IMAGE_BASE: mdl4eo/otbtf${OTBTF_VERSION}
+ CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE:$OTBTF_VERSION
+ DOCKERHUB_IMAGE_BASE: mdl4eo/otbtf:${OTBTF_VERSION}
workflow:
rules:
@@ -186,10 +186,10 @@ deploy_cpu-dev-testing:
deploy_cpu:
extends: .ship base
variables:
- IMAGE_CPU: $CI_REGISTRY_PUBIMG:cpu
- IMAGE_CPUDEV: $CI_REGISTRY_PUBIMG:cpu-dev
- DOCKERHUB_CPU: $DOCKERHUB_IMAGE_BASE:cpu
- DOCKERHUB_CPUDEV: $DOCKERHUB_IMAGE_BASE:cpu-dev
+ IMAGE_CPU: $CI_REGISTRY_PUBIMG-cpu
+ IMAGE_CPUDEV: $CI_REGISTRY_PUBIMG-cpu-dev
+ DOCKERHUB_CPU: $DOCKERHUB_IMAGE_BASE-cpu
+ DOCKERHUB_CPUDEV: $DOCKERHUB_IMAGE_BASE-cpu-dev
script:
# cpu
- docker build --network='host' --tag $IMAGE_CPU --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" .
@@ -207,12 +207,12 @@ deploy_cpu:
deploy_gpu:
extends: .ship base
variables:
- IMAGE_GPU: $CI_REGISTRY_PUBIMG:gpu
- IMAGE_GPUDEV: $CI_REGISTRY_PUBIMG:gpu-dev
- IMAGE_GPUOPT: $CI_REGISTRY_PUBIMG:gpu-opt
- IMAGE_GPUOPTDEV: $CI_REGISTRY_PUBIMG:gpu-opt-dev
- DOCKERHUB_GPU: $DOCKERHUB_IMAGE_BASE:gpu
- DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE:gpu-dev
+ IMAGE_GPU: $CI_REGISTRY_PUBIMG-gpu
+ IMAGE_GPUDEV: $CI_REGISTRY_PUBIMG-gpu-dev
+ IMAGE_GPUOPT: $CI_REGISTRY_PUBIMG-gpu-opt
+ IMAGE_GPUOPTDEV: $CI_REGISTRY_PUBIMG-gpu-opt-dev
+ DOCKERHUB_GPU: $DOCKERHUB_IMAGE_BASE-gpu
+ DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE-gpu-dev
script:
# gpu-opt
- docker build --network='host' --tag $IMAGE_GPUOPT --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 .
--
GitLab
From 94317745e6633b8b1d3015fb60ac5ac532c6b726 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 25 Jul 2022 22:12:55 +0200
Subject: [PATCH 62/77] DOC: bump otbtf docker image version in examples
---
doc/DOCKERUSE.md | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/doc/DOCKERUSE.md b/doc/DOCKERUSE.md
index 33da5788..f336623b 100644
--- a/doc/DOCKERUSE.md
+++ b/doc/DOCKERUSE.md
@@ -31,12 +31,12 @@ Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](htt
| **mdl4eo/otbtf:3.1-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.1-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.2.1-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.2.1-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.2.1-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.2.1-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.2.1-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.2.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
@@ -64,7 +64,7 @@ For instance, suppose you have some data in `/mnt/my_device/` that you want to u
The following command shows you how to access the folder from the docker image.
```bash
-docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf:3.2.1-cpu bash -c "ls /data"
+docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf:3.3.0-cpu bash -c "ls /data"
```
Beware of ownership issues! see the last section of this doc.
@@ -77,13 +77,13 @@ You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime*
With Docker version earlier than 19.03 :
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf:3.2.1-gpu bash
+docker run --runtime=nvidia -ti mdl4eo/otbtf:3.3.0-gpu bash
```
With Docker version including and after 19.03 :
```bash
-docker run --gpus all -ti mdl4eo/otbtf:3.2.1-gpu bash
+docker run --gpus all -ti mdl4eo/otbtf:3.3.0-gpu bash
```
You can find some details on the **GPU docker image** and some **docker tips and tricks** on [this blog](https://mdl4eo.irstea.fr/2019/10/15/otbtf-docker-image-with-gpu/).
@@ -96,7 +96,7 @@ Be careful though, these infos might be a bit outdated...
1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux)
2. Install [docker desktop](https://www.docker.com/products/docker-desktop)
3. Start **docker desktop** and **enable WSL2** from *Settings* > *General* then tick the box *Use the WSL2 based engine*
-3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:3.2.1-cpu`
+3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:3.3.0-cpu`
4. Open **docker desktop**, and check that the docker is running in the **Container/Apps** menu
5. From **docker desktop**, click on the icon highlighted as shown below, and use the bash terminal that should pop up!
@@ -145,12 +145,12 @@ sudo systemctl {status,enable,disable,start,stop} docker
Run a simple command in a one-shot container:
```bash
-docker run mdl4eo/otbtf:3.2.1-cpu otbcli_PatchesExtraction
+docker run mdl4eo/otbtf:3.3.0-cpu otbcli_PatchesExtraction
```
You can also use the image in interactive mode with bash:
```bash
-docker run -ti mdl4eo/otbtf:3.2.1-cpu bash
+docker run -ti mdl4eo/otbtf:3.3.0-cpu bash
```
### Persistent container
@@ -160,7 +160,7 @@ Beware of ownership issues, see the last section of this doc.
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \
- --name otbtf mdl4eo/otbtf:3.2.1-cpu /bin/bash
+ --name otbtf mdl4eo/otbtf:3.3.0-cpu /bin/bash
```
### Interactive session
@@ -224,7 +224,7 @@ Create a named container (here with your HOME as volume), Docker will automatica
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser \
- --name otbtf mdl4eo/otbtf:3.2.1-cpu /bin/bash
+ --name otbtf mdl4eo/otbtf:3.3.0-cpu /bin/bash
```
Start a background container process:
--
GitLab
From 46960dfcc727dfccce3efac4cb37355ee8679e5e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 25 Jul 2022 22:22:12 +0200
Subject: [PATCH 63/77] ADD: update README with TensorfFlow 2x/Keras examples
---
README.md | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index b69601ab..c2a13a9c 100644
--- a/README.md
+++ b/README.md
@@ -19,9 +19,12 @@ Applications can be used to build OTB pipelines from Python or C++ APIs.
### Python
-`otbtf.py` targets python developers that want to train their own model from python with TensorFlow or Keras.
+The `otbtf` module targets python developers that want to train their own model from python with TensorFlow or Keras.
It provides various classes for datasets and iterators to handle the _patches images_ generated from the `PatchesExtraction` OTB application.
-For instance, the `otbtf.Dataset` class provides a method `get_tf_dataset()` which returns a `tf.dataset` that can be used in your favorite TensorFlow pipelines, or convert your patches into TFRecords.
+For instance, the `otbtf.DatasetFromPatchesImages` can be instantiated from a set of _patches images_
+and delivering samples as `tf.dataset` that can be used in your favorite TensorFlow pipelines, or convert your patches into TFRecords.
+The `otbtf.TFRecords` enables you train networks from TFRecords files, which is quite suited for
+distributed training. Read more in the [tutorial for keras](otbtf/examples/tensorflow_v2x/fcnn/README.md).
`tricks.py` is here for backward compatibility with codes based on OTBTF 1.x and 2.x.
--
GitLab
From fd0e043ba2ce457c027a754dc32ad50f3916b4a3 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 26 Jul 2022 11:35:28 +0200
Subject: [PATCH 64/77] REFAC: cropping is performed in default postprocessing
implementation
---
otbtf/model.py | 36 ++++++++++++++----------------------
1 file changed, 14 insertions(+), 22 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index d727827a..2db99938 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -93,15 +93,26 @@ class ModelBase(abc.ABC):
"""
Post-process the model outputs.
Takes the dicts of inputs and outputs, and returns a dict of post-processed outputs.
+ The default implementation provides a set of cropped output tensors
:param outputs: dict of model outputs
:param inputs: dict of model inputs (optional)
:param normalized_inputs: dict of normalized model inputs (optional)
:return: a dict of post-processed model outputs
"""
- logging.warning("postprocess_outputs() undefined. No post-processing of the model inputs will be performed. "
- "You can implement the function in your model class if you want.")
- return outputs
+
+ # Add extra outputs for inference
+ extra_outputs = {}
+ for out_key, out_tensor in outputs.items():
+ for crop in self.inference_cropping:
+ extra_output_key = cropped_tensor_name(out_key, crop)
+ extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
+ logging.info("Adding extra output for tensor %s with crop %s (%s)", out_key, crop, extra_output_name)
+ cropped = out_tensor[:, crop:-crop, crop:-crop, :]
+ identity = tensorflow.keras.layers.Activation('linear', name=extra_output_name)
+ extra_outputs[extra_output_key] = identity(cropped)
+
+ return extra_outputs
def create_network(self):
"""
@@ -126,25 +137,6 @@ class ModelBase(abc.ABC):
# Post-processing for inference
postprocessed_outputs = self.postprocess_outputs(outputs=outputs, inputs=inputs,
normalized_inputs=normalized_inputs)
-
- # Add extra outputs for inference
- extra_outputs = {}
- for out_key, out_tensor in postprocessed_outputs.items():
- for crop in self.inference_cropping:
- extra_output_key = cropped_tensor_name(out_key, crop)
- extra_output_name = cropped_tensor_name(out_tensor._keras_history.layer.name, crop)
- logging.info("Adding extra output for tensor %s with crop %s (%s)", out_key, crop, extra_output_name)
- # Does not work anymore when crop > patch size:
- # extra_output = tensorflow.keras.layers.Cropping2D(cropping=crop, name=extra_output_name)(out_tensor)
- # Works when crop > patch size, but we lose tensors names:
- # extra_output = tensorflow.identity(out_tensor[:, crop:-crop, crop:-crop, :], name=extra_output_name)
- # Works when crop > patch size, but doesn't work when len(self.inference_cropping) > 1!
- # extra_output = tensorflow.keras.layers.Lambda(x: x[:, crop:-crop, crop:-crop, :],
- # name=extra_output_name)(out_tensor)
- slice = out_tensor[:, crop:-crop, crop:-crop, :]
- identity = tensorflow.keras.layers.Activation('linear', name=extra_output_name)
- extra_outputs[extra_output_key] = identity(slice)
- postprocessed_outputs.update(extra_outputs)
outputs.update(postprocessed_outputs)
# Return the keras model
--
GitLab
From 3aaf2b1a5e9c7cbccd226227d1543fc30275d92b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 26 Jul 2022 12:11:27 +0200
Subject: [PATCH 65/77] STYLE: remove whitespace
---
otbtf/model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index 2db99938..970deaed 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -111,7 +111,7 @@ class ModelBase(abc.ABC):
cropped = out_tensor[:, crop:-crop, crop:-crop, :]
identity = tensorflow.keras.layers.Activation('linear', name=extra_output_name)
extra_outputs[extra_output_key] = identity(cropped)
-
+
return extra_outputs
def create_network(self):
--
GitLab
From 468b12910dccd63292d4441b2d6e14acfe9e09ef Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 26 Jul 2022 12:12:04 +0200
Subject: [PATCH 66/77] ENH: expose tf.TFRecordDataset parameters
---
otbtf/tfrecords.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index d04e9fcb..123bdea5 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -143,7 +143,8 @@ class TFRecords:
def read(self, batch_size, target_keys, n_workers=1, drop_remainder=True, shuffle_buffer_size=None,
preprocessing_fn=None, shard_policy=tf.data.experimental.AutoShardPolicy.AUTO,
- prefetch_buffer_size=tf.data.experimental.AUTOTUNE, **kwargs):
+ prefetch_buffer_size=tf.data.experimental.AUTOTUNE,
+ num_parallel_calls=tf.data.experimental.AUTOTUNE, **kwargs):
"""
Read all tfrecord files matching with pattern and convert data to tensorflow dataset.
:param batch_size: Size of tensorflow batch
@@ -166,8 +167,9 @@ class TFRecords:
preprocessing_fn should not implement such things as radiometric transformations from
input to input_preprocessed, because those are performed inside the model itself
(see `otbtf.ModelBase.normalize_inputs()`).
- :param shard_policy: sharding policy
- :param prefetch_buffer_size: prefetch buffer size
+ :param shard_policy: sharding policy for the TFRecordDataset options
+ :param prefetch_buffer_size: buffer size for the prefetch operation
+ :param num_parallel_calls: number of parallel calls for the parsing + preprocessing step
:param kwargs: some keywords arguments for preprocessing_fn
"""
options = tf.data.Options()
@@ -176,7 +178,6 @@ class TFRecords:
options.experimental_distribute.auto_shard_policy = shard_policy # for multiworker
parse = partial(self.parse_tfrecord, target_keys=target_keys, preprocessing_fn=preprocessing_fn, **kwargs)
- # TODO: to be investigated :
# 1/ num_parallel_reads useful ? I/O bottleneck of not ?
# 2/ num_parallel_calls=tf.data.experimental.AUTOTUNE useful ?
tfrecords_pattern_path = os.path.join(self.dirpath, "*.records")
@@ -191,7 +192,7 @@ class TFRecords:
logging.info('Reducing number of records to : %s', nb_matching_files)
dataset = tf.data.TFRecordDataset(matching_files) # , num_parallel_reads=2) # interleaves reads from xxx files
dataset = dataset.with_options(options) # uses data as soon as it streams in, rather than in its original order
- dataset = dataset.map(parse, num_parallel_calls=tf.data.experimental.AUTOTUNE)
+ dataset = dataset.map(parse, num_parallel_calls=num_parallel_calls)
if shuffle_buffer_size:
dataset = dataset.shuffle(buffer_size=shuffle_buffer_size)
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
--
GitLab
From 02640fdb5933a8ff707f47ba4edf428657a960d1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 26 Jul 2022 12:25:14 +0200
Subject: [PATCH 67/77] ENH: use strides 2 for fcnn example
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index ca559ab4..8be7cabe 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -47,10 +47,11 @@ class FCNNModel(ModelBase):
norm_inp = normalized_inputs[INPUT_NAME]
def _conv(inp, depth, name):
- return tf.keras.layers.Conv2D(filters=depth, kernel_size=3, activation="relu", name=name)(inp)
+ return tf.keras.layers.Conv2D(filters=depth, kernel_size=3, strides=2, activation="relu", name=name)(inp)
def _tconv(inp, depth, name, activation="relu"):
- return tf.keras.layers.Conv2DTranspose(filters=depth, kernel_size=3, activation=activation, name=name)(inp)
+ return tf.keras.layers.Conv2DTranspose(filters=depth, kernel_size=3, strides=2, activation=activation,
+ name=name)(inp)
out_conv1 = _conv(norm_inp, 16, "conv1")
out_conv2 = _conv(out_conv1, 32, "conv2")
--
GitLab
From d703ff0518eff17b23ce92f67675e44d727667df Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 26 Jul 2022 14:30:46 +0200
Subject: [PATCH 68/77] ENH: use strides 2 for fcnn example
---
otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index 8be7cabe..95f2d017 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -47,11 +47,12 @@ class FCNNModel(ModelBase):
norm_inp = normalized_inputs[INPUT_NAME]
def _conv(inp, depth, name):
- return tf.keras.layers.Conv2D(filters=depth, kernel_size=3, strides=2, activation="relu", name=name)(inp)
+ return tf.keras.layers.Conv2D(filters=depth, kernel_size=3, strides=2, activation="relu",
+ padding="same", name=name)(inp)
def _tconv(inp, depth, name, activation="relu"):
return tf.keras.layers.Conv2DTranspose(filters=depth, kernel_size=3, strides=2, activation=activation,
- name=name)(inp)
+ padding="same", name=name)(inp)
out_conv1 = _conv(norm_inp, 16, "conv1")
out_conv2 = _conv(out_conv1, 32, "conv2")
--
GitLab
From 1e67f213f807a2c799aa0827a350644644702672 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Tue, 26 Jul 2022 15:00:52 +0200
Subject: [PATCH 69/77] REFAC: move function to utis + expose Model class in
__init__.py
---
otbtf/__init__.py | 4 +++-
otbtf/model.py | 11 +----------
otbtf/utils.py | 10 ++++++++++
3 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/otbtf/__init__.py b/otbtf/__init__.py
index ac36018a..5321e3db 100644
--- a/otbtf/__init__.py
+++ b/otbtf/__init__.py
@@ -20,7 +20,9 @@
"""
OTBTF python module
"""
+
from otbtf.utils import read_as_np_arr, gdal_open
from otbtf.dataset import Buffer, PatchesReaderBase, PatchesImagesReader, IteratorBase, RandomIterator, Dataset, \
- DatasetFromPatchesImages
+ DatasetFromPatchesImages
from otbtf.tfrecords import TFRecords
+from otbtf.model import ModelBase
diff --git a/otbtf/model.py b/otbtf/model.py
index 970deaed..2a9fd345 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -3,7 +3,7 @@
import abc
import logging
import tensorflow
-from otbtf.utils import _is_chief
+from otbtf.utils import _is_chief, cropped_tensor_name
class ModelBase(abc.ABC):
@@ -166,12 +166,3 @@ class ModelBase(abc.ABC):
name=self.__class__.__name__ + '_simplified')
tensorflow.keras.utils.plot_model(model_simplified, output_path)
-
-def cropped_tensor_name(tensor_name, crop):
- """
- A name for the padded tensor
- :param tensor_name: tensor name
- :param pad: pad value
- :return: name
- """
- return "{}_crop{}".format(tensor_name, crop)
diff --git a/otbtf/utils.py b/otbtf/utils.py
index 989db35e..7aa777af 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -87,3 +87,13 @@ def _is_chief(strategy):
return (task_type == 'chief') or (task_type == 'worker' and task_id == 0) or task_type is None
# strategy with only one worker
return True
+
+
+def cropped_tensor_name(tensor_name, crop):
+ """
+ A name for the padded tensor
+ :param tensor_name: tensor name
+ :param pad: pad value
+ :return: name
+ """
+ return "{}_crop{}".format(tensor_name, crop)
--
GitLab
From 2bf5d8bd4a8bc4b025653b229e2c0bec747455c6 Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Tue, 26 Jul 2022 18:06:16 +0200
Subject: [PATCH 70/77] DOC: refac put the old releases at the bottom
---
doc/DOCKERUSE.md | 60 ++++++++++++++++++++++++++++--------------------
1 file changed, 35 insertions(+), 25 deletions(-)
diff --git a/doc/DOCKERUSE.md b/doc/DOCKERUSE.md
index f336623b..d88963d5 100644
--- a/doc/DOCKERUSE.md
+++ b/doc/DOCKERUSE.md
@@ -2,35 +2,11 @@
### Available images
-Here is the list of OTBTF docker images hosted on [dockerhub](https://hub.docker.com/u/mdl4eo).
+Here is the list of the latest OTBTF docker images hosted on [dockerhub](https://hub.docker.com/u/mdl4eo).
Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
|------------------------------------------------------------------------------------| ------------- | ------ |-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:1.6-cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
-| **mdl4eo/otbtf:1.7-cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
-| **mdl4eo/otbtf:1.7-gpu** | Ubuntu Xenial | r1.14 | 7.0.0 | GPU | yes | 5.2,6.1,7.0 |
-| **mdl4eo/otbtf:2.0-cpu** | Ubuntu Xenial | r2.1 | 7.1.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf:2.0-gpu** | Ubuntu Xenial | r2.1 | 7.1.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf:2.4-cpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf:2.4-cpu-opt** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, few optimizations | no | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf:2.4-cpu-mkl** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, Intel MKL, AVX512 | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf:2.4-gpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
-| **mdl4eo/otbtf:2.5-cpu** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:2.5:cpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:2.5-cpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, few optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:2.5-gpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:2.5-gpu-opt-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.0-cpu** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.0-cpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.0-gpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.0-gpu-opt-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.1-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.1-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.1-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.1-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.1-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
@@ -44,8 +20,11 @@ Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](htt
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+The list of older releases is available [here](#older-docker-releases).
+
You can also find more interesting OTBTF flavored images at [LaTelescop gitlab registry](https://gitlab.com/latelescop/docker/otbtf/container_registry/).
+
### Development ready images
Until r2.4, all images are development-ready, and the sources are located in `/work/`.
@@ -261,3 +240,34 @@ id
ls -Alh /home/otbuser
touch /home/otbuser/test.txt
```
+
+# Older docker releases
+
+Here you can find the list of older releases of OTBTF:
+
+| Name | Os | TF | OTB | Description | Dev files | Compute capability |
+|------------------------------------------------------------------------------------| ------------- | ------ |-------| ---------------------- | --------- | ------------------ |
+| **mdl4eo/otbtf:1.6-cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
+| **mdl4eo/otbtf:1.7-cpu** | Ubuntu Xenial | r1.14 | 7.0.0 | CPU, no optimization | yes | 5.2,6.1,7.0 |
+| **mdl4eo/otbtf:1.7-gpu** | Ubuntu Xenial | r1.14 | 7.0.0 | GPU | yes | 5.2,6.1,7.0 |
+| **mdl4eo/otbtf:2.0-cpu** | Ubuntu Xenial | r2.1 | 7.1.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.0-gpu** | Ubuntu Xenial | r2.1 | 7.1.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-cpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, no optimization | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-cpu-opt** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, few optimizations | no | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-cpu-mkl** | Ubuntu Focal | r2.4.1 | 7.2.0 | CPU, Intel MKL, AVX512 | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.4-gpu** | Ubuntu Focal | r2.4.1 | 7.2.0 | GPU | yes | 5.2,6.1,7.0,7.5 |
+| **mdl4eo/otbtf:2.5-cpu** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5:cpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5-cpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, few optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5-gpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:2.5-gpu-opt-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-cpu** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-cpu-dev** | Ubuntu Focal | r2.5 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-gpu-opt** | Ubuntu Focal | r2.5 | 7.4.0 | GPU | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.0-gpu-opt-dev** | Ubuntu Focal | r2.5 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
\ No newline at end of file
--
GitLab
From 61b8d89578595769c7f7cddadad25231f7476c5a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 27 Jul 2022 09:40:30 +0200
Subject: [PATCH 71/77] DOC: fix otb version in latest docker images
---
doc/DOCKERUSE.md | 6 ------
1 file changed, 6 deletions(-)
diff --git a/doc/DOCKERUSE.md b/doc/DOCKERUSE.md
index d88963d5..50e505de 100644
--- a/doc/DOCKERUSE.md
+++ b/doc/DOCKERUSE.md
@@ -7,12 +7,6 @@ Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](htt
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
|------------------------------------------------------------------------------------| ------------- | ------ |-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.3.0-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
--
GitLab
From 76e2f436bd136ebdaee568bccd84b05f3e7b3598 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 27 Jul 2022 14:05:09 +0200
Subject: [PATCH 72/77] DOC: update release notes
---
RELEASE_NOTES.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 890209fc..cdc5ceed 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,7 +1,9 @@
Version 3.3.0 (27 jul 2022)
----------------------------------------------------------------
* Improves the `dataset` classes (`DatasetFromPatchesImages`, `TFRecords`) to use them easily in keras
-* Document the python API (`otbtf.dataset` and `otbtf.tfrecords`)
+* Add the `ModelBase` class, which eases considerably the creation of deep nets for Keras/TF/TensorflowModelServe
+* Add an example explaining how to use python classes to build and train models with Keras, and use models in OTB.
+* Document the python API (`otbtf.dataset`, `otbtf.tfrecords`, `otbtf.ModelBase`)
* Test the python API in the CI, using the (XS, labels) patches of the Amsterdam dataset (from CRC book)
* Upgrade OTB to version 8.0.1
* Upgrade GDAL to version
--
GitLab
From 85bc289506e2a28200cf8ff363007ea62baa5758 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 27 Jul 2022 16:57:18 +0200
Subject: [PATCH 73/77] COMP: update OTB version in dockerimage + try to remove
manually installed protobuf pkg
---
Dockerfile | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index da634cea..df0d9a00 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,8 +25,7 @@ RUN if $GUI; then \
RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip
# NumPy version is conflicting with system's gdal dep and may require venv
ARG NUMPY_SPEC="==1.22.*"
-ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U pip wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
+RUN pip install --no-cache-dir -U pip wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" \
&& pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
# ----------------------------------------------------------------------------
@@ -85,7 +84,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
### OTB
ARG GUI=false
-ARG OTB=7.4.0
+ARG OTB=8.0.1
ARG OTBTESTS=false
RUN mkdir /src/otb
--
GitLab
From 335a55713a87b06554558a3eca1fee73bd984294 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 27 Jul 2022 19:01:35 +0200
Subject: [PATCH 74/77] COMP: remove ossim, openthreads
---
tools/docker/build-flags-otb.txt | 2 --
1 file changed, 2 deletions(-)
diff --git a/tools/docker/build-flags-otb.txt b/tools/docker/build-flags-otb.txt
index 56b0434c..def7bd2b 100644
--- a/tools/docker/build-flags-otb.txt
+++ b/tools/docker/build-flags-otb.txt
@@ -16,8 +16,6 @@
-DUSE_SYSTEM_MUPARSER=ON
-DUSE_SYSTEM_MUPARSERX=ON
-DUSE_SYSTEM_OPENCV=ON
--DUSE_SYSTEM_OPENTHREADS=ON
--DUSE_SYSTEM_OSSIM=ON
-DUSE_SYSTEM_PNG=ON
-DUSE_SYSTEM_QT5=ON
-DUSE_SYSTEM_QWT=ON
--
GitLab
From dea23d2d9cdac58be69cf6360bfac543c0b1e1e0 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 27 Jul 2022 20:50:54 +0200
Subject: [PATCH 75/77] COMP: let protobuf version 3.20
---
Dockerfile | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 529dbec6..9c905bd1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,7 +25,8 @@ RUN if $GUI; then \
RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip
# NumPy version is conflicting with system's gdal dep and may require venv
ARG NUMPY_SPEC="==1.22.*"
-RUN pip install --no-cache-dir -U pip wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" \
+ARG PROTO_SPEC="==3.20.*"
+RUN pip install --no-cache-dir -U pip wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
&& pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
# ----------------------------------------------------------------------------
@@ -148,7 +149,7 @@ COPY --from=builder /src /src
# System-wide ENV
ENV PATH="/opt/otbtf/bin:$PATH"
ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
-ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
+ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
# Default user, directory and command (bash is the entrypoint when using 'docker create')
--
GitLab
From d7e9fef5f16b0b8c0e8636f43b22acf8085bb438 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 27 Jul 2022 22:40:26 +0200
Subject: [PATCH 76/77] COMP: refix dockerfile
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 9c905bd1..990c55f5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -149,7 +149,7 @@ COPY --from=builder /src /src
# System-wide ENV
ENV PATH="/opt/otbtf/bin:$PATH"
ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
-ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
+ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
# Default user, directory and command (bash is the entrypoint when using 'docker create')
--
GitLab
From 316cf27fa7cd263e7ba1ae7c9f0907a408cbf001 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 27 Jul 2022 22:40:43 +0200
Subject: [PATCH 77/77] DOC: update documentation and release notes
---
RELEASE_NOTES.txt | 4 ++--
doc/EXAMPLES.md | 1 +
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index cdc5ceed..5d78c0d7 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -4,9 +4,9 @@ Version 3.3.0 (27 jul 2022)
* Add the `ModelBase` class, which eases considerably the creation of deep nets for Keras/TF/TensorflowModelServe
* Add an example explaining how to use python classes to build and train models with Keras, and use models in OTB.
* Document the python API (`otbtf.dataset`, `otbtf.tfrecords`, `otbtf.ModelBase`)
-* Test the python API in the CI, using the (XS, labels) patches of the Amsterdam dataset (from CRC book)
+* Test the python API in the CI, using the (XS, labels) patches of the Amsterdam dataset from CRC book
* Upgrade OTB to version 8.0.1
-* Upgrade GDAL to version
+* Upgrade GDAL to version 3.4.2
Version 3.2.1 (1 jun 2022)
----------------------------------------------------------------
diff --git a/doc/EXAMPLES.md b/doc/EXAMPLES.md
index 0f5282f3..d48c56a9 100644
--- a/doc/EXAMPLES.md
+++ b/doc/EXAMPLES.md
@@ -312,5 +312,6 @@ otbcli_TensorflowModelServe \
-source2.il $pan -source2.rfieldx 32 -source2.rfieldy 32 -source2.placeholder "x2" \
-model.dir $modeldir \
-model.fullyconv on \
+-output.names "prediction" \
-out $output_classif
```
--
GitLab