From f25ac18ff477490b2dcf88634ec8e2d1bb5d4e13 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 3 Feb 2021 15:33:29 -0500 Subject: [PATCH 001/132] Add kp_module --- centernet/modeling/layers/nn_blocks.py | 90 ++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 centernet/modeling/layers/nn_blocks.py diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py new file mode 100644 index 000000000..0985f67e8 --- /dev/null +++ b/centernet/modeling/layers/nn_blocks.py @@ -0,0 +1,90 @@ +import tensorflow as tf + +from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock + +class kp_module(tf.keras.Model): + def __init__( + self, n, dims, modules, layer=ResidualBlock, **kwargs + ): + super().__init__() + + self.n = n + + curr_mod = modules[0] + next_mod = modules[1] + + curr_dim = dims[0] + next_dim = dims[1] + + self.up1 = self.make_up_layer( + 3, curr_dim, curr_dim, curr_mod, + layer=layer, **kwargs + ) + self.max1 = self.make_pool_layer(curr_dim) + self.low1 = self.make_hg_layer( + 3, curr_dim, next_dim, curr_mod, + layer=layer, **kwargs + ) + self.low2 = type(self)( + n - 1, dims[1:], modules[1:], layer=layer, **kwargs + ) if self.n > 1 else \ + self.make_low_layer( + 3, next_dim, next_dim, next_mod, + layer=layer, **kwargs + ) + self.low3 = self.make_hg_layer_revr( + 3, next_dim, curr_dim, curr_mod, + layer=layer, **kwargs + ) + self.up2 = self.make_unpool_layer(curr_dim) + + self.merge = self.make_merge_layer(curr_dim) + + def call(self, x): + up1 = self.up1(x) + max1 = self.max1(x) + low1 = self.low1(max1) + low2 = self.low2(low1) + low3 = self.low3(low2) + up2 = self.up2(low3) + return self.merge([up1, up2]) + + def make_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): + layers = [layer(k, inp_dim, out_dim, **kwargs)] + for _ in range(1, modules): + layers.append(layer(k, out_dim, out_dim, **kwargs)) + return tf.keras.Sequential(layers) + + def make_layer_revr(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): + layers = [] + for _ in range(modules - 1): + layers.append(layer(k, inp_dim, inp_dim, **kwargs)) + layers.append(layer(k, inp_dim, out_dim, **kwargs)) + return tf.keras.Sequential(layers) + + def make_up_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + + def make_low_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + + def make_hg_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + + def make_hg_layer_revr(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): + return self.make_layer_revr(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + + def make_pool_layer(self, dim): + return tf.identity #tf.keras.Sequential([]) # tf.keras.layers.MaxPool2D(strides=2) + + def make_unpool_layer(self, dim): + return tf.keras.layers.UpSampling2D(2) + + def make_merge_layer(self, dim): + return tf.keras.layers.Add() + +def test(): + n = 5 + dims = [256, 256, 384, 384, 384, 512] + modules = [2, 2, 2, 2, 2, 4] + return kp_module(n, dims, modules), tf.keras.Input((1, 408, 408, 3)) From 8d2722573f01612c3f36720c41209b79250333df Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 3 Feb 2021 16:27:57 -0500 Subject: [PATCH 002/132] Some bug fixes --- centernet/modeling/layers/nn_blocks.py | 83 ++++++++++++++++---------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 0985f67e8..d6331c41f 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -1,40 +1,56 @@ import tensorflow as tf from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock +# from yolo.modeling.layers.nn_blocks import DarkResidual as ResidualBlock -class kp_module(tf.keras.Model): +def ConvBNRelu2D(*args, **kwargs): + return tf.keras.layers.Conv2D( + *args, + activation='relu', + use_bias=True, + **kwargs + ) + +class HourglassBlock(tf.keras.layers.Layer): def __init__( - self, n, dims, modules, layer=ResidualBlock, **kwargs + self, n, dims, modules, k=0, **kwargs ): super().__init__() self.n = n + self.k = k + self.modules = modules + self.dims = dims + + self._kwargs = kwargs - curr_mod = modules[0] - next_mod = modules[1] + def build(self, input_shape): + modules = self.modules + dims = self.dims + k = self.k + kwargs = self._kwargs - curr_dim = dims[0] - next_dim = dims[1] + curr_mod = modules[k] + next_mod = modules[k + 1] + + curr_dim = dims[k + 0] + next_dim = dims[k + 1] self.up1 = self.make_up_layer( - 3, curr_dim, curr_dim, curr_mod, - layer=layer, **kwargs + 3, curr_dim, curr_dim, curr_mod, **kwargs ) self.max1 = self.make_pool_layer(curr_dim) self.low1 = self.make_hg_layer( - 3, curr_dim, next_dim, curr_mod, - layer=layer, **kwargs + 3, curr_dim, next_dim, curr_mod, **kwargs ) self.low2 = type(self)( - n - 1, dims[1:], modules[1:], layer=layer, **kwargs - ) if self.n > 1 else \ + self.n, dims, modules, k=k+1, **kwargs + ) if self.n - k > 1 else \ self.make_low_layer( - 3, next_dim, next_dim, next_mod, - layer=layer, **kwargs + 3, next_dim, next_dim, next_mod, **kwargs ) self.low3 = self.make_hg_layer_revr( - 3, next_dim, curr_dim, curr_mod, - layer=layer, **kwargs + 3, next_dim, curr_dim, curr_mod, **kwargs ) self.up2 = self.make_unpool_layer(curr_dim) @@ -49,30 +65,30 @@ def call(self, x): up2 = self.up2(low3) return self.merge([up1, up2]) - def make_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): - layers = [layer(k, inp_dim, out_dim, **kwargs)] + def make_layer(self, k, inp_dim, out_dim, modules, **kwargs): + layers = [ResidualBlock(out_dim, 1, **kwargs)] for _ in range(1, modules): - layers.append(layer(k, out_dim, out_dim, **kwargs)) + layers.append(ResidualBlock(out_dim, 1, **kwargs)) return tf.keras.Sequential(layers) - def make_layer_revr(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): + def make_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): layers = [] for _ in range(modules - 1): - layers.append(layer(k, inp_dim, inp_dim, **kwargs)) - layers.append(layer(k, inp_dim, out_dim, **kwargs)) + layers.append(ResidualBlock(inp_dim, 1, **kwargs)) # inp_dim is not a bug + layers.append(ResidualBlock(out_dim, 1, **kwargs)) return tf.keras.Sequential(layers) - def make_up_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): - return self.make_layer(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + def make_up_layer(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) - def make_low_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): - return self.make_layer(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + def make_low_layer(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) - def make_hg_layer(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): - return self.make_layer(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + def make_hg_layer(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) - def make_hg_layer_revr(self, k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs): - return self.make_layer_revr(k, inp_dim, out_dim, modules, layer=tf.keras.layers.Conv2D, **kwargs) + def make_hg_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer_revr(k, inp_dim, out_dim, modules, **kwargs) def make_pool_layer(self, dim): return tf.identity #tf.keras.Sequential([]) # tf.keras.layers.MaxPool2D(strides=2) @@ -83,8 +99,13 @@ def make_unpool_layer(self, dim): def make_merge_layer(self, dim): return tf.keras.layers.Add() +kp_module = HourglassBlock + def test(): n = 5 dims = [256, 256, 384, 384, 384, 512] modules = [2, 2, 2, 2, 2, 4] - return kp_module(n, dims, modules), tf.keras.Input((1, 408, 408, 3)) + # n = 1 + # dims = [384, 512] + # modules = [2, 4] + return kp_module(n, dims, modules), tf.keras.Input((512, 512, 256)) From d30d4b0f7fb44217ca2b3a8d43ccbb3b0a2d66a8 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 3 Feb 2021 17:08:01 -0500 Subject: [PATCH 003/132] Doesn't crash anymore --- centernet/modeling/layers/nn_blocks.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index d6331c41f..7662e704d 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -1,15 +1,6 @@ import tensorflow as tf from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock -# from yolo.modeling.layers.nn_blocks import DarkResidual as ResidualBlock - -def ConvBNRelu2D(*args, **kwargs): - return tf.keras.layers.Conv2D( - *args, - activation='relu', - use_bias=True, - **kwargs - ) class HourglassBlock(tf.keras.layers.Layer): def __init__( @@ -66,7 +57,7 @@ def call(self, x): return self.merge([up1, up2]) def make_layer(self, k, inp_dim, out_dim, modules, **kwargs): - layers = [ResidualBlock(out_dim, 1, **kwargs)] + layers = [ResidualBlock(out_dim, 1, use_projection=True, **kwargs)] for _ in range(1, modules): layers.append(ResidualBlock(out_dim, 1, **kwargs)) return tf.keras.Sequential(layers) @@ -75,7 +66,7 @@ def make_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): layers = [] for _ in range(modules - 1): layers.append(ResidualBlock(inp_dim, 1, **kwargs)) # inp_dim is not a bug - layers.append(ResidualBlock(out_dim, 1, **kwargs)) + layers.append(ResidualBlock(out_dim, 1, use_projection=True, **kwargs)) return tf.keras.Sequential(layers) def make_up_layer(self, k, inp_dim, out_dim, modules, **kwargs): @@ -91,7 +82,7 @@ def make_hg_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): return self.make_layer_revr(k, inp_dim, out_dim, modules, **kwargs) def make_pool_layer(self, dim): - return tf.identity #tf.keras.Sequential([]) # tf.keras.layers.MaxPool2D(strides=2) + return tf.keras.layers.MaxPool2D(strides=2) #tf.identity def make_unpool_layer(self, dim): return tf.keras.layers.UpSampling2D(2) @@ -105,7 +96,4 @@ def test(): n = 5 dims = [256, 256, 384, 384, 384, 512] modules = [2, 2, 2, 2, 2, 4] - # n = 1 - # dims = [384, 512] - # modules = [2, 4] return kp_module(n, dims, modules), tf.keras.Input((512, 512, 256)) From ab739f708d67a86d002f929a2965c465330ae580 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 3 Feb 2021 17:52:22 -0500 Subject: [PATCH 004/132] Some very basic documentation --- centernet/modeling/layers/nn_blocks.py | 33 +++++++++++++-------- centernet/modeling/layers/nn_blocks_test.py | 15 ++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 centernet/modeling/layers/nn_blocks_test.py diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 7662e704d..9bb29d449 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -3,16 +3,33 @@ from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock class HourglassBlock(tf.keras.layers.Layer): + """ + An CornerNet-style implementation of the hourglass block used in the + Hourglass-104 backbone. + """ def __init__( - self, n, dims, modules, k=0, **kwargs + self, dims, modules, k=0, **kwargs ): + """ + Initializes a block for the hourglass backbone. The first k entries of dims + and modules are ignored. + + Args: + dims: input sizes of residual blocks + modules: number of repetitions of the residual blocks in each hourglass + upsampling and downsampling + k: recursive parameter + """ super().__init__() - self.n = n + self.n = len(dims) - 1 self.k = k self.modules = modules self.dims = dims + assert len(dims) == len(modules), "dims and modules lists must have the " \ + "same length" + self._kwargs = kwargs def build(self, input_shape): @@ -35,7 +52,7 @@ def build(self, input_shape): 3, curr_dim, next_dim, curr_mod, **kwargs ) self.low2 = type(self)( - self.n, dims, modules, k=k+1, **kwargs + dims, modules, k=k+1, **kwargs ) if self.n - k > 1 else \ self.make_low_layer( 3, next_dim, next_dim, next_mod, **kwargs @@ -82,18 +99,10 @@ def make_hg_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): return self.make_layer_revr(k, inp_dim, out_dim, modules, **kwargs) def make_pool_layer(self, dim): - return tf.keras.layers.MaxPool2D(strides=2) #tf.identity + return tf.keras.layers.MaxPool2D(strides=2) def make_unpool_layer(self, dim): return tf.keras.layers.UpSampling2D(2) def make_merge_layer(self, dim): return tf.keras.layers.Add() - -kp_module = HourglassBlock - -def test(): - n = 5 - dims = [256, 256, 384, 384, 384, 512] - modules = [2, 2, 2, 2, 2, 4] - return kp_module(n, dims, modules), tf.keras.Input((512, 512, 256)) diff --git a/centernet/modeling/layers/nn_blocks_test.py b/centernet/modeling/layers/nn_blocks_test.py new file mode 100644 index 000000000..f836e978f --- /dev/null +++ b/centernet/modeling/layers/nn_blocks_test.py @@ -0,0 +1,15 @@ +from absl.testing import parameterized +import tensorflow as tf + +from centernet.modeling.layers import nn_blocks + +class NNBlocksTest(parameterized.TestCase, tf.test.TestCase): + def test_hourglass_block(self): + dims = [256, 256, 384, 384, 384, 512] + modules = [2, 2, 2, 2, 2, 4] + model = nn_blocks.HourglassBlock(dims, modules) + test_input = tf.keras.Input((512, 512, 256)) + _ = model(test_input) + +if __name__ == '__main__': + tf.test.main() From 831956caea625dedaf0f7b4fa8e9283356bf7064 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 4 Feb 2021 04:16:18 -0500 Subject: [PATCH 005/132] backbone and hourglass, TODO: pre-heatmap layers --- .../modeling/backbones/centernet_backbone.py | 127 +++++++++++++ .../backbones/centernet_backbone_test.py | 33 ++++ centernet/modeling/backbones/residual.py | 175 ++++++++++++++++++ 3 files changed, 335 insertions(+) create mode 100644 centernet/modeling/backbones/centernet_backbone.py create mode 100644 centernet/modeling/backbones/centernet_backbone_test.py create mode 100644 centernet/modeling/backbones/residual.py diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py new file mode 100644 index 000000000..1cd25544c --- /dev/null +++ b/centernet/modeling/backbones/centernet_backbone.py @@ -0,0 +1,127 @@ +import tensorflow as tf +# from centernet.modeling.layers.nn_blocks import kp_module +# from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock +from centernet.modeling.backbones.residual import ResidualBlock + +class CenterNetBackbone(tf.keras.Model): + """ + CenterNet Hourglass backbone + """ + def __init__(self, + order, + filter_sizes, + rep_sizes, + n_stacks=2, + pre_layers=None, + **kwargs): + """ + Args: + order: integer, number of downsampling (and subsequent upsampling) + steps per hourglass module + filter_sizes: list of filter sizes for Residual blocks + rep_sizes: list of residual block repetitions per down/upsample + n_stacks: integer, number of hourglass modules in backbone + pre_layers: tf.keras layer to process input before stacked hourglasses + """ + self._n_stacks = n_stacks + self._pre_layers = pre_layers + self._order = order + self._filter_sizes = filter_sizes + self._rep_sizes = rep_sizes + + super().__init__(**kwargs) + + def build(self, input_shape): + if self._pre_layers is None: + self._pre_layers = tf.keras.Sequential([ + tf.keras.layers.Conv2D(filters=128, kernel_size=7, strides=2, padding='same'), + tf.keras.layers.BatchNormalization(), + tf.keras.layers.ReLU(), + ResidualBlock(filters=256, use_projection=True, strides=2) # shape errors happening + ]) + + # Create hourglass stacks + self.hgs = tf.keras.Sequential([Hourglass(order=self._order, + filter_sizes=self._filter_sizes, + rep_sizes=self._rep_sizes) for _ in range(self._n_stacks)]) + + super(CenterNetBackbone, self).build(input_shape) + + def call(self, x): + x = self._pre_layers(x) + + # TODO: add intermediate layers + return self.hgs(x) + +class Hourglass(tf.keras.Model): + """ + Hourglass module + """ + def __init__(self, + order, + filter_sizes, + rep_sizes, + strides=1, + **kwargs): + """ + Args: + order: integer, number of downsampling (and subsequent upsampling) steps + filter_sizes: list of filter sizes for Residual blocks + rep_sizes: list of residual block repetitions per down/upsample + strides: integer, stride parameter to the Residual block + """ + self._order = order + self._filter_sizes = filter_sizes + self._rep_sizes = rep_sizes + self._strides = strides + + self._filters = filter_sizes[0] + self._reps = rep_sizes[0] + + super(Hourglass, self).__init__() + + def build(self, input_shape): + if self._order == 1: + # base case, residual block repetitions in most inner part of hourglass + blocks = [ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + self.blocks = tf.keras.Sequential(blocks) + + else: + # outer hourglass structures + main_block = [ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + side_block = [ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + self.pool = tf.keras.layers.MaxPool2D(pool_size=2) + + # recursively define inner hourglasses + self.inner_hg = Hourglass(order=self._order-1, + filter_sizes=self._filter_sizes[1:], + rep_sizes=self._rep_sizes[1:], + stride=self._strides) + end_block = [ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + self.upsample_layer = tf.keras.layers.UpSampling2D(size=2, + interpolation='nearest') + + self.main_block = tf.keras.Sequential(main_block, name="Main_Block") + self.side_block = tf.keras.Sequential(side_block, name="Side_Block") + self.end_block = tf.keras.Sequential(end_block, name="End_Block") + super(Hourglass, self).build(input_shape) + + def call(self, x): + if self._order == 1: + return self.blocks(x) + else: + x_pre_pooled = self.main_block(x) + x_side = self.side_block(x_pre_pooled) + x_pooled = self.pool(x_pre_pooled) + inner_output = self.inner_hg(x_pooled) + hg_output = self.end_block(inner_output) + + return self.upsample_layer(hg_output) + x_side \ No newline at end of file diff --git a/centernet/modeling/backbones/centernet_backbone_test.py b/centernet/modeling/backbones/centernet_backbone_test.py new file mode 100644 index 000000000..5a7ff6ce2 --- /dev/null +++ b/centernet/modeling/backbones/centernet_backbone_test.py @@ -0,0 +1,33 @@ +import tensorflow as tf +from centernet.modeling.backbones.centernet_backbone import Hourglass +from centernet.modeling.backbones.centernet_backbone import CenterNetBackbone + +if __name__ == '__main__': + order = 5 + filter_sizes = [256, 256, 384, 384, 384, 512] + rep_sizes = [2, 2, 2, 2, 2, 4] + + # Testing single hourglass + hg = Hourglass(order=order, + filter_sizes=filter_sizes, + rep_sizes=rep_sizes) + + test_input_shape = (1, 512, 512, 256) + + hg.build(input_shape=test_input_shape) + hg.summary() + tf.print("Made an hourglass!") + + x = tf.keras.Input(shape=(512, 512, 256), batch_size=1) + out = hg(x) + tf.print("Output shape of hourglass module is {} should be {}".format(hg.output_shape, test_input_shape)) + + # Testing backbone + backbone = CenterNetBackbone(n_stacks=2, + pre_layers=None, + order=order, + filter_sizes=filter_sizes, + rep_sizes=rep_sizes) + backbone.build(input_shape=(1, 512, 512, 3)) + backbone.summary() + tf.print("Made backbone!") diff --git a/centernet/modeling/backbones/residual.py b/centernet/modeling/backbones/residual.py new file mode 100644 index 000000000..afe97fa4e --- /dev/null +++ b/centernet/modeling/backbones/residual.py @@ -0,0 +1,175 @@ +# Copy and paste from official.vision.beta.modeling.layers.nn_blocks +# because had trouble pip installing some dependencies that are not +# used for the residual block +import tensorflow as tf +from official.modeling import tf_utils + + +class ResidualBlock(tf.keras.layers.Layer): + """A residual block.""" + + def __init__(self, + filters, + strides, + use_projection=False, + se_ratio=None, + stochastic_depth_drop_rate=None, + kernel_initializer='VarianceScaling', + kernel_regularizer=None, + bias_regularizer=None, + activation='relu', + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + **kwargs): + """A residual block with BN after convolutions. + + Args: + filters: `int` number of filters for the first two convolutions. Note that + the third and final convolution will use 4 times as many filters. + strides: `int` block stride. If greater than 1, this block will ultimately + downsample the input. + use_projection: `bool` for whether this block should use a projection + shortcut (versus the default identity shortcut). This is usually `True` + for the first block of a block group, which may change the number of + filters and the resolution. + se_ratio: `float` or None. Ratio of the Squeeze-and-Excitation layer. + stochastic_depth_drop_rate: `float` or None. if not None, drop rate for + the stochastic depth layer. + kernel_initializer: kernel_initializer for convolutional layers. + kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + Default to None. + bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. + Default to None. + activation: `str` name of the activation function. + use_sync_bn: if True, use synchronized batch normalization. + norm_momentum: `float` normalization omentum for the moving average. + norm_epsilon: `float` small float added to variance to avoid dividing by + zero. + **kwargs: keyword arguments to be passed. + """ + super(ResidualBlock, self).__init__(**kwargs) + + self._filters = filters + self._strides = strides + self._use_projection = use_projection + self._se_ratio = se_ratio + self._use_sync_bn = use_sync_bn + self._activation = activation + self._stochastic_depth_drop_rate = stochastic_depth_drop_rate + self._kernel_initializer = kernel_initializer + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon + self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + + if use_sync_bn: + self._norm = tf.keras.layers.experimental.SyncBatchNormalization + else: + self._norm = tf.keras.layers.BatchNormalization + if tf.keras.backend.image_data_format() == 'channels_last': + self._bn_axis = -1 + else: + self._bn_axis = 1 + self._activation_fn = tf_utils.get_activation(activation) + + def build(self, input_shape): + if self._use_projection: + self._shortcut = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=1, + strides=self._strides, + use_bias=False, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + self._norm0 = self._norm( + axis=self._bn_axis, + momentum=self._norm_momentum, + epsilon=self._norm_epsilon) + + self._conv1 = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=3, + strides=self._strides, + padding='same', + use_bias=False, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + self._norm1 = self._norm( + axis=self._bn_axis, + momentum=self._norm_momentum, + epsilon=self._norm_epsilon) + + self._conv2 = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=3, + strides=1, + padding='same', + use_bias=False, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + self._norm2 = self._norm( + axis=self._bn_axis, + momentum=self._norm_momentum, + epsilon=self._norm_epsilon) + + if self._se_ratio and self._se_ratio > 0 and self._se_ratio <= 1: + self._squeeze_excitation = nn_layers.SqueezeExcitation( + in_filters=self._filters, + out_filters=self._filters, + se_ratio=self._se_ratio, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + else: + self._squeeze_excitation = None + + if self._stochastic_depth_drop_rate: + self._stochastic_depth = nn_layers.StochasticDepth( + self._stochastic_depth_drop_rate) + else: + self._stochastic_depth = None + + super(ResidualBlock, self).build(input_shape) + + def get_config(self): + config = { + 'filters': self._filters, + 'strides': self._strides, + 'use_projection': self._use_projection, + 'se_ratio': self._se_ratio, + 'stochastic_depth_drop_rate': self._stochastic_depth_drop_rate, + 'kernel_initializer': self._kernel_initializer, + 'kernel_regularizer': self._kernel_regularizer, + 'bias_regularizer': self._bias_regularizer, + 'activation': self._activation, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon + } + base_config = super(ResidualBlock, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def call(self, inputs, training=None): + shortcut = inputs + if self._use_projection: + shortcut = self._shortcut(shortcut) + shortcut = self._norm0(shortcut) + + x = self._conv1(inputs) + x = self._norm1(x) + x = self._activation_fn(x) + + x = self._conv2(x) + x = self._norm2(x) + + if self._squeeze_excitation: + x = self._squeeze_excitation(x) + + if self._stochastic_depth: + x = self._stochastic_depth(x, training=training) + + return self._activation_fn(x + shortcut) \ No newline at end of file From e7905b5ab7af46b862dd32500793cf593622e45c Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 4 Feb 2021 11:34:03 -0500 Subject: [PATCH 006/132] Reorganize code --- .../modeling/backbones/centernet_backbone.py | 98 ++-------- .../backbones/centernet_backbone_test.py | 15 +- centernet/modeling/layers/nn_blocks.py | 174 ++++++++---------- centernet/modeling/layers/nn_blocks_test.py | 112 ++++++++++- 4 files changed, 203 insertions(+), 196 deletions(-) diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py index 1cd25544c..5157741c5 100644 --- a/centernet/modeling/backbones/centernet_backbone.py +++ b/centernet/modeling/backbones/centernet_backbone.py @@ -1,27 +1,28 @@ import tensorflow as tf -# from centernet.modeling.layers.nn_blocks import kp_module -# from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock -from centernet.modeling.backbones.residual import ResidualBlock +from centernet.modeling.layers import nn_blocks + +# from official.vision.beta.modeling.layers import official_nn_blocks +from centernet.modeling.backbones import residual as official_nn_blocks class CenterNetBackbone(tf.keras.Model): """ CenterNet Hourglass backbone """ - def __init__(self, - order, - filter_sizes, + def __init__(self, + order, + filter_sizes, rep_sizes, n_stacks=2, pre_layers=None, **kwargs): """ Args: - order: integer, number of downsampling (and subsequent upsampling) + order: integer, number of downsampling (and subsequent upsampling) steps per hourglass module filter_sizes: list of filter sizes for Residual blocks rep_sizes: list of residual block repetitions per down/upsample n_stacks: integer, number of hourglass modules in backbone - pre_layers: tf.keras layer to process input before stacked hourglasses + pre_layers: tf.keras layer to process input before stacked hourglasses """ self._n_stacks = n_stacks self._pre_layers = pre_layers @@ -30,98 +31,25 @@ def __init__(self, self._rep_sizes = rep_sizes super().__init__(**kwargs) - + def build(self, input_shape): if self._pre_layers is None: self._pre_layers = tf.keras.Sequential([ tf.keras.layers.Conv2D(filters=128, kernel_size=7, strides=2, padding='same'), tf.keras.layers.BatchNormalization(), tf.keras.layers.ReLU(), - ResidualBlock(filters=256, use_projection=True, strides=2) # shape errors happening + official_nn_blocks.ResidualBlock(filters=256, use_projection=True, strides=2) # shape errors happening ]) # Create hourglass stacks - self.hgs = tf.keras.Sequential([Hourglass(order=self._order, + self.hgs = tf.keras.Sequential([nn_blocks.HourglassBlock(order=self._order, filter_sizes=self._filter_sizes, rep_sizes=self._rep_sizes) for _ in range(self._n_stacks)]) - super(CenterNetBackbone, self).build(input_shape) + super().build(input_shape) def call(self, x): x = self._pre_layers(x) # TODO: add intermediate layers return self.hgs(x) - -class Hourglass(tf.keras.Model): - """ - Hourglass module - """ - def __init__(self, - order, - filter_sizes, - rep_sizes, - strides=1, - **kwargs): - """ - Args: - order: integer, number of downsampling (and subsequent upsampling) steps - filter_sizes: list of filter sizes for Residual blocks - rep_sizes: list of residual block repetitions per down/upsample - strides: integer, stride parameter to the Residual block - """ - self._order = order - self._filter_sizes = filter_sizes - self._rep_sizes = rep_sizes - self._strides = strides - - self._filters = filter_sizes[0] - self._reps = rep_sizes[0] - - super(Hourglass, self).__init__() - - def build(self, input_shape): - if self._order == 1: - # base case, residual block repetitions in most inner part of hourglass - blocks = [ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - self.blocks = tf.keras.Sequential(blocks) - - else: - # outer hourglass structures - main_block = [ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - side_block = [ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - self.pool = tf.keras.layers.MaxPool2D(pool_size=2) - - # recursively define inner hourglasses - self.inner_hg = Hourglass(order=self._order-1, - filter_sizes=self._filter_sizes[1:], - rep_sizes=self._rep_sizes[1:], - stride=self._strides) - end_block = [ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - self.upsample_layer = tf.keras.layers.UpSampling2D(size=2, - interpolation='nearest') - - self.main_block = tf.keras.Sequential(main_block, name="Main_Block") - self.side_block = tf.keras.Sequential(side_block, name="Side_Block") - self.end_block = tf.keras.Sequential(end_block, name="End_Block") - super(Hourglass, self).build(input_shape) - - def call(self, x): - if self._order == 1: - return self.blocks(x) - else: - x_pre_pooled = self.main_block(x) - x_side = self.side_block(x_pre_pooled) - x_pooled = self.pool(x_pre_pooled) - inner_output = self.inner_hg(x_pooled) - hg_output = self.end_block(inner_output) - - return self.upsample_layer(hg_output) + x_side \ No newline at end of file diff --git a/centernet/modeling/backbones/centernet_backbone_test.py b/centernet/modeling/backbones/centernet_backbone_test.py index 5a7ff6ce2..8d4511ac4 100644 --- a/centernet/modeling/backbones/centernet_backbone_test.py +++ b/centernet/modeling/backbones/centernet_backbone_test.py @@ -1,5 +1,5 @@ import tensorflow as tf -from centernet.modeling.backbones.centernet_backbone import Hourglass +from centernet.modeling.layers.nn_blocks import HourglassBlock from centernet.modeling.backbones.centernet_backbone import CenterNetBackbone if __name__ == '__main__': @@ -8,14 +8,13 @@ rep_sizes = [2, 2, 2, 2, 2, 4] # Testing single hourglass - hg = Hourglass(order=order, + hg = HourglassBlock(order=order, filter_sizes=filter_sizes, rep_sizes=rep_sizes) - + test_input_shape = (1, 512, 512, 256) hg.build(input_shape=test_input_shape) - hg.summary() tf.print("Made an hourglass!") x = tf.keras.Input(shape=(512, 512, 256), batch_size=1) @@ -23,10 +22,10 @@ tf.print("Output shape of hourglass module is {} should be {}".format(hg.output_shape, test_input_shape)) # Testing backbone - backbone = CenterNetBackbone(n_stacks=2, - pre_layers=None, - order=order, - filter_sizes=filter_sizes, + backbone = CenterNetBackbone(n_stacks=2, + pre_layers=None, + order=order, + filter_sizes=filter_sizes, rep_sizes=rep_sizes) backbone.build(input_shape=(1, 512, 512, 3)) backbone.summary() diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 9bb29d449..35dc4c9c9 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -1,108 +1,78 @@ import tensorflow as tf -from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock +# from centernet.modeling.layers.nn_blocks import kp_module +# from official.vision.beta.modeling.layers import official_nn_blocks +from centernet.modeling.backbones import residual as official_nn_blocks class HourglassBlock(tf.keras.layers.Layer): - """ - An CornerNet-style implementation of the hourglass block used in the - Hourglass-104 backbone. - """ - def __init__( - self, dims, modules, k=0, **kwargs - ): """ - Initializes a block for the hourglass backbone. The first k entries of dims - and modules are ignored. - - Args: - dims: input sizes of residual blocks - modules: number of repetitions of the residual blocks in each hourglass - upsampling and downsampling - k: recursive parameter + Hourglass module """ - super().__init__() - - self.n = len(dims) - 1 - self.k = k - self.modules = modules - self.dims = dims - - assert len(dims) == len(modules), "dims and modules lists must have the " \ - "same length" - - self._kwargs = kwargs - - def build(self, input_shape): - modules = self.modules - dims = self.dims - k = self.k - kwargs = self._kwargs - - curr_mod = modules[k] - next_mod = modules[k + 1] - - curr_dim = dims[k + 0] - next_dim = dims[k + 1] - - self.up1 = self.make_up_layer( - 3, curr_dim, curr_dim, curr_mod, **kwargs - ) - self.max1 = self.make_pool_layer(curr_dim) - self.low1 = self.make_hg_layer( - 3, curr_dim, next_dim, curr_mod, **kwargs - ) - self.low2 = type(self)( - dims, modules, k=k+1, **kwargs - ) if self.n - k > 1 else \ - self.make_low_layer( - 3, next_dim, next_dim, next_mod, **kwargs - ) - self.low3 = self.make_hg_layer_revr( - 3, next_dim, curr_dim, curr_mod, **kwargs - ) - self.up2 = self.make_unpool_layer(curr_dim) - - self.merge = self.make_merge_layer(curr_dim) - - def call(self, x): - up1 = self.up1(x) - max1 = self.max1(x) - low1 = self.low1(max1) - low2 = self.low2(low1) - low3 = self.low3(low2) - up2 = self.up2(low3) - return self.merge([up1, up2]) - - def make_layer(self, k, inp_dim, out_dim, modules, **kwargs): - layers = [ResidualBlock(out_dim, 1, use_projection=True, **kwargs)] - for _ in range(1, modules): - layers.append(ResidualBlock(out_dim, 1, **kwargs)) - return tf.keras.Sequential(layers) - - def make_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): - layers = [] - for _ in range(modules - 1): - layers.append(ResidualBlock(inp_dim, 1, **kwargs)) # inp_dim is not a bug - layers.append(ResidualBlock(out_dim, 1, use_projection=True, **kwargs)) - return tf.keras.Sequential(layers) - - def make_up_layer(self, k, inp_dim, out_dim, modules, **kwargs): - return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) - - def make_low_layer(self, k, inp_dim, out_dim, modules, **kwargs): - return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) - - def make_hg_layer(self, k, inp_dim, out_dim, modules, **kwargs): - return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) - - def make_hg_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): - return self.make_layer_revr(k, inp_dim, out_dim, modules, **kwargs) - - def make_pool_layer(self, dim): - return tf.keras.layers.MaxPool2D(strides=2) - - def make_unpool_layer(self, dim): - return tf.keras.layers.UpSampling2D(2) - - def make_merge_layer(self, dim): - return tf.keras.layers.Add() + def __init__(self, + order, + filter_sizes, + rep_sizes, + strides=1, + **kwargs): + """ + Args: + order: integer, number of downsampling (and subsequent upsampling) steps + filter_sizes: list of filter sizes for Residual blocks + rep_sizes: list of residual block repetitions per down/upsample + strides: integer, stride parameter to the Residual block + """ + self._order = order + self._filter_sizes = filter_sizes + self._rep_sizes = rep_sizes + self._strides = strides + + self._filters = filter_sizes[0] + self._reps = rep_sizes[0] + + super().__init__() + + def build(self, input_shape): + if self._order == 1: + # base case, residual block repetitions in most inner part of hourglass + blocks = [official_nn_blocks.ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + self.blocks = tf.keras.Sequential(blocks) + + else: + # outer hourglass structures + main_block = [official_nn_blocks.ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + side_block = [official_nn_blocks.ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + self.pool = tf.keras.layers.MaxPool2D(pool_size=2) + + # recursively define inner hourglasses + self.inner_hg = type(self)(order=self._order-1, + filter_sizes=self._filter_sizes[1:], + rep_sizes=self._rep_sizes[1:], + stride=self._strides) + end_block = [official_nn_blocks.ResidualBlock(filters=self._filters, + strides=self._strides, + use_projection=True) for _ in range(self._reps)] + self.upsample_layer = tf.keras.layers.UpSampling2D(size=2, + interpolation='nearest') + + self.main_block = tf.keras.Sequential(main_block, name="Main_Block") + self.side_block = tf.keras.Sequential(side_block, name="Side_Block") + self.end_block = tf.keras.Sequential(end_block, name="End_Block") + super().build(input_shape) + + def call(self, x): + if self._order == 1: + return self.blocks(x) + else: + x_pre_pooled = self.main_block(x) + x_side = self.side_block(x_pre_pooled) + x_pooled = self.pool(x_pre_pooled) + inner_output = self.inner_hg(x_pooled) + hg_output = self.end_block(inner_output) + + return self.upsample_layer(hg_output) + x_side diff --git a/centernet/modeling/layers/nn_blocks_test.py b/centernet/modeling/layers/nn_blocks_test.py index f836e978f..3952d37c7 100644 --- a/centernet/modeling/layers/nn_blocks_test.py +++ b/centernet/modeling/layers/nn_blocks_test.py @@ -3,11 +3,121 @@ from centernet.modeling.layers import nn_blocks +# the following is a literal translation of the PyTorch implementation +# https://github.com/xingyizhou/CenterNet/blob/master/src/lib/models/networks/large_hourglass.py +from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock + +class HourglassBlockPyTorch(tf.keras.layers.Layer): + """ + An CornerNet-style implementation of the hourglass block used in the + Hourglass-104 backbone. + """ + def __init__( + self, dims, modules, k=0, **kwargs + ): + """ + Initializes a block for the hourglass backbone. The first k entries of dims + and modules are ignored. + + Args: + dims: input sizes of residual blocks + modules: number of repetitions of the residual blocks in each hourglass + upsampling and downsampling + k: recursive parameter + """ + super().__init__() + + self.n = len(dims) - 1 + self.k = k + self.modules = modules + self.dims = dims + + assert len(dims) == len(modules), "dims and modules lists must have the " \ + "same length" + + self._kwargs = kwargs + + def build(self, input_shape): + modules = self.modules + dims = self.dims + k = self.k + kwargs = self._kwargs + + curr_mod = modules[k] + next_mod = modules[k + 1] + + curr_dim = dims[k + 0] + next_dim = dims[k + 1] + + self.up1 = self.make_up_layer( + 3, curr_dim, curr_dim, curr_mod, **kwargs + ) + self.max1 = self.make_pool_layer(curr_dim) + self.low1 = self.make_hg_layer( + 3, curr_dim, next_dim, curr_mod, **kwargs + ) + self.low2 = type(self)( + dims, modules, k=k+1, **kwargs + ) if self.n - k > 1 else \ + self.make_low_layer( + 3, next_dim, next_dim, next_mod, **kwargs + ) + self.low3 = self.make_hg_layer_revr( + 3, next_dim, curr_dim, curr_mod, **kwargs + ) + self.up2 = self.make_unpool_layer(curr_dim) + + self.merge = self.make_merge_layer(curr_dim) + + def call(self, x): + up1 = self.up1(x) + max1 = self.max1(x) + low1 = self.low1(max1) + low2 = self.low2(low1) + low3 = self.low3(low2) + up2 = self.up2(low3) + return self.merge([up1, up2]) + + def make_layer(self, k, inp_dim, out_dim, modules, **kwargs): + layers = [ResidualBlock(out_dim, 1, use_projection=True, **kwargs)] + for _ in range(1, modules): + layers.append(ResidualBlock(out_dim, 1, **kwargs)) + return tf.keras.Sequential(layers) + + def make_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): + layers = [] + for _ in range(modules - 1): + layers.append(ResidualBlock(inp_dim, 1, **kwargs)) # inp_dim is not a bug + layers.append(ResidualBlock(out_dim, 1, use_projection=True, **kwargs)) + return tf.keras.Sequential(layers) + + def make_up_layer(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) + + def make_low_layer(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) + + def make_hg_layer(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) + + def make_hg_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): + return self.make_layer_revr(k, inp_dim, out_dim, modules, **kwargs) + + def make_pool_layer(self, dim): + return tf.keras.layers.MaxPool2D(strides=2) + + def make_unpool_layer(self, dim): + return tf.keras.layers.UpSampling2D(2) + + def make_merge_layer(self, dim): + return tf.keras.layers.Add() + + class NNBlocksTest(parameterized.TestCase, tf.test.TestCase): def test_hourglass_block(self): dims = [256, 256, 384, 384, 384, 512] modules = [2, 2, 2, 2, 2, 4] - model = nn_blocks.HourglassBlock(dims, modules) + model = nn_blocks.HourglassBlock(len(dims) - 1, dims, modules) test_input = tf.keras.Input((512, 512, 256)) _ = model(test_input) From 889c8a6b9665603dbffa01b6e17f0c69508c0f7a Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 4 Feb 2021 11:52:32 -0500 Subject: [PATCH 007/132] Remove order parameter and infer it from the filter sizes --- .../modeling/backbones/centernet_backbone.py | 97 +++++++------ .../backbones/centernet_backbone_test.py | 44 +++--- centernet/modeling/backbones/residual.py | 4 +- centernet/modeling/layers/nn_blocks.py | 131 ++++++++++-------- centernet/modeling/layers/nn_blocks_test.py | 47 +++---- 5 files changed, 166 insertions(+), 157 deletions(-) diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py index 5157741c5..323634233 100644 --- a/centernet/modeling/backbones/centernet_backbone.py +++ b/centernet/modeling/backbones/centernet_backbone.py @@ -4,52 +4,57 @@ # from official.vision.beta.modeling.layers import official_nn_blocks from centernet.modeling.backbones import residual as official_nn_blocks + class CenterNetBackbone(tf.keras.Model): + """ + CenterNet Hourglass backbone + """ + + def __init__(self, + filter_sizes, + rep_sizes, + n_stacks=2, + pre_layers=None, + **kwargs): """ - CenterNet Hourglass backbone + Args: + order: integer, number of downsampling (and subsequent upsampling) + steps per hourglass module + filter_sizes: list of filter sizes for Residual blocks + rep_sizes: list of residual block repetitions per down/upsample + n_stacks: integer, number of hourglass modules in backbone + pre_layers: tf.keras layer to process input before stacked hourglasses """ - def __init__(self, - order, - filter_sizes, - rep_sizes, - n_stacks=2, - pre_layers=None, - **kwargs): - """ - Args: - order: integer, number of downsampling (and subsequent upsampling) - steps per hourglass module - filter_sizes: list of filter sizes for Residual blocks - rep_sizes: list of residual block repetitions per down/upsample - n_stacks: integer, number of hourglass modules in backbone - pre_layers: tf.keras layer to process input before stacked hourglasses - """ - self._n_stacks = n_stacks - self._pre_layers = pre_layers - self._order = order - self._filter_sizes = filter_sizes - self._rep_sizes = rep_sizes - - super().__init__(**kwargs) - - def build(self, input_shape): - if self._pre_layers is None: - self._pre_layers = tf.keras.Sequential([ - tf.keras.layers.Conv2D(filters=128, kernel_size=7, strides=2, padding='same'), - tf.keras.layers.BatchNormalization(), - tf.keras.layers.ReLU(), - official_nn_blocks.ResidualBlock(filters=256, use_projection=True, strides=2) # shape errors happening - ]) - - # Create hourglass stacks - self.hgs = tf.keras.Sequential([nn_blocks.HourglassBlock(order=self._order, - filter_sizes=self._filter_sizes, - rep_sizes=self._rep_sizes) for _ in range(self._n_stacks)]) - - super().build(input_shape) - - def call(self, x): - x = self._pre_layers(x) - - # TODO: add intermediate layers - return self.hgs(x) + self._n_stacks = n_stacks + self._pre_layers = pre_layers + self._filter_sizes = filter_sizes + self._rep_sizes = rep_sizes + + super().__init__(**kwargs) + + def build(self, input_shape): + if self._pre_layers is None: + self._pre_layers = tf.keras.Sequential([ + tf.keras.layers.Conv2D( + filters=128, kernel_size=7, strides=2, padding='same'), + tf.keras.layers.BatchNormalization(), + tf.keras.layers.ReLU(), + official_nn_blocks.ResidualBlock( + filters=256, use_projection=True, + strides=2) # shape errors happening + ]) + + # Create hourglass stacks + self.hgs = tf.keras.Sequential([ + nn_blocks.HourglassBlock( + filter_sizes=self._filter_sizes, rep_sizes=self._rep_sizes) + for _ in range(self._n_stacks) + ]) + + super().build(input_shape) + + def call(self, x): + x = self._pre_layers(x) + + # TODO: add intermediate layers + return self.hgs(x) diff --git a/centernet/modeling/backbones/centernet_backbone_test.py b/centernet/modeling/backbones/centernet_backbone_test.py index 8d4511ac4..dd2bb4c4a 100644 --- a/centernet/modeling/backbones/centernet_backbone_test.py +++ b/centernet/modeling/backbones/centernet_backbone_test.py @@ -1,32 +1,30 @@ import tensorflow as tf -from centernet.modeling.layers.nn_blocks import HourglassBlock +from centernet.modeling.layers.nn_blocks import HourglassBlock from centernet.modeling.backbones.centernet_backbone import CenterNetBackbone if __name__ == '__main__': - order = 5 - filter_sizes = [256, 256, 384, 384, 384, 512] - rep_sizes = [2, 2, 2, 2, 2, 4] + filter_sizes = [256, 256, 384, 384, 384, 512] + rep_sizes = [2, 2, 2, 2, 2, 4] - # Testing single hourglass - hg = HourglassBlock(order=order, - filter_sizes=filter_sizes, - rep_sizes=rep_sizes) + # Testing single hourglass + hg = HourglassBlock(filter_sizes=filter_sizes, rep_sizes=rep_sizes) - test_input_shape = (1, 512, 512, 256) + test_input_shape = (1, 512, 512, 256) - hg.build(input_shape=test_input_shape) - tf.print("Made an hourglass!") + hg.build(input_shape=test_input_shape) + tf.print('Made an hourglass!') - x = tf.keras.Input(shape=(512, 512, 256), batch_size=1) - out = hg(x) - tf.print("Output shape of hourglass module is {} should be {}".format(hg.output_shape, test_input_shape)) + x = tf.keras.Input(shape=(512, 512, 256), batch_size=1) + out = hg(x) + tf.print('Output shape of hourglass module is {} should be {}'.format( + hg.output_shape, test_input_shape)) - # Testing backbone - backbone = CenterNetBackbone(n_stacks=2, - pre_layers=None, - order=order, - filter_sizes=filter_sizes, - rep_sizes=rep_sizes) - backbone.build(input_shape=(1, 512, 512, 3)) - backbone.summary() - tf.print("Made backbone!") + # Testing backbone + backbone = CenterNetBackbone( + n_stacks=2, + pre_layers=None, + filter_sizes=filter_sizes, + rep_sizes=rep_sizes) + backbone.build(input_shape=(1, 512, 512, 3)) + backbone.summary() + tf.print('Made backbone!') diff --git a/centernet/modeling/backbones/residual.py b/centernet/modeling/backbones/residual.py index afe97fa4e..70ac7091e 100644 --- a/centernet/modeling/backbones/residual.py +++ b/centernet/modeling/backbones/residual.py @@ -1,5 +1,5 @@ # Copy and paste from official.vision.beta.modeling.layers.nn_blocks -# because had trouble pip installing some dependencies that are not +# because had trouble pip installing some dependencies that are not # used for the residual block import tensorflow as tf from official.modeling import tf_utils @@ -172,4 +172,4 @@ def call(self, inputs, training=None): if self._stochastic_depth: x = self._stochastic_depth(x, training=training) - return self._activation_fn(x + shortcut) \ No newline at end of file + return self._activation_fn(x + shortcut) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 35dc4c9c9..05da7f4f7 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -4,75 +4,84 @@ # from official.vision.beta.modeling.layers import official_nn_blocks from centernet.modeling.backbones import residual as official_nn_blocks + class HourglassBlock(tf.keras.layers.Layer): + """ + Hourglass module + """ + + def __init__(self, filter_sizes, rep_sizes, strides=1, **kwargs): """ - Hourglass module + Args: + order: integer, number of downsampling (and subsequent upsampling) + steps + filter_sizes: list of filter sizes for Residual blocks + rep_sizes: list of residual block repetitions per down/upsample + strides: integer, stride parameter to the Residual block """ - def __init__(self, - order, - filter_sizes, - rep_sizes, - strides=1, - **kwargs): - """ - Args: - order: integer, number of downsampling (and subsequent upsampling) steps - filter_sizes: list of filter sizes for Residual blocks - rep_sizes: list of residual block repetitions per down/upsample - strides: integer, stride parameter to the Residual block - """ - self._order = order - self._filter_sizes = filter_sizes - self._rep_sizes = rep_sizes - self._strides = strides + self._order = len(filter_sizes) - 1 + self._filter_sizes = filter_sizes + self._rep_sizes = rep_sizes + self._strides = strides + + assert len(filter_sizes) == len(rep_sizes), 'filter size and ' \ + 'residual block repetition lists must have the same length' - self._filters = filter_sizes[0] - self._reps = rep_sizes[0] + self._filters = filter_sizes[0] + self._reps = rep_sizes[0] - super().__init__() + super().__init__() - def build(self, input_shape): - if self._order == 1: - # base case, residual block repetitions in most inner part of hourglass - blocks = [official_nn_blocks.ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - self.blocks = tf.keras.Sequential(blocks) + def build(self, input_shape): + if self._order == 1: + # base case, residual block repetitions in most inner part of hourglass + blocks = [ + official_nn_blocks.ResidualBlock( + filters=self._filters, strides=self._strides, use_projection=True) + for _ in range(self._reps) + ] + self.blocks = tf.keras.Sequential(blocks) - else: - # outer hourglass structures - main_block = [official_nn_blocks.ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - side_block = [official_nn_blocks.ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - self.pool = tf.keras.layers.MaxPool2D(pool_size=2) + else: + # outer hourglass structures + main_block = [ + official_nn_blocks.ResidualBlock( + filters=self._filters, strides=self._strides, use_projection=True) + for _ in range(self._reps) + ] + side_block = [ + official_nn_blocks.ResidualBlock( + filters=self._filters, strides=self._strides, use_projection=True) + for _ in range(self._reps) + ] + self.pool = tf.keras.layers.MaxPool2D(pool_size=2) - # recursively define inner hourglasses - self.inner_hg = type(self)(order=self._order-1, - filter_sizes=self._filter_sizes[1:], - rep_sizes=self._rep_sizes[1:], - stride=self._strides) - end_block = [official_nn_blocks.ResidualBlock(filters=self._filters, - strides=self._strides, - use_projection=True) for _ in range(self._reps)] - self.upsample_layer = tf.keras.layers.UpSampling2D(size=2, - interpolation='nearest') + # recursively define inner hourglasses + self.inner_hg = type(self)( + filter_sizes=self._filter_sizes[1:], + rep_sizes=self._rep_sizes[1:], + stride=self._strides) + end_block = [ + official_nn_blocks.ResidualBlock( + filters=self._filters, strides=self._strides, use_projection=True) + for _ in range(self._reps) + ] + self.upsample_layer = tf.keras.layers.UpSampling2D( + size=2, interpolation='nearest') - self.main_block = tf.keras.Sequential(main_block, name="Main_Block") - self.side_block = tf.keras.Sequential(side_block, name="Side_Block") - self.end_block = tf.keras.Sequential(end_block, name="End_Block") - super().build(input_shape) + self.main_block = tf.keras.Sequential(main_block, name='Main_Block') + self.side_block = tf.keras.Sequential(side_block, name='Side_Block') + self.end_block = tf.keras.Sequential(end_block, name='End_Block') + super().build(input_shape) - def call(self, x): - if self._order == 1: - return self.blocks(x) - else: - x_pre_pooled = self.main_block(x) - x_side = self.side_block(x_pre_pooled) - x_pooled = self.pool(x_pre_pooled) - inner_output = self.inner_hg(x_pooled) - hg_output = self.end_block(inner_output) + def call(self, x): + if self._order == 1: + return self.blocks(x) + else: + x_pre_pooled = self.main_block(x) + x_side = self.side_block(x_pre_pooled) + x_pooled = self.pool(x_pre_pooled) + inner_output = self.inner_hg(x_pooled) + hg_output = self.end_block(inner_output) - return self.upsample_layer(hg_output) + x_side + return self.upsample_layer(hg_output) + x_side diff --git a/centernet/modeling/layers/nn_blocks_test.py b/centernet/modeling/layers/nn_blocks_test.py index 3952d37c7..951193145 100644 --- a/centernet/modeling/layers/nn_blocks_test.py +++ b/centernet/modeling/layers/nn_blocks_test.py @@ -7,14 +7,14 @@ # https://github.com/xingyizhou/CenterNet/blob/master/src/lib/models/networks/large_hourglass.py from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock + class HourglassBlockPyTorch(tf.keras.layers.Layer): """ An CornerNet-style implementation of the hourglass block used in the Hourglass-104 backbone. """ - def __init__( - self, dims, modules, k=0, **kwargs - ): + + def __init__(self, dims, modules, k=0, **kwargs): """ Initializes a block for the hourglass backbone. The first k entries of dims and modules are ignored. @@ -27,13 +27,13 @@ def __init__( """ super().__init__() - self.n = len(dims) - 1 - self.k = k + self.n = len(dims) - 1 + self.k = k self.modules = modules self.dims = dims - assert len(dims) == len(modules), "dims and modules lists must have the " \ - "same length" + assert len(dims) == len(modules), 'dims and modules lists must have the ' \ + 'same length' self._kwargs = kwargs @@ -49,33 +49,28 @@ def build(self, input_shape): curr_dim = dims[k + 0] next_dim = dims[k + 1] - self.up1 = self.make_up_layer( - 3, curr_dim, curr_dim, curr_mod, **kwargs - ) + self.up1 = self.make_up_layer(3, curr_dim, curr_dim, curr_mod, **kwargs) self.max1 = self.make_pool_layer(curr_dim) - self.low1 = self.make_hg_layer( - 3, curr_dim, next_dim, curr_mod, **kwargs - ) + self.low1 = self.make_hg_layer(3, curr_dim, next_dim, curr_mod, **kwargs) self.low2 = type(self)( dims, modules, k=k+1, **kwargs ) if self.n - k > 1 else \ self.make_low_layer( 3, next_dim, next_dim, next_mod, **kwargs ) - self.low3 = self.make_hg_layer_revr( - 3, next_dim, curr_dim, curr_mod, **kwargs - ) - self.up2 = self.make_unpool_layer(curr_dim) + self.low3 = self.make_hg_layer_revr(3, next_dim, curr_dim, curr_mod, + **kwargs) + self.up2 = self.make_unpool_layer(curr_dim) self.merge = self.make_merge_layer(curr_dim) def call(self, x): - up1 = self.up1(x) + up1 = self.up1(x) max1 = self.max1(x) low1 = self.low1(max1) low2 = self.low2(low1) low3 = self.low3(low2) - up2 = self.up2(low3) + up2 = self.up2(low3) return self.merge([up1, up2]) def make_layer(self, k, inp_dim, out_dim, modules, **kwargs): @@ -85,11 +80,11 @@ def make_layer(self, k, inp_dim, out_dim, modules, **kwargs): return tf.keras.Sequential(layers) def make_layer_revr(self, k, inp_dim, out_dim, modules, **kwargs): - layers = [] - for _ in range(modules - 1): - layers.append(ResidualBlock(inp_dim, 1, **kwargs)) # inp_dim is not a bug - layers.append(ResidualBlock(out_dim, 1, use_projection=True, **kwargs)) - return tf.keras.Sequential(layers) + layers = [] + for _ in range(modules - 1): + layers.append(ResidualBlock(inp_dim, 1, **kwargs)) # inp_dim is not a bug + layers.append(ResidualBlock(out_dim, 1, use_projection=True, **kwargs)) + return tf.keras.Sequential(layers) def make_up_layer(self, k, inp_dim, out_dim, modules, **kwargs): return self.make_layer(k, inp_dim, out_dim, modules, **kwargs) @@ -114,12 +109,14 @@ def make_merge_layer(self, dim): class NNBlocksTest(parameterized.TestCase, tf.test.TestCase): + def test_hourglass_block(self): - dims = [256, 256, 384, 384, 384, 512] + dims = [256, 256, 384, 384, 384, 512] modules = [2, 2, 2, 2, 2, 4] model = nn_blocks.HourglassBlock(len(dims) - 1, dims, modules) test_input = tf.keras.Input((512, 512, 256)) _ = model(test_input) + if __name__ == '__main__': tf.test.main() From b31f2c7241f0a8cca43667992f407c01a2debdb0 Mon Sep 17 00:00:00 2001 From: Jack LeCroy <3073035+jacklecroy@users.noreply.github.com> Date: Thu, 4 Feb 2021 13:20:50 -0500 Subject: [PATCH 008/132] Add gaussian penalty function --- centernet/utils/groundtruth.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 centernet/utils/groundtruth.py diff --git a/centernet/utils/groundtruth.py b/centernet/utils/groundtruth.py new file mode 100644 index 000000000..1f11ebf32 --- /dev/null +++ b/centernet/utils/groundtruth.py @@ -0,0 +1,18 @@ +import tensorflow as tf + + +def gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: + """ + This represents the penalty reduction around a point. + Params: + radius (int): integer for radius of penalty reduction + type (tf.dtypes.DType): datatype of returned tensor + Returns: + tf.Tensor of shape (2 * radius + 1, 2 * radius + 1). + """ + width = 2 * radius + 1 + sigma = radius / 3 + x = tf.reshape(tf.range(width, dtype=type) - radius, (width, 1)) + y = tf.reshape(tf.range(width, dtype=type) - radius, (1, width)) + exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) + return tf.math.exp(exponent) From b46871c0438cd893cddb18091fce2c9da11c948d Mon Sep 17 00:00:00 2001 From: David Li Date: Sat, 6 Feb 2021 02:34:37 -0500 Subject: [PATCH 009/132] finished backbone, adding documentation+tests --- .../modeling/backbones/centernet_backbone.py | 91 ++++++-- .../backbones/centernet_backbone_test.py | 38 ++-- centernet/modeling/backbones/config.py | 4 + centernet/modeling/backbones/residual.py | 175 --------------- centernet/modeling/layers/nn_blocks.py | 199 ++++++++++++++++-- 5 files changed, 284 insertions(+), 223 deletions(-) create mode 100644 centernet/modeling/backbones/config.py delete mode 100644 centernet/modeling/backbones/residual.py diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py index 323634233..278c22d2c 100644 --- a/centernet/modeling/backbones/centernet_backbone.py +++ b/centernet/modeling/backbones/centernet_backbone.py @@ -1,9 +1,7 @@ import tensorflow as tf from centernet.modeling.layers import nn_blocks -# from official.vision.beta.modeling.layers import official_nn_blocks -from centernet.modeling.backbones import residual as official_nn_blocks - +from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks class CenterNetBackbone(tf.keras.Model): """ @@ -14,47 +12,96 @@ def __init__(self, filter_sizes, rep_sizes, n_stacks=2, - pre_layers=None, + downsample=True, **kwargs): """ Args: - order: integer, number of downsampling (and subsequent upsampling) - steps per hourglass module filter_sizes: list of filter sizes for Residual blocks rep_sizes: list of residual block repetitions per down/upsample n_stacks: integer, number of hourglass modules in backbone pre_layers: tf.keras layer to process input before stacked hourglasses """ self._n_stacks = n_stacks - self._pre_layers = pre_layers + self._downsample = downsample self._filter_sizes = filter_sizes self._rep_sizes = rep_sizes - + super().__init__(**kwargs) def build(self, input_shape): - if self._pre_layers is None: + # Create prelayers if downsampling input + if self._downsample: self._pre_layers = tf.keras.Sequential([ - tf.keras.layers.Conv2D( - filters=128, kernel_size=7, strides=2, padding='same'), - tf.keras.layers.BatchNormalization(), - tf.keras.layers.ReLU(), - official_nn_blocks.ResidualBlock( - filters=256, use_projection=True, - strides=2) # shape errors happening - ]) + nn_blocks.ConvBN( + filters=128, kernel_size=(7, 7), strides=(2, 2), + padding='same', use_bn=True, activation='reLU'), + official_nn_blocks.ResidualBlock( + filters=256, use_projection=True, + strides=2) + ], name="Prelayers") # Create hourglass stacks - self.hgs = tf.keras.Sequential([ + self.hgs = [ nn_blocks.HourglassBlock( filter_sizes=self._filter_sizes, rep_sizes=self._rep_sizes) for _ in range(self._n_stacks) - ]) + ] + + # Create some intermediate and postlayers to generate the heatmaps (document and make cleaner later) + inp_filters = self._filter_sizes[0] + + # cnvs + self.post_hg_convs = [nn_blocks.ConvBN( + filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), + padding='same', use_bn=True, activation='reLU') + for _ in range(self._n_stacks) + ] + #cnvs_ + self.inter_hg_convs1 = [nn_blocks.ConvBN( + filters=256, kernel_size=(1, 1), strides=(1, 1), + padding='same', use_bn=True, activation='linear') + for _ in range(self._n_stacks - 1) + ] + #inters_ + self.inter_hg_convs2 = [nn_blocks.ConvBN( + filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), + padding='same', use_bn=True, activation='linear') + for _ in range(self._n_stacks - 1) + ] + # inters + self.res = [official_nn_blocks.ResidualBlock( + filters=inp_filters, use_projection=True, + strides=2) + for _ in range(self._n_stacks - 1) + ] + + self.relu = tf.keras.layers.ReLU() super().build(input_shape) def call(self, x): - x = self._pre_layers(x) + x_inter = x + if self._downsample: + x_inter = self._pre_layers(x_inter) + + all_heatmaps = [] + + for i in range(self._n_stacks): + hg = self.hgs[i] + post_conv = self.post_hg_convs[i] + + x_hg = hg(x_inter) + x_hg = post_conv(x_hg) + + all_heatmaps.append(x_hg) + + if i < self._n_stacks - 1: + inter_hg_conv1 = self.inter_hg_convs1[i] + inter_hg_conv2 = self.inter_hg_convs2[i] + res = self.res[i] + + x_inter = inter_hg_conv1(x_inter) + inter_hg_conv2(x_hg) + x_inter = self.relu(x_inter) + x_inter = res(x_inter) - # TODO: add intermediate layers - return self.hgs(x) + return x_hg \ No newline at end of file diff --git a/centernet/modeling/backbones/centernet_backbone_test.py b/centernet/modeling/backbones/centernet_backbone_test.py index dd2bb4c4a..2a7d40a3f 100644 --- a/centernet/modeling/backbones/centernet_backbone_test.py +++ b/centernet/modeling/backbones/centernet_backbone_test.py @@ -5,26 +5,38 @@ if __name__ == '__main__': filter_sizes = [256, 256, 384, 384, 384, 512] rep_sizes = [2, 2, 2, 2, 2, 4] + hg_test_input_shape = (1, 512, 512, 256) + bb_test_input_shape = (1, 512, 512, 3) + x_hg = tf.ones(shape=hg_test_input_shape) + x_bb = tf.ones(shape=bb_test_input_shape) - # Testing single hourglass + tf.print("Testing Single hourglass") hg = HourglassBlock(filter_sizes=filter_sizes, rep_sizes=rep_sizes) - test_input_shape = (1, 512, 512, 256) - - hg.build(input_shape=test_input_shape) + hg.build(input_shape=hg_test_input_shape) tf.print('Made an hourglass!') + out = hg(x_hg) + tf.print('Hourglass module output shape:{} Expected shape:{}'.format( + hg.output_shape, hg_test_input_shape)) - x = tf.keras.Input(shape=(512, 512, 256), batch_size=1) - out = hg(x) - tf.print('Output shape of hourglass module is {} should be {}'.format( - hg.output_shape, test_input_shape)) - - # Testing backbone + tf.print("Testing backbone") backbone = CenterNetBackbone( n_stacks=2, - pre_layers=None, + downsample=True, filter_sizes=filter_sizes, rep_sizes=rep_sizes) - backbone.build(input_shape=(1, 512, 512, 3)) - backbone.summary() + + backbone.build(input_shape=bb_test_input_shape) + + # Backbone summary shows output shape to be multiple for hg modules + # Maybe this is because the hg call method has a conditional? + # This is also causing the final print statement to error. + backbone.summary() + + out = backbone(x_bb) + tf.print(tf.shape(out)) + tf.print('Made backbone!') + tf.print('Backbone output shape: {} Expected shape: {}'.format( + backbone.output_shape, test_input_shape + )) diff --git a/centernet/modeling/backbones/config.py b/centernet/modeling/backbones/config.py new file mode 100644 index 000000000..f57a2415a --- /dev/null +++ b/centernet/modeling/backbones/config.py @@ -0,0 +1,4 @@ +# (name, filterSizes, repSizes) +backbone = [ + ("Hourglass", ) +] \ No newline at end of file diff --git a/centernet/modeling/backbones/residual.py b/centernet/modeling/backbones/residual.py deleted file mode 100644 index 70ac7091e..000000000 --- a/centernet/modeling/backbones/residual.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copy and paste from official.vision.beta.modeling.layers.nn_blocks -# because had trouble pip installing some dependencies that are not -# used for the residual block -import tensorflow as tf -from official.modeling import tf_utils - - -class ResidualBlock(tf.keras.layers.Layer): - """A residual block.""" - - def __init__(self, - filters, - strides, - use_projection=False, - se_ratio=None, - stochastic_depth_drop_rate=None, - kernel_initializer='VarianceScaling', - kernel_regularizer=None, - bias_regularizer=None, - activation='relu', - use_sync_bn=False, - norm_momentum=0.99, - norm_epsilon=0.001, - **kwargs): - """A residual block with BN after convolutions. - - Args: - filters: `int` number of filters for the first two convolutions. Note that - the third and final convolution will use 4 times as many filters. - strides: `int` block stride. If greater than 1, this block will ultimately - downsample the input. - use_projection: `bool` for whether this block should use a projection - shortcut (versus the default identity shortcut). This is usually `True` - for the first block of a block group, which may change the number of - filters and the resolution. - se_ratio: `float` or None. Ratio of the Squeeze-and-Excitation layer. - stochastic_depth_drop_rate: `float` or None. if not None, drop rate for - the stochastic depth layer. - kernel_initializer: kernel_initializer for convolutional layers. - kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. - Default to None. - bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. - Default to None. - activation: `str` name of the activation function. - use_sync_bn: if True, use synchronized batch normalization. - norm_momentum: `float` normalization omentum for the moving average. - norm_epsilon: `float` small float added to variance to avoid dividing by - zero. - **kwargs: keyword arguments to be passed. - """ - super(ResidualBlock, self).__init__(**kwargs) - - self._filters = filters - self._strides = strides - self._use_projection = use_projection - self._se_ratio = se_ratio - self._use_sync_bn = use_sync_bn - self._activation = activation - self._stochastic_depth_drop_rate = stochastic_depth_drop_rate - self._kernel_initializer = kernel_initializer - self._norm_momentum = norm_momentum - self._norm_epsilon = norm_epsilon - self._kernel_regularizer = kernel_regularizer - self._bias_regularizer = bias_regularizer - - if use_sync_bn: - self._norm = tf.keras.layers.experimental.SyncBatchNormalization - else: - self._norm = tf.keras.layers.BatchNormalization - if tf.keras.backend.image_data_format() == 'channels_last': - self._bn_axis = -1 - else: - self._bn_axis = 1 - self._activation_fn = tf_utils.get_activation(activation) - - def build(self, input_shape): - if self._use_projection: - self._shortcut = tf.keras.layers.Conv2D( - filters=self._filters, - kernel_size=1, - strides=self._strides, - use_bias=False, - kernel_initializer=self._kernel_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) - self._norm0 = self._norm( - axis=self._bn_axis, - momentum=self._norm_momentum, - epsilon=self._norm_epsilon) - - self._conv1 = tf.keras.layers.Conv2D( - filters=self._filters, - kernel_size=3, - strides=self._strides, - padding='same', - use_bias=False, - kernel_initializer=self._kernel_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) - self._norm1 = self._norm( - axis=self._bn_axis, - momentum=self._norm_momentum, - epsilon=self._norm_epsilon) - - self._conv2 = tf.keras.layers.Conv2D( - filters=self._filters, - kernel_size=3, - strides=1, - padding='same', - use_bias=False, - kernel_initializer=self._kernel_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) - self._norm2 = self._norm( - axis=self._bn_axis, - momentum=self._norm_momentum, - epsilon=self._norm_epsilon) - - if self._se_ratio and self._se_ratio > 0 and self._se_ratio <= 1: - self._squeeze_excitation = nn_layers.SqueezeExcitation( - in_filters=self._filters, - out_filters=self._filters, - se_ratio=self._se_ratio, - kernel_initializer=self._kernel_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) - else: - self._squeeze_excitation = None - - if self._stochastic_depth_drop_rate: - self._stochastic_depth = nn_layers.StochasticDepth( - self._stochastic_depth_drop_rate) - else: - self._stochastic_depth = None - - super(ResidualBlock, self).build(input_shape) - - def get_config(self): - config = { - 'filters': self._filters, - 'strides': self._strides, - 'use_projection': self._use_projection, - 'se_ratio': self._se_ratio, - 'stochastic_depth_drop_rate': self._stochastic_depth_drop_rate, - 'kernel_initializer': self._kernel_initializer, - 'kernel_regularizer': self._kernel_regularizer, - 'bias_regularizer': self._bias_regularizer, - 'activation': self._activation, - 'use_sync_bn': self._use_sync_bn, - 'norm_momentum': self._norm_momentum, - 'norm_epsilon': self._norm_epsilon - } - base_config = super(ResidualBlock, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - - def call(self, inputs, training=None): - shortcut = inputs - if self._use_projection: - shortcut = self._shortcut(shortcut) - shortcut = self._norm0(shortcut) - - x = self._conv1(inputs) - x = self._norm1(x) - x = self._activation_fn(x) - - x = self._conv2(x) - x = self._norm2(x) - - if self._squeeze_excitation: - x = self._squeeze_excitation(x) - - if self._stochastic_depth: - x = self._stochastic_depth(x, training=training) - - return self._activation_fn(x + shortcut) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 05da7f4f7..4db1d1ed8 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -1,9 +1,6 @@ import tensorflow as tf - -# from centernet.modeling.layers.nn_blocks import kp_module -# from official.vision.beta.modeling.layers import official_nn_blocks -from centernet.modeling.backbones import residual as official_nn_blocks - +from official.modeling import tf_utils +from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks class HourglassBlock(tf.keras.layers.Layer): """ @@ -13,8 +10,6 @@ class HourglassBlock(tf.keras.layers.Layer): def __init__(self, filter_sizes, rep_sizes, strides=1, **kwargs): """ Args: - order: integer, number of downsampling (and subsequent upsampling) - steps filter_sizes: list of filter sizes for Residual blocks rep_sizes: list of residual block repetitions per down/upsample strides: integer, stride parameter to the Residual block @@ -49,29 +44,40 @@ def build(self, input_shape): filters=self._filters, strides=self._strides, use_projection=True) for _ in range(self._reps) ] + self.main_block = tf.keras.Sequential(main_block, name='Main_Block') + side_block = [ official_nn_blocks.ResidualBlock( filters=self._filters, strides=self._strides, use_projection=True) for _ in range(self._reps) ] + self.side_block = tf.keras.Sequential(side_block, name='Side_Block') + self.pool = tf.keras.layers.MaxPool2D(pool_size=2) # recursively define inner hourglasses - self.inner_hg = type(self)( + if (self._order == 1): + self.inner_hg = HourglassBlockBase( + filters=self._filter_sizes[-1], + reps=self._rep_sizes[-1], + strides=self._strides) + else: + self.inner_hg = type(self)( filter_sizes=self._filter_sizes[1:], rep_sizes=self._rep_sizes[1:], - stride=self._strides) + strides=self._strides) + + # outer hourglass structures end_block = [ official_nn_blocks.ResidualBlock( filters=self._filters, strides=self._strides, use_projection=True) for _ in range(self._reps) ] + self.end_block = tf.keras.Sequential(end_block, name='End_Block') + self.upsample_layer = tf.keras.layers.UpSampling2D( size=2, interpolation='nearest') - self.main_block = tf.keras.Sequential(main_block, name='Main_Block') - self.side_block = tf.keras.Sequential(side_block, name='Side_Block') - self.end_block = tf.keras.Sequential(end_block, name='End_Block') super().build(input_shape) def call(self, x): @@ -83,5 +89,172 @@ def call(self, x): x_pooled = self.pool(x_pre_pooled) inner_output = self.inner_hg(x_pooled) hg_output = self.end_block(inner_output) - return self.upsample_layer(hg_output) + x_side + +# Copied from YOLO (yolo/modeling/layers/nn_blocks) +class Identity(tf.keras.layers.Layer): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def call(self, input): + return input + +# Copied from YOLO (yolo/modeling/layers/nn_blocks) +class ConvBN(tf.keras.layers.Layer): + """ + Modified Convolution layer to match that of the DarkNet Library. The Layer is a standards combination of Conv BatchNorm Activation, + however, the use of bias in the conv is determined by the use of batch normalization. + Cross Stage Partial networks (CSPNets) were proposed in: + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang Chen, Jun-Wei Hsieh + CSPNet: A New Backbone that can Enhance Learning Capability of CNN. arXiv:1911.11929 + Args: + filters: integer for output depth, or the number of features to learn + kernel_size: integer or tuple for the shape of the weight matrix or kernel to learn + strides: integer of tuple how much to move the kernel after each kernel use + padding: string 'valid' or 'same', if same, then pad the image, else do not + dialtion_rate: tuple to indicate how much to modulate kernel weights and + how many pixels in a feature map to skip + kernel_initializer: string to indicate which function to use to initialize weights + bias_initializer: string to indicate which function to use to initialize bias + kernel_regularizer: string to indicate which function to use to regularizer weights + bias_regularizer: string to indicate which function to use to regularizer bias + use_bn: boolean for whether to use batch normalization + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics (across all input batches) + norm_moment: float for moment to use for batch normalization + norm_epsilon: float for batch normalization epsilon + activation: string or None for activation function to use in layer, + if None activation is replaced by linear + leaky_alpha: float to use as alpha if activation function is leaky + **kwargs: Keyword Arguments + """ + + def __init__( + self, + filters=1, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + dilation_rate=(1, 1), + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + bias_regularizer=None, + kernel_regularizer=None, # Specify the weight decay as the default will not work. + use_bn=True, + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + activation='leaky', + leaky_alpha=0.1, + **kwargs): + + # convolution params + self._filters = filters + self._kernel_size = kernel_size + self._strides = strides + self._padding = padding + self._dilation_rate = dilation_rate + self._kernel_initializer = kernel_initializer + self._bias_initializer = bias_initializer + self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + + # batch normalization params + self._use_bn = use_bn + self._use_sync_bn = use_sync_bn + self._norm_moment = norm_momentum + self._norm_epsilon = norm_epsilon + + if tf.keras.backend.image_data_format() == 'channels_last': + # format: (batch_size, height, width, channels) + self._bn_axis = -1 + else: + # format: (batch_size, channels, width, height) + self._bn_axis = 1 + + # activation params + self._activation = activation + self._leaky_alpha = leaky_alpha + + super().__init__(**kwargs) + + def build(self, input_shape): + kernel_size = self._kernel_size if isinstance(self._kernel_size, + int) else self._kernel_size[0] + dilation_rate = self._dilation_rate if isinstance( + self._dilation_rate, int) else self._dilation_rate[0] + if self._padding == 'same' and kernel_size != 1: + padding = dilation_rate * (kernel_size - 1) + left_shift = padding // 2 + self._zeropad = tf.keras.layers.ZeroPadding2D([[left_shift, left_shift], + [left_shift, left_shift]]) + else: + self._zeropad = Identity() + + use_bias = not self._use_bn + + self.conv = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=self._kernel_size, + strides=self._strides, + padding='valid', + dilation_rate=self._dilation_rate, + use_bias=use_bias, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + + if self._use_bn: + if self._use_sync_bn: + self.bn = tf.keras.layers.experimental.SyncBatchNormalization( + momentum=self._norm_moment, + epsilon=self._norm_epsilon, + axis=self._bn_axis) + else: + self.bn = tf.keras.layers.BatchNormalization( + momentum=self._norm_moment, + epsilon=self._norm_epsilon, + axis=self._bn_axis) + else: + self.bn = Identity() + + if self._activation == 'leaky': + self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) + elif self._activation == 'mish': + self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) + else: + self._activation_fn = tf_utils.get_activation(self._activation) #tf.keras.layers.Activation(self._activation) + + def call(self, x): + x = self._zeropad(x) + x = self.conv(x) + x = self.bn(x) + x = self._activation_fn(x) + return x + + def get_config(self): + # used to store/share parameters to reconstruct the model + layer_config = { + 'filters': self._filters, + 'kernel_size': self._kernel_size, + 'strides': self._strides, + 'padding': self._padding, + 'dilation_rate': self._dilation_rate, + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'kernel_regularizer': self._kernel_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_moment': self._norm_moment, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._activation, + 'leaky_alpha': self._leaky_alpha + } + layer_config.update(super().get_config()) + return layer_config + + def __repr__(self): + return repr(self.get_config()) \ No newline at end of file From 4a304858b4478a942aa979edf9b6f86fe4f7eac5 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Sun, 7 Feb 2021 23:51:48 -0500 Subject: [PATCH 010/132] Use regular Conv2D --- .../modeling/backbones/centernet_backbone.py | 32 ++-- centernet/modeling/layers/nn_blocks.py | 161 +----------------- 2 files changed, 17 insertions(+), 176 deletions(-) diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py index 278c22d2c..dcb1b76f3 100644 --- a/centernet/modeling/backbones/centernet_backbone.py +++ b/centernet/modeling/backbones/centernet_backbone.py @@ -25,16 +25,16 @@ def __init__(self, self._downsample = downsample self._filter_sizes = filter_sizes self._rep_sizes = rep_sizes - + super().__init__(**kwargs) def build(self, input_shape): # Create prelayers if downsampling input if self._downsample: self._pre_layers = tf.keras.Sequential([ - nn_blocks.ConvBN( - filters=128, kernel_size=(7, 7), strides=(2, 2), - padding='same', use_bn=True, activation='reLU'), + tf.keras.layers.Conv2D( + filters=128, kernel_size=(7, 7), strides=(2, 2), + padding='same', use_bias=True, activation='relu'), official_nn_blocks.ResidualBlock( filters=256, use_projection=True, strides=2) @@ -49,23 +49,23 @@ def build(self, input_shape): # Create some intermediate and postlayers to generate the heatmaps (document and make cleaner later) inp_filters = self._filter_sizes[0] - - # cnvs - self.post_hg_convs = [nn_blocks.ConvBN( - filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), - padding='same', use_bn=True, activation='reLU') + + # cnvs + self.post_hg_convs = [tf.keras.layers.Conv2D( + filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), + padding='same', use_bias=True, activation='relu') for _ in range(self._n_stacks) ] #cnvs_ - self.inter_hg_convs1 = [nn_blocks.ConvBN( - filters=256, kernel_size=(1, 1), strides=(1, 1), - padding='same', use_bn=True, activation='linear') + self.inter_hg_convs1 = [tf.keras.layers.Conv2D( + filters=256, kernel_size=(1, 1), strides=(1, 1), + padding='same', use_bias=True, activation='linear') for _ in range(self._n_stacks - 1) ] #inters_ - self.inter_hg_convs2 = [nn_blocks.ConvBN( - filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), - padding='same', use_bn=True, activation='linear') + self.inter_hg_convs2 = [tf.keras.layers.Conv2D( + filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), + padding='same', use_bias=True, activation='linear') for _ in range(self._n_stacks - 1) ] # inters @@ -104,4 +104,4 @@ def call(self, x): x_inter = self.relu(x_inter) x_inter = res(x_inter) - return x_hg \ No newline at end of file + return x_hg diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 4db1d1ed8..815e905e4 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -66,7 +66,7 @@ def build(self, input_shape): filter_sizes=self._filter_sizes[1:], rep_sizes=self._rep_sizes[1:], strides=self._strides) - + # outer hourglass structures end_block = [ official_nn_blocks.ResidualBlock( @@ -99,162 +99,3 @@ def __init__(self, **kwargs): def call(self, input): return input - -# Copied from YOLO (yolo/modeling/layers/nn_blocks) -class ConvBN(tf.keras.layers.Layer): - """ - Modified Convolution layer to match that of the DarkNet Library. The Layer is a standards combination of Conv BatchNorm Activation, - however, the use of bias in the conv is determined by the use of batch normalization. - Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang Chen, Jun-Wei Hsieh - CSPNet: A New Backbone that can Enhance Learning Capability of CNN. arXiv:1911.11929 - Args: - filters: integer for output depth, or the number of features to learn - kernel_size: integer or tuple for the shape of the weight matrix or kernel to learn - strides: integer of tuple how much to move the kernel after each kernel use - padding: string 'valid' or 'same', if same, then pad the image, else do not - dialtion_rate: tuple to indicate how much to modulate kernel weights and - how many pixels in a feature map to skip - kernel_initializer: string to indicate which function to use to initialize weights - bias_initializer: string to indicate which function to use to initialize bias - kernel_regularizer: string to indicate which function to use to regularizer weights - bias_regularizer: string to indicate which function to use to regularizer bias - use_bn: boolean for whether to use batch normalization - use_sync_bn: boolean for whether sync batch normalization statistics - of all batch norm layers to the models global statistics (across all input batches) - norm_moment: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon - activation: string or None for activation function to use in layer, - if None activation is replaced by linear - leaky_alpha: float to use as alpha if activation function is leaky - **kwargs: Keyword Arguments - """ - - def __init__( - self, - filters=1, - kernel_size=(1, 1), - strides=(1, 1), - padding='same', - dilation_rate=(1, 1), - kernel_initializer='glorot_uniform', - bias_initializer='zeros', - bias_regularizer=None, - kernel_regularizer=None, # Specify the weight decay as the default will not work. - use_bn=True, - use_sync_bn=False, - norm_momentum=0.99, - norm_epsilon=0.001, - activation='leaky', - leaky_alpha=0.1, - **kwargs): - - # convolution params - self._filters = filters - self._kernel_size = kernel_size - self._strides = strides - self._padding = padding - self._dilation_rate = dilation_rate - self._kernel_initializer = kernel_initializer - self._bias_initializer = bias_initializer - self._kernel_regularizer = kernel_regularizer - self._bias_regularizer = bias_regularizer - - # batch normalization params - self._use_bn = use_bn - self._use_sync_bn = use_sync_bn - self._norm_moment = norm_momentum - self._norm_epsilon = norm_epsilon - - if tf.keras.backend.image_data_format() == 'channels_last': - # format: (batch_size, height, width, channels) - self._bn_axis = -1 - else: - # format: (batch_size, channels, width, height) - self._bn_axis = 1 - - # activation params - self._activation = activation - self._leaky_alpha = leaky_alpha - - super().__init__(**kwargs) - - def build(self, input_shape): - kernel_size = self._kernel_size if isinstance(self._kernel_size, - int) else self._kernel_size[0] - dilation_rate = self._dilation_rate if isinstance( - self._dilation_rate, int) else self._dilation_rate[0] - if self._padding == 'same' and kernel_size != 1: - padding = dilation_rate * (kernel_size - 1) - left_shift = padding // 2 - self._zeropad = tf.keras.layers.ZeroPadding2D([[left_shift, left_shift], - [left_shift, left_shift]]) - else: - self._zeropad = Identity() - - use_bias = not self._use_bn - - self.conv = tf.keras.layers.Conv2D( - filters=self._filters, - kernel_size=self._kernel_size, - strides=self._strides, - padding='valid', - dilation_rate=self._dilation_rate, - use_bias=use_bias, - kernel_initializer=self._kernel_initializer, - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) - - if self._use_bn: - if self._use_sync_bn: - self.bn = tf.keras.layers.experimental.SyncBatchNormalization( - momentum=self._norm_moment, - epsilon=self._norm_epsilon, - axis=self._bn_axis) - else: - self.bn = tf.keras.layers.BatchNormalization( - momentum=self._norm_moment, - epsilon=self._norm_epsilon, - axis=self._bn_axis) - else: - self.bn = Identity() - - if self._activation == 'leaky': - self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) - elif self._activation == 'mish': - self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) - else: - self._activation_fn = tf_utils.get_activation(self._activation) #tf.keras.layers.Activation(self._activation) - - def call(self, x): - x = self._zeropad(x) - x = self.conv(x) - x = self.bn(x) - x = self._activation_fn(x) - return x - - def get_config(self): - # used to store/share parameters to reconstruct the model - layer_config = { - 'filters': self._filters, - 'kernel_size': self._kernel_size, - 'strides': self._strides, - 'padding': self._padding, - 'dilation_rate': self._dilation_rate, - 'kernel_initializer': self._kernel_initializer, - 'bias_initializer': self._bias_initializer, - 'bias_regularizer': self._bias_regularizer, - 'kernel_regularizer': self._kernel_regularizer, - 'use_bn': self._use_bn, - 'use_sync_bn': self._use_sync_bn, - 'norm_moment': self._norm_moment, - 'norm_epsilon': self._norm_epsilon, - 'activation': self._activation, - 'leaky_alpha': self._leaky_alpha - } - layer_config.update(super().get_config()) - return layer_config - - def __repr__(self): - return repr(self.get_config()) \ No newline at end of file From c73c7407dfcd19d4e276b71f52fe05f01a020efa Mon Sep 17 00:00:00 2001 From: anivegesana Date: Mon, 8 Feb 2021 00:29:36 -0500 Subject: [PATCH 011/132] Copy losses from ODAPI --- centernet/losses/l1_localization_loss.py | 75 +++++++++++++++++++ .../penalty_reduced_logistic_focal_loss.py | 60 +++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 centernet/losses/l1_localization_loss.py create mode 100644 centernet/losses/penalty_reduced_logistic_focal_loss.py diff --git a/centernet/losses/l1_localization_loss.py b/centernet/losses/l1_localization_loss.py new file mode 100644 index 000000000..fe189e1dc --- /dev/null +++ b/centernet/losses/l1_localization_loss.py @@ -0,0 +1,75 @@ +import tensorflow as tf + +try: + # Try to get TF 1.x version if possible + import tensorflow.compat.v1 as tf_v1 + absolute_difference = tf_v1.losses.absolute_difference +except (ImportError, AttributeError): + # The following code was adapted from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/losses/losses_impl.py + from tensorflow.python.keras.utils import losses_utils + + def absolute_difference( + labels, predictions, weights=1.0, + reduction=tf.keras.losses.Reduction.SUM_BY_NONZERO_WEIGHTS): + """Adds an Absolute Difference loss to the training procedure. + `weights` acts as a coefficient for the loss. If a scalar is provided, then + the loss is simply scaled by the given value. If `weights` is a `Tensor` of + shape `[batch_size]`, then the total loss for each sample of the batch is + rescaled by the corresponding element in the `weights` vector. If the shape + of `weights` matches the shape of `predictions`, then the loss of each + measurable element of `predictions` is scaled by the corresponding value of + `weights`. + Args: + labels: The ground truth output tensor, same dimensions as 'predictions'. + predictions: The predicted outputs. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `labels`, and must be broadcastable to `labels` (i.e., all dimensions + must be either `1`, or the same as the corresponding `losses` + dimension). + reduction: Type of reduction to apply to loss. + Returns: + Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same + shape as `labels`; otherwise, it is scalar. + Raises: + ValueError: If the shape of `predictions` doesn't match that of + `labels` or if the shape of `weights` is invalid or if `labels` + or `predictions` is None. + """ + if labels is None: + raise ValueError("labels must not be None.") + if predictions is None: + raise ValueError("predictions must not be None.") + with tf.name_scope(scope, "absolute_difference", + (predictions, labels, weights)) as scope: + predictions = tf.cast(predictions, dtype=tf.float32) + labels = tf.cast(labels, dtype=tf.float32) + predictions.get_shape().assert_is_compatible_with(labels.get_shape()) + losses = tf.abs(tf.subtract(predictions, labels)) + return losses_utils.compute_weighted_loss( + losses, weights, reduction=reduction) + +class L1LocalizationLoss(tf.keras.losses.Loss): + """L1 loss or absolute difference. + When used in a per-pixel manner, each pixel should be given as an anchor. + """ + + def __call__(self, y_true, y_pred, sample_weight=None): + """Compute loss function. + Args: + y_true: A float tensor of shape [batch_size, num_anchors] + representing the regression targets + y_pred: A float tensor of shape [batch_size, num_anchors] + representing the (encoded) predicted locations of objects. + sample_weight: a float tensor of shape [batch_size, num_anchors] + Returns: + loss: a float tensor of shape [batch_size, num_anchors] tensor + representing the value of the loss function. + """ + return absolute_difference( + y_true, + y_pred, + weights=sample_weight, + reduction=tf.keras.losses.Reduction.NONE + ) + + call = __call__ diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss.py b/centernet/losses/penalty_reduced_logistic_focal_loss.py new file mode 100644 index 000000000..387540c83 --- /dev/null +++ b/centernet/losses/penalty_reduced_logistic_focal_loss.py @@ -0,0 +1,60 @@ +# may be a dupe of retinanet_losses. need to look later +import tensorflow as tf + +class PenaltyReducedLogisticFocalLoss(tf.keras.losses.Loss): + """Penalty-reduced pixelwise logistic regression with focal loss. + The loss is defined in Equation (1) of the Objects as Points[1] paper. + Although the loss is defined per-pixel in the output space, this class + assumes that each pixel is an anchor to be compatible with the base class. + [1]: https://arxiv.org/abs/1904.07850 + """ + + def __init__(self, alpha=2.0, beta=4.0, sigmoid_clip_value=1e-4): + """Constructor. + Args: + alpha: Focussing parameter of the focal loss. Increasing this will + decrease the loss contribution of the well classified examples. + beta: The local penalty reduction factor. Increasing this will decrease + the contribution of loss due to negative pixels near the keypoint. + sigmoid_clip_value: The sigmoid operation used internally will be clipped + between [sigmoid_clip_value, 1 - sigmoid_clip_value) + """ + self._alpha = alpha + self._beta = beta + self._sigmoid_clip_value = sigmoid_clip_value + super(PenaltyReducedLogisticFocalLoss, self).__init__() + + def call(self, y_true, y_pred): + """Compute loss function. + In all input tensors, `num_anchors` is the total number of pixels in the + the output space. + Args: + prediction_tensor: A float tensor of shape [batch_size, num_anchors, + num_classes] representing the predicted unscaled logits for each class. + The function will compute sigmoid on this tensor internally. + target_tensor: A float tensor of shape [batch_size, num_anchors, + num_classes] representing a tensor with the 'splatted' keypoints, + possibly using a gaussian kernel. This function assumes that + the target is bounded between [0, 1]. + weights: a float tensor of shape, either [batch_size, num_anchors, + num_classes] or [batch_size, num_anchors, 1]. If the shape is + [batch_size, num_anchors, 1], all the classses are equally weighted. + Returns: + loss: a float tensor of shape [batch_size, num_anchors, num_classes] + representing the value of the loss function. + """ + + is_present_tensor = tf.math.equal(target_tensor, 1.0) + prediction_tensor = tf.clip_by_value(tf.sigmoid(y_pred), + self._sigmoid_clip_value, + 1 - self._sigmoid_clip_value) + target_tensor = y_true + + positive_loss = (tf.math.pow((1 - prediction_tensor), self._alpha)* + tf.math.log(prediction_tensor)) + negative_loss = (tf.math.pow((1 - target_tensor), self._beta)* + tf.math.pow(prediction_tensor, self._alpha)* + tf.math.log(1 - prediction_tensor)) + + loss = -tf.where(is_present_tensor, positive_loss, negative_loss) + return loss From a00fd78d36fc30ae35916c2bce1620ab1d4bf734 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 8 Feb 2021 12:13:50 -0500 Subject: [PATCH 012/132] cleaned and updated tests --- .../modeling/backbones/centernet_backbone.py | 3 ++- .../backbones/centernet_backbone_test.py | 7 +++--- centernet/modeling/layers/nn_blocks.py | 25 ++++--------------- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py index dcb1b76f3..130535c9b 100644 --- a/centernet/modeling/backbones/centernet_backbone.py +++ b/centernet/modeling/backbones/centernet_backbone.py @@ -95,6 +95,7 @@ def call(self, x): all_heatmaps.append(x_hg) + # between hourglasses, we insert intermediate layers if i < self._n_stacks - 1: inter_hg_conv1 = self.inter_hg_convs1[i] inter_hg_conv2 = self.inter_hg_convs2[i] @@ -104,4 +105,4 @@ def call(self, x): x_inter = self.relu(x_inter) x_inter = res(x_inter) - return x_hg + return x_hg, all_heatmaps diff --git a/centernet/modeling/backbones/centernet_backbone_test.py b/centernet/modeling/backbones/centernet_backbone_test.py index 2a7d40a3f..a8144e861 100644 --- a/centernet/modeling/backbones/centernet_backbone_test.py +++ b/centernet/modeling/backbones/centernet_backbone_test.py @@ -17,7 +17,7 @@ tf.print('Made an hourglass!') out = hg(x_hg) tf.print('Hourglass module output shape:{} Expected shape:{}'.format( - hg.output_shape, hg_test_input_shape)) + tf.shape(out), hg_test_input_shape)) tf.print("Testing backbone") backbone = CenterNetBackbone( @@ -37,6 +37,7 @@ tf.print(tf.shape(out)) tf.print('Made backbone!') + tf.print('Backbone output shape: {} Expected shape: {}'.format( - backbone.output_shape, test_input_shape - )) + tf.shape(out), (bb_test_input_shape[0], bb_test_input_shape[1]//8, + bb_test_input_shape[2]//8, filter_sizes[0]))) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 815e905e4..4fdc91d68 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -56,16 +56,10 @@ def build(self, input_shape): self.pool = tf.keras.layers.MaxPool2D(pool_size=2) # recursively define inner hourglasses - if (self._order == 1): - self.inner_hg = HourglassBlockBase( - filters=self._filter_sizes[-1], - reps=self._rep_sizes[-1], - strides=self._strides) - else: - self.inner_hg = type(self)( - filter_sizes=self._filter_sizes[1:], - rep_sizes=self._rep_sizes[1:], - strides=self._strides) + self.inner_hg = type(self)( + filter_sizes=self._filter_sizes[1:], + rep_sizes=self._rep_sizes[1:], + strides=self._strides) # outer hourglass structures end_block = [ @@ -89,13 +83,4 @@ def call(self, x): x_pooled = self.pool(x_pre_pooled) inner_output = self.inner_hg(x_pooled) hg_output = self.end_block(inner_output) - return self.upsample_layer(hg_output) + x_side - -# Copied from YOLO (yolo/modeling/layers/nn_blocks) -class Identity(tf.keras.layers.Layer): - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def call(self, input): - return input + return self.upsample_layer(hg_output) + x_side \ No newline at end of file From b33d3860cd1ec93268c3c7679acea0b34a7638b2 Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Mon, 8 Feb 2021 11:39:25 -0800 Subject: [PATCH 013/132] added gaussian utils --- centernet/utils/groundtruth.py | 84 ++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/centernet/utils/groundtruth.py b/centernet/utils/groundtruth.py index 1f11ebf32..55d1855a9 100644 --- a/centernet/utils/groundtruth.py +++ b/centernet/utils/groundtruth.py @@ -1,5 +1,63 @@ import tensorflow as tf +def _smallest_positive_root(a, b, c) -> tf.Tensor: + """ + Returns the smallest positive root of a quadratic equation. + This implements the fixed version in https://github.com/princeton-vl/CornerNet. + """ + + discriminant = tf.sqrt(b ** 2 - 4 * a * c) + + root1 = (-b - discriminant) / (2 * a) + root2 = (-b + discriminant) / (2 * a) + + return tf.where(tf.less(root1, 0), root2, root1) + +def gaussian_radius(det_size, min_overlap=0.7) -> int: + """ + Given a bounding box size, returns a lower bound on how far apart the + corners of another bounding box can lie while still maintaining the given + minimum overlap, or IoU. Modified from implementation found in + https://github.com/tensorflow/models/blob/master/research/object_detection/core/target_assigner.py. + + Params: + det_size (tuple): tuple of integers representing height and width + min_overlap (tf.float32): minimum IoU desired + Returns: + int representing desired gaussian radius + """ + height, width = det_size + + # Case where detected box is offset from ground truth and no box completely + # contains the other. + + a1 = 1 + b1 = (height + width) + c1 = width * height * (1 - min_overlap) / (1 + min_overlap) + distance_detection_offset = _smallest_positive_root(a1, b1, c1) + r1 = (b1 + distance_detection_offset) / 2 + + # Case where detection is smaller than ground truth and completely contained + # in it. + + a2 = 4 + b2 = 2 * (height + width) + c2 = (1 - min_overlap) * width * height + distance_detection_in_gt = _smallest_positive_root(a2, b2, c2) + r2 = (b2 + distance_detection_in_gt) / 2 + + # Case where ground truth is smaller than detection and completely contained + # in it. + + a3 = 4 * min_overlap + b3 = -2 * min_overlap * (height + width) + c3 = (min_overlap - 1) * width * height + distance_gt_in_detection = _smallest_positive_root(a3, b3, c3) + r3 = (b3 + distance_gt_in_detection) / 2 + # TODO discuss whether to return scalar or tensor + # return tf.reduce_min([r1, r2, r3], axis=0) + + return min(r1, r2, r3) def gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: """ @@ -16,3 +74,29 @@ def gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: y = tf.reshape(tf.range(width, dtype=type) - radius, (1, width)) exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) return tf.math.exp(exponent) + +def draw_gaussian(heatmap, center, radius, k=1): + """ + Draws a gaussian heatmap around a center point given a radius. + Params: + heatmap (tf.Tensor): heatmap placeholder to fill + center (int): integer for center of gaussian + radius (int): integer for radius of gaussian + k (int): scaling factor for gaussian + """ + + diameter = 2 * radius + 1 + gaussian = gaussian_penalty(radius) + + x, y = center + + height, width = heatmap.shape[0:2] + + left, right = min(x, radius), min(width - x, radius + 1) + top, bottom = min(y, radius), min(height - y, radius + 1) + + masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] + # TODO: make sure this replicates original functionality + # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) + masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * k) From 48d652ce684df8f1cf7b98d6eeced1f8e1913f54 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 8 Feb 2021 15:53:07 -0500 Subject: [PATCH 014/132] built basic config loader, adapted tests --- .../modeling/backbones/backbone_builder.py | 34 +++++++++++++++++++ .../modeling/backbones/centernet_backbone.py | 14 ++++---- .../backbones/centernet_backbone_test.py | 22 +++++------- centernet/modeling/backbones/config.py | 8 +++-- 4 files changed, 56 insertions(+), 22 deletions(-) create mode 100644 centernet/modeling/backbones/backbone_builder.py diff --git a/centernet/modeling/backbones/backbone_builder.py b/centernet/modeling/backbones/backbone_builder.py new file mode 100644 index 000000000..087b0eb6f --- /dev/null +++ b/centernet/modeling/backbones/backbone_builder.py @@ -0,0 +1,34 @@ +from centernet.modeling.backbones.centernet_backbone import CenterNetBackbone +import centernet.modeling.backbones.config + +def buildCenterNetBackbone(config): + downsample = False + n_stacks = 0 + all_filter_sizes = [] + all_rep_sizes = [] + all_strides = [] + + for layer in config: + name, filterSizes, repSizes, strides = layer + + if name == "Downsample": + downsample = True + elif name == "HourglassBlock": + n_stacks += 1 + if len(filterSizes) != len(repSizes): + print("Number of filter sizes and rep sizes must be equal") + break + all_filter_sizes.append(filterSizes) + all_rep_sizes.append(repSizes) + all_strides.append(strides) + else: + print("Invalid layer name provided") + break + + backbone = CenterNetBackbone(filter_sizes=all_filter_sizes, + rep_sizes=all_rep_sizes, + n_stacks=n_stacks, + strides=all_strides, + downsample=downsample) + + return backbone \ No newline at end of file diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py index 130535c9b..b6ae794ce 100644 --- a/centernet/modeling/backbones/centernet_backbone.py +++ b/centernet/modeling/backbones/centernet_backbone.py @@ -11,7 +11,8 @@ class CenterNetBackbone(tf.keras.Model): def __init__(self, filter_sizes, rep_sizes, - n_stacks=2, + n_stacks, + strides, downsample=True, **kwargs): """ @@ -21,10 +22,11 @@ def __init__(self, n_stacks: integer, number of hourglass modules in backbone pre_layers: tf.keras layer to process input before stacked hourglasses """ - self._n_stacks = n_stacks - self._downsample = downsample self._filter_sizes = filter_sizes self._rep_sizes = rep_sizes + self._n_stacks = n_stacks + self._strides = strides + self._downsample = downsample super().__init__(**kwargs) @@ -43,12 +45,12 @@ def build(self, input_shape): # Create hourglass stacks self.hgs = [ nn_blocks.HourglassBlock( - filter_sizes=self._filter_sizes, rep_sizes=self._rep_sizes) - for _ in range(self._n_stacks) + filter_sizes=self._filter_sizes[n], rep_sizes=self._rep_sizes[n]) + for n in range(self._n_stacks) ] # Create some intermediate and postlayers to generate the heatmaps (document and make cleaner later) - inp_filters = self._filter_sizes[0] + inp_filters = self._filter_sizes[0][0] # cnvs self.post_hg_convs = [tf.keras.layers.Conv2D( diff --git a/centernet/modeling/backbones/centernet_backbone_test.py b/centernet/modeling/backbones/centernet_backbone_test.py index a8144e861..193d3f06c 100644 --- a/centernet/modeling/backbones/centernet_backbone_test.py +++ b/centernet/modeling/backbones/centernet_backbone_test.py @@ -2,9 +2,10 @@ from centernet.modeling.layers.nn_blocks import HourglassBlock from centernet.modeling.backbones.centernet_backbone import CenterNetBackbone +from centernet.modeling.backbones.backbone_builder import buildCenterNetBackbone +from centernet.modeling.backbones.config import CENTERNET_HG104_CFG + if __name__ == '__main__': - filter_sizes = [256, 256, 384, 384, 384, 512] - rep_sizes = [2, 2, 2, 2, 2, 4] hg_test_input_shape = (1, 512, 512, 256) bb_test_input_shape = (1, 512, 512, 3) x_hg = tf.ones(shape=hg_test_input_shape) @@ -20,24 +21,19 @@ tf.shape(out), hg_test_input_shape)) tf.print("Testing backbone") - backbone = CenterNetBackbone( - n_stacks=2, - downsample=True, - filter_sizes=filter_sizes, - rep_sizes=rep_sizes) + + backbone = buildCenterNetBackbone(CENTERNET_HG104_CFG) backbone.build(input_shape=bb_test_input_shape) # Backbone summary shows output shape to be multiple for hg modules # Maybe this is because the hg call method has a conditional? - # This is also causing the final print statement to error. backbone.summary() - out = backbone(x_bb) - tf.print(tf.shape(out)) - + out_final, all_outs = backbone(x_bb) tf.print('Made backbone!') + expected_out_filters = CENTERNET_HG104_CFG[-1][1][0] # last hourglass filters tf.print('Backbone output shape: {} Expected shape: {}'.format( - tf.shape(out), (bb_test_input_shape[0], bb_test_input_shape[1]//8, - bb_test_input_shape[2]//8, filter_sizes[0]))) + tf.shape(out_final), (bb_test_input_shape[0], bb_test_input_shape[1]//8, + bb_test_input_shape[2]//8, expected_out_filters))) diff --git a/centernet/modeling/backbones/config.py b/centernet/modeling/backbones/config.py index f57a2415a..0a202be90 100644 --- a/centernet/modeling/backbones/config.py +++ b/centernet/modeling/backbones/config.py @@ -1,4 +1,6 @@ -# (name, filterSizes, repSizes) -backbone = [ - ("Hourglass", ) +# (name, filterSizes, repSizes, strides) +CENTERNET_HG104_CFG = [ + ("Downsample", None, None, None), + ("HourglassBlock", [256, 256, 384, 384, 384, 512], [2, 2, 2, 2, 2, 4], 1), + ("HourglassBlock", [256, 256, 384, 384, 384, 512], [2, 2, 2, 2, 2, 4], 1), ] \ No newline at end of file From 1cbfc42600fe9cee90fc15944dfb9a42d02adfc4 Mon Sep 17 00:00:00 2001 From: Jack LeCroy <3073035+jacklecroy@users.noreply.github.com> Date: Wed, 10 Feb 2021 13:25:55 -0500 Subject: [PATCH 015/132] add parser centernet --- centernet/dataloaders/centernet_input.py | 98 ++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 centernet/dataloaders/centernet_input.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py new file mode 100644 index 000000000..cf4f07dec --- /dev/null +++ b/centernet/dataloaders/centernet_input.py @@ -0,0 +1,98 @@ +import tensorflow as tf +from official.vision.beta.dataloaders import parser + + +class CenterNetParser(parser.Parser): + def __init__( + self, + num_classes: int, + gaussian_iou: float + ): + self.num_classes = num_classes + self.gaussian_io = gaussian_iou + + def _parse_train_data(self, decoded_tensors): + """Generates images and labels that are usable for model training. + + Args: + decoded_tensors: a dict of Tensors produced by the decoder. + + Returns: + images: the image tensor. + labels: a dict of Tensors that contains labels. + """ + # TODO: input size, output size + image = decoded_tensors["image"] + + width_ratio = output_size[1] / input_size[1] + height_ratio = output_size[0] / input_size[0] + + for ind, detection in enumerate(decoded_tensors["groundtruth_boxes"]): + category = int(detection[-1]) - 1 + #category = 0 + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. + + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + xtl = int(fxtl) + ytl = int(fytl) + xbr = int(fxbr) + ybr = int(fybr) + xct = int(fxct) + yct = int(fyct) + + if gaussian_bump: + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + width = math.ceil(width * width_ratio) + height = math.ceil(height * height_ratio) + + if gaussian_rad == -1: + radius = gaussian_radius((height, width), self.gaussian_iou) + radius = max(0, int(radius)) + else: + radius = gaussian_rad + # TODO: implement gaussian + # draw_gaussian(tl_heatmaps[b_ind, category], [xtl, ytl], radius) + # draw_gaussian(br_heatmaps[b_ind, category], [xbr, ybr], radius) + # draw_gaussian(ct_heatmaps[b_ind, category], [xct, yct], radius, delte=5) + + else: + tl_heatmaps[category, ytl, xtl] = 1 + br_heatmaps[category, ybr, xbr] = 1 + ct_heatmaps[category, yct, xct] = 1 + + tag_ind = tag_lens + tl_regrs[tag_ind, :] = [fxtl - xtl, fytl - ytl] + br_regrs[tag_ind, :] = [fxbr - xbr, fybr - ybr] + ct_regrs[tag_ind, :] = [fxct - xct, fyct - yct] + tl_tags[tag_ind] = ytl * output_size[1] + xtl + br_tags[tag_ind] = ybr * output_size[1] + xbr + ct_tags[tag_ind] = yct * output_size[1] + xct + tag_lens += 1 + + labels = { + 'tl_tags': tl_tags, + 'br_tags': br_tags, + 'ct_tags': ct_tags, + 'tl_heatmaps': tl_heatmaps, + 'br_heatmaps': br_heatmaps, + 'ct_heatmaps': ct_heatmaps, + 'tag_masks': tag_masks, + 'tl_regrs': tl_regrs, + 'br_regrs', br_regrs, + 'ct_regrs': ct_regrs, + } + return image, labels + + def _parse_eval_data(self, data): + pass From 19f6b4c338e32a38599f8d058fbfecc3648a5a57 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 10 Feb 2021 16:38:41 -0500 Subject: [PATCH 016/132] Add test cases --- centernet/losses/__init__.py | 3 + centernet/losses/l1_localization_loss_test.py | 20 +++++ ...enalty_reduced_logistic_focal_loss_test.py | 89 +++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 centernet/losses/l1_localization_loss_test.py create mode 100644 centernet/losses/penalty_reduced_logistic_focal_loss_test.py diff --git a/centernet/losses/__init__.py b/centernet/losses/__init__.py index e69de29bb..9bf1c1685 100644 --- a/centernet/losses/__init__.py +++ b/centernet/losses/__init__.py @@ -0,0 +1,3 @@ +from centernet.losses.l1_localization_loss import L1LocalizationLoss +from centernet.losses.penalty_reduced_logistic_focal_loss import \ + PenaltyReducedLogisticFocalLoss diff --git a/centernet/losses/l1_localization_loss_test.py b/centernet/losses/l1_localization_loss_test.py new file mode 100644 index 000000000..eeed410ce --- /dev/null +++ b/centernet/losses/l1_localization_loss_test.py @@ -0,0 +1,20 @@ +import tensorflow as tf +import numpy as np + +from centernet import losses + +class L1LocalizationLossTest(tf.test.TestCase): + + def test_returns_correct_loss(self): + def graph_fn(): + loss = losses.L1LocalizationLoss() + pred = [[0.1, 0.2], [0.7, 0.5]] + target = [[0.9, 1.0], [0.1, 0.4]] + + weights = [[1.0, 0.0], [1.0, 1.0]] + return loss(pred, target, weights) + computed_value = graph_fn() + self.assertAllClose(computed_value, [[0.8, 0.0], [0.6, 0.1]], rtol=1e-6) + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py new file mode 100644 index 000000000..9c12dd682 --- /dev/null +++ b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py @@ -0,0 +1,89 @@ +import tensorflow as tf +import numpy as np + +from centernet import losses + +class PenaltyReducedLogisticFocalLossTest(tf.test.TestCase): + """Testing loss function from Equation (1) in [1]. + [1]: https://arxiv.org/abs/1904.07850 + """ + + def setUp(self): + super(PenaltyReducedLogisticFocalLossTest, self).setUp() + self._prediction = np.array([ + # First batch + [[1 / 2, 1 / 4, 3 / 4], + [3 / 4, 1 / 3, 1 / 3]], + # Second Batch + [[0.0, 1.0, 1 / 2], + [3 / 4, 2 / 3, 1 / 3]]], np.float32) + self._prediction = np.log(self._prediction/(1 - self._prediction)) + + self._target = np.array([ + # First batch + [[1.0, 0.91, 1.0], + [0.36, 0.84, 1.0]], + # Second Batch + [[0.01, 1.0, 0.75], + [0.96, 1.0, 1.0]]], np.float32) + + def test_returns_correct_loss(self): + def graph_fn(prediction, target): + weights = tf.constant([ + [[1.0], [1.0]], + [[1.0], [1.0]], + ]) + loss = losses.PenaltyReducedLogisticFocalLoss(alpha=2.0, beta=0.5) + computed_value = loss._compute_loss(prediction, target, + weights) + return computed_value + computed_value = self.execute(graph_fn, [self._prediction, self._target]) + expected_value = np.array([ + # First batch + [[1 / 4 * LOG_2, + 0.3 * 0.0625 * (2 * LOG_2 - LOG_3), + 1 / 16 * (2 * LOG_2 - LOG_3)], + [0.8 * 9 / 16 * 2 * LOG_2, + 0.4 * 1 / 9 * (LOG_3 - LOG_2), + 4 / 9 * LOG_3]], + # Second Batch + [[0.0, + 0.0, + 1 / 2 * 1 / 4 * LOG_2], + [0.2 * 9 / 16 * 2 * LOG_2, + 1 / 9 * (LOG_3 - LOG_2), + 4 / 9 * LOG_3]]]) + self.assertAllClose(computed_value, expected_value, rtol=1e-3, atol=1e-3) + + def test_returns_correct_loss_weighted(self): + def graph_fn(prediction, target): + weights = tf.constant([ + [[1.0, 0.0, 1.0], [0.0, 0.0, 1.0]], + [[1.0, 1.0, 1.0], [0.0, 0.0, 0.0]], + ]) + + loss = losses.PenaltyReducedLogisticFocalLoss(alpha=2.0, beta=0.5) + + computed_value = loss(prediction, target, weights) + return computed_value + computed_value = graph_fn(self._prediction, self._target) + expected_value = np.array([ + # First batch + [[1 / 4 * LOG_2, + 0.0, + 1 / 16 * (2 * LOG_2 - LOG_3)], + [0.0, + 0.0, + 4 / 9 * LOG_3]], + # Second Batch + [[0.0, + 0.0, + 1 / 2 * 1 / 4 * LOG_2], + [0.0, + 0.0, + 0.0]]]) + + self.assertAllClose(computed_value, expected_value, rtol=1e-3, atol=1e-3) + +if __name__ == '__main__': + tf.test.main() From 49d5e806b68181664ef40a9aaa33349f64107339 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 10 Feb 2021 20:35:50 -0500 Subject: [PATCH 017/132] Bug fix --- centernet/losses/penalty_reduced_logistic_focal_loss.py | 3 ++- centernet/losses/penalty_reduced_logistic_focal_loss_test.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss.py b/centernet/losses/penalty_reduced_logistic_focal_loss.py index 387540c83..2f5cb5bad 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss.py @@ -44,11 +44,12 @@ def call(self, y_true, y_pred): representing the value of the loss function. """ + target_tensor = y_true + is_present_tensor = tf.math.equal(target_tensor, 1.0) prediction_tensor = tf.clip_by_value(tf.sigmoid(y_pred), self._sigmoid_clip_value, 1 - self._sigmoid_clip_value) - target_tensor = y_true positive_loss = (tf.math.pow((1 - prediction_tensor), self._alpha)* tf.math.log(prediction_tensor)) diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py index 9c12dd682..7c16a904f 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py @@ -34,10 +34,9 @@ def graph_fn(prediction, target): [[1.0], [1.0]], ]) loss = losses.PenaltyReducedLogisticFocalLoss(alpha=2.0, beta=0.5) - computed_value = loss._compute_loss(prediction, target, - weights) + computed_value = loss(prediction, target, weights) return computed_value - computed_value = self.execute(graph_fn, [self._prediction, self._target]) + computed_value = graph_fn(self._prediction, self._target) expected_value = np.array([ # First batch [[1 / 4 * LOG_2, From fd1102a339c497c3d6cd7bde3e45c06d0eb92bae Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 10 Feb 2021 22:02:12 -0500 Subject: [PATCH 018/132] Restructured backbone Co-Authored-By: David Li --- centernet/configs/backbones.py | 31 ++++ centernet/modeling/backbones/hourglass.py | 103 +++++++++++++ centernet/modeling/layers/nn_blocks.py | 24 ++-- utils/register.py | 167 ++++++++++++++++++++++ 4 files changed, 313 insertions(+), 12 deletions(-) create mode 100644 centernet/configs/backbones.py create mode 100644 centernet/modeling/backbones/hourglass.py create mode 100644 utils/register.py diff --git a/centernet/configs/backbones.py b/centernet/configs/backbones.py new file mode 100644 index 000000000..a1f266ef4 --- /dev/null +++ b/centernet/configs/backbones.py @@ -0,0 +1,31 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Backbones configurations.""" +from typing import Tuple + +# Import libraries +import dataclasses + +from official.modeling import hyperparams + +@dataclasses.dataclass +class Hourglass(hyperparams.Config): + """Hourglass config.""" + input_channel_dims : int = 128 + channel_dims_per_stage: Tuple[int] = (256, 256, 384, 384, 384, 512) + blocks_per_stage: Tuple[int] = (2, 2, 2, 2, 2, 4) + num_hourglasses: int = 2 + initial_downsample: bool = True diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py new file mode 100644 index 000000000..0473efe9f --- /dev/null +++ b/centernet/modeling/backbones/hourglass.py @@ -0,0 +1,103 @@ +import tensorflow as tf + +from centernet.configs import backbones +from centernet.modeling.layers import nn_blocks + +# from official.vision.beta.modeling.backbones import factory +from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks +from utils import register + +from typing import List + +class Hourglass(tf.keras.Model): + """ + CenterNet Hourglass backbone + """ + + def __init__(self, + input_channel_dims : int, + channel_dims_per_stage: List[int], + blocks_per_stage: List[int], + num_hourglasses: int, + initial_downsample: bool = True, + input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 3]), + **kwargs): + """ + Args: + channel_dims_per_stage: list of filter sizes for Residual blocks + blocks_per_stage: list of residual block repetitions per down/upsample + num_hourglasses: integer, number of hourglass modules in backbone + pre_layers: tf.keras layer to process input before stacked hourglasses + """ + + input = tf.keras.layers.Input(shape=input_specs.shape[1:]) + x_inter = input + + # Create prelayers if downsampling input + if initial_downsample: + x_inter = tf.keras.Sequential([ + tf.keras.layers.Conv2D( + filters=input_channel_dims, kernel_size=(7, 7), strides=(2, 2), + padding='same', use_bias=True, activation='relu'), + official_nn_blocks.ResidualBlock( + filters=input_channel_dims*2, use_projection=True, + strides=2) + ], name="Prelayers")(x_inter) + + all_heatmaps = [] + + for i in range(num_hourglasses): + # Create hourglass stacks + x_hg = nn_blocks.HourglassBlock(channel_dims_per_stage=channel_dims_per_stage, blocks_per_stage=blocks_per_stage)(x_inter) + + # Create some intermediate and postlayers to generate the heatmaps (document and make cleaner later) + inp_filters = channel_dims_per_stage[0] + + # cnvs + x_hg = tf.keras.layers.Conv2D( + filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), + padding='same', use_bias=True, activation='relu')(x_hg) + + all_heatmaps.append(x_hg) + + # between hourglasses, we insert intermediate layers + if i < num_hourglasses - 1: + #cnvs_ + inter_hg_conv1 = tf.keras.layers.Conv2D( + filters=input_channel_dims*2, kernel_size=(1, 1), strides=(1, 1), + padding='same', use_bias=True, activation='linear')(x_inter) + + #inters_ + inter_hg_conv2 = tf.keras.layers.Conv2D( + filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), + padding='same', use_bias=True, activation='linear')(x_hg) + + x_inter = inter_hg_conv1 + inter_hg_conv2 + x_inter = tf.keras.layers.ReLU()(x_inter) + + # inters + x_inter = official_nn_blocks.ResidualBlock( + filters=inp_filters, use_projection=True, + strides=2)(x_inter) + + super().__init__(inputs=input, outputs=[x_hg, all_heatmaps], **kwargs) + +# @factory.register_backbone_builder('hourglass') +@register.backbone('hourglass', backbones.Hourglass) +def build_hourglass( + input_specs: tf.keras.layers.InputSpec, + model_config, + l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: + """Builds ResNet backbone from a config.""" + backbone_type = model_config.backbone.type + backbone_cfg = model_config.backbone.get() + assert backbone_type == 'hourglass', (f'Inconsistent backbone type ' + f'{backbone_type}') + + return Hourglass( + input_channel_dims=backbone_cfg.input_channel_dims, + channel_dims_per_stage=backbone_cfg.channel_dims_per_stage, + blocks_per_stage=backbone_cfg.blocks_per_stage, + num_hourglasses=backbone_cfg.num_hourglasses, + initial_downsample=backbone_cfg.initial_downsample, + input_specs=input_specs) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 4fdc91d68..67f385930 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -7,23 +7,23 @@ class HourglassBlock(tf.keras.layers.Layer): Hourglass module """ - def __init__(self, filter_sizes, rep_sizes, strides=1, **kwargs): + def __init__(self, channel_dims_per_stage, blocks_per_stage, strides=1, **kwargs): """ Args: - filter_sizes: list of filter sizes for Residual blocks - rep_sizes: list of residual block repetitions per down/upsample + channel_dims_per_stage: list of filter sizes for Residual blocks + blocks_per_stage: list of residual block repetitions per down/upsample strides: integer, stride parameter to the Residual block """ - self._order = len(filter_sizes) - 1 - self._filter_sizes = filter_sizes - self._rep_sizes = rep_sizes + self._order = len(channel_dims_per_stage) - 1 + self._channel_dims_per_stage = channel_dims_per_stage + self._blocks_per_stage = blocks_per_stage self._strides = strides - assert len(filter_sizes) == len(rep_sizes), 'filter size and ' \ + assert len(channel_dims_per_stage) == len(blocks_per_stage), 'filter size and ' \ 'residual block repetition lists must have the same length' - self._filters = filter_sizes[0] - self._reps = rep_sizes[0] + self._filters = channel_dims_per_stage[0] + self._reps = blocks_per_stage[0] super().__init__() @@ -57,8 +57,8 @@ def build(self, input_shape): # recursively define inner hourglasses self.inner_hg = type(self)( - filter_sizes=self._filter_sizes[1:], - rep_sizes=self._rep_sizes[1:], + channel_dims_per_stage=self._channel_dims_per_stage[1:], + blocks_per_stage=self._blocks_per_stage[1:], strides=self._strides) # outer hourglass structures @@ -83,4 +83,4 @@ def call(self, x): x_pooled = self.pool(x_pre_pooled) inner_output = self.inner_hg(x_pooled) hg_output = self.end_block(inner_output) - return self.upsample_layer(hg_output) + x_side \ No newline at end of file + return self.upsample_layer(hg_output) + x_side diff --git a/utils/register.py b/utils/register.py new file mode 100644 index 000000000..b253c94b8 --- /dev/null +++ b/utils/register.py @@ -0,0 +1,167 @@ +import dataclasses +import functools +import inspect +import string +import warnings +from typing import ClassVar, Union + +from official.vision.beta.configs import backbones, backbones_3d +from official.vision.beta.modeling.backbones import factory as backbones_factory +from official.vision.beta.modeling import factory_3d as models_3d_factory +from official.core import task_factory +from official.core import exp_factory + +from official.core import registry +from official.modeling import hyperparams + + +def _deduce_type(fn, config_param=1): + if inspect.isclass(fn): + fn = fn_or_cls.__init__ + if isinstance(config_param, int): + sig = list(inspect.signature(fn).parameters.items())[ + config_param][1] + return sig.annotation, sig.default + elif isinstance(config_param, str): + sig = inspect.signature(fn).parameters[config_param] + return sig.annotation, sig.default + else: + return None, inspect.Signature.empty + +def _snake_case(class_name: str): + words = [] + + for c in class_name: + if c in string.ascii_uppercase: + words.append('_' + c.lower()) + else: + words.append(c) + + return ''.join(words).strip('_') + +def _inject_dataclass(dcls, name, cls, value): + del dcls.__init__, dcls.__repr__, dcls.__eq__ + dcls.__annotations__[name] = cls + setattr(dcls, name, value) + dataclasses.dataclass(dcls) + +def _make_registry_shim(dataclass, register, config_param=1): + def shim(name: str = None, config_class: type = None, + default = inspect.Signature.empty): + nonlocal dataclass, register + + def decorator(builder): + nonlocal dataclass, register, name, config_class, default + + if config_class is None: + config_class, deduced_default = _deduce_type(builder, config_param) + else: + deduced_default = inspect.Signature.empty + + if name is None: + assert config_class is not None, 'Either the name or the class must ' \ + 'be specified' + name = _snake_case(config_class.__name__) + + if default is inspect.Signature.empty: + if deduced_default is not inspect.Signature.empty: + default = deduced_default + elif config_class is not None: + default = config_class() + + if config_class is not None: + _inject_dataclass(dataclass, name, config_class, default) + else: + warnings.warn(f'Config class for {name} was not specified', + stacklevel=2) + return register(name)(builder) + + if callable(name): + return decorator(name) + return decorator + + return shim + +backbone = _make_registry_shim(backbones.Backbone, + backbones_factory.register_backbone_builder, None) +backbone_3d = _make_registry_shim(backbones_3d.Backbone3D, + backbones_factory.register_backbone_builder, None) + +task = task_factory.register_task_cls +experiment = exp_factory.register_config_factory + + +class Registry(dict): + + def register(self, key): + return registry.register(self, key) + + def _lookup(self, key): + return registry.lookup(self, key) + + +class RegistryOneOfConfigMetaclass(type): + + def __new__(cls, name, bases, dct): + dct['_REGISTRY'] = Registry() + if '__annotations__' not in dct: + dct['__annotations__'] = {'_REGISTRY': ClassVar[dict]} + else: + dct['__annotations__']['_REGISTRY'] = ClassVar[dict] + + _CONFIG_PARAM = dct.get('_CONFIG_PARAM', 1) + obj = super().__new__(cls, name, bases, dct) + obj.register = _make_registry_shim(obj, obj._old_register, + config_param=_CONFIG_PARAM) + return obj + + +@dataclasses.dataclass +class RegistryOneOfConfig( + hyperparams.OneOfConfig, metaclass=RegistryOneOfConfigMetaclass): + _CONFIG_PARAM: ClassVar[Union[str, int]] = None + + @classmethod + def _old_register(cls, key): + return registry.register(cls._REGISTRY, key) + + @classmethod + def _lookup(cls, key): + return registry.lookup(cls._REGISTRY, key) + +if __name__ == '__main__': + from official.vision.beta.configs import backbones + import tensorflow as tf + + @dataclasses.dataclass + class Backbone(backbones.Backbone, RegistryOneOfConfig): + # _CONFIG_PARAM = 1 + pass + + @dataclasses.dataclass + class DarkNet(hyperparams.Config): + """DarkNet config.""" + model_id: str = 'darknet53' + + mobilenet = Backbone({'type': 'mobilenet'}) + print(mobilenet) + print(mobilenet.mobilenet) + + print(Backbone._REGISTRY) + + @Backbone.register('darknet') + # @factory.register_backbone_builder("darknet") + def build_darknet( + input_specs: tf.keras.layers.InputSpec, + model_config: DarkNet, + l2_regularizer: tf.keras.regularizers.Regularizer = None + ) -> tf.keras.Model: + pass + + print(build_darknet) + + darknet = Backbone({'type': 'darknet'}) + print(darknet) + print(mobilenet) + print(Backbone._REGISTRY) + print(Backbone._lookup('darknet')) From b2314eeadeb180e823e0efe5af7dd5dccca594f1 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 11 Feb 2021 16:15:49 -0500 Subject: [PATCH 019/132] Remove old backbone implementation Co-Authored-By: David Li --- centernet/configs/backbones.py | 3 +- .../modeling/backbones/backbone_builder.py | 34 ------ .../modeling/backbones/centernet_backbone.py | 110 ------------------ .../backbones/centernet_backbone_test.py | 39 ------- centernet/modeling/backbones/config.py | 6 - centernet/modeling/backbones/hourglass.py | 92 ++++++++++----- .../modeling/backbones/hourglass_test.py | 38 ++++++ centernet/modeling/layers/nn_blocks.py | 17 ++- centernet/modeling/layers/nn_blocks_test.py | 29 ++++- 9 files changed, 138 insertions(+), 230 deletions(-) delete mode 100644 centernet/modeling/backbones/backbone_builder.py delete mode 100644 centernet/modeling/backbones/centernet_backbone.py delete mode 100644 centernet/modeling/backbones/centernet_backbone_test.py delete mode 100644 centernet/modeling/backbones/config.py create mode 100644 centernet/modeling/backbones/hourglass_test.py diff --git a/centernet/configs/backbones.py b/centernet/configs/backbones.py index a1f266ef4..4121e8410 100644 --- a/centernet/configs/backbones.py +++ b/centernet/configs/backbones.py @@ -21,10 +21,11 @@ from official.modeling import hyperparams + @dataclasses.dataclass class Hourglass(hyperparams.Config): """Hourglass config.""" - input_channel_dims : int = 128 + input_channel_dims: int = 128 channel_dims_per_stage: Tuple[int] = (256, 256, 384, 384, 384, 512) blocks_per_stage: Tuple[int] = (2, 2, 2, 2, 2, 4) num_hourglasses: int = 2 diff --git a/centernet/modeling/backbones/backbone_builder.py b/centernet/modeling/backbones/backbone_builder.py deleted file mode 100644 index 087b0eb6f..000000000 --- a/centernet/modeling/backbones/backbone_builder.py +++ /dev/null @@ -1,34 +0,0 @@ -from centernet.modeling.backbones.centernet_backbone import CenterNetBackbone -import centernet.modeling.backbones.config - -def buildCenterNetBackbone(config): - downsample = False - n_stacks = 0 - all_filter_sizes = [] - all_rep_sizes = [] - all_strides = [] - - for layer in config: - name, filterSizes, repSizes, strides = layer - - if name == "Downsample": - downsample = True - elif name == "HourglassBlock": - n_stacks += 1 - if len(filterSizes) != len(repSizes): - print("Number of filter sizes and rep sizes must be equal") - break - all_filter_sizes.append(filterSizes) - all_rep_sizes.append(repSizes) - all_strides.append(strides) - else: - print("Invalid layer name provided") - break - - backbone = CenterNetBackbone(filter_sizes=all_filter_sizes, - rep_sizes=all_rep_sizes, - n_stacks=n_stacks, - strides=all_strides, - downsample=downsample) - - return backbone \ No newline at end of file diff --git a/centernet/modeling/backbones/centernet_backbone.py b/centernet/modeling/backbones/centernet_backbone.py deleted file mode 100644 index b6ae794ce..000000000 --- a/centernet/modeling/backbones/centernet_backbone.py +++ /dev/null @@ -1,110 +0,0 @@ -import tensorflow as tf -from centernet.modeling.layers import nn_blocks - -from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks - -class CenterNetBackbone(tf.keras.Model): - """ - CenterNet Hourglass backbone - """ - - def __init__(self, - filter_sizes, - rep_sizes, - n_stacks, - strides, - downsample=True, - **kwargs): - """ - Args: - filter_sizes: list of filter sizes for Residual blocks - rep_sizes: list of residual block repetitions per down/upsample - n_stacks: integer, number of hourglass modules in backbone - pre_layers: tf.keras layer to process input before stacked hourglasses - """ - self._filter_sizes = filter_sizes - self._rep_sizes = rep_sizes - self._n_stacks = n_stacks - self._strides = strides - self._downsample = downsample - - super().__init__(**kwargs) - - def build(self, input_shape): - # Create prelayers if downsampling input - if self._downsample: - self._pre_layers = tf.keras.Sequential([ - tf.keras.layers.Conv2D( - filters=128, kernel_size=(7, 7), strides=(2, 2), - padding='same', use_bias=True, activation='relu'), - official_nn_blocks.ResidualBlock( - filters=256, use_projection=True, - strides=2) - ], name="Prelayers") - - # Create hourglass stacks - self.hgs = [ - nn_blocks.HourglassBlock( - filter_sizes=self._filter_sizes[n], rep_sizes=self._rep_sizes[n]) - for n in range(self._n_stacks) - ] - - # Create some intermediate and postlayers to generate the heatmaps (document and make cleaner later) - inp_filters = self._filter_sizes[0][0] - - # cnvs - self.post_hg_convs = [tf.keras.layers.Conv2D( - filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), - padding='same', use_bias=True, activation='relu') - for _ in range(self._n_stacks) - ] - #cnvs_ - self.inter_hg_convs1 = [tf.keras.layers.Conv2D( - filters=256, kernel_size=(1, 1), strides=(1, 1), - padding='same', use_bias=True, activation='linear') - for _ in range(self._n_stacks - 1) - ] - #inters_ - self.inter_hg_convs2 = [tf.keras.layers.Conv2D( - filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), - padding='same', use_bias=True, activation='linear') - for _ in range(self._n_stacks - 1) - ] - # inters - self.res = [official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, - strides=2) - for _ in range(self._n_stacks - 1) - ] - - self.relu = tf.keras.layers.ReLU() - - super().build(input_shape) - - def call(self, x): - x_inter = x - if self._downsample: - x_inter = self._pre_layers(x_inter) - - all_heatmaps = [] - - for i in range(self._n_stacks): - hg = self.hgs[i] - post_conv = self.post_hg_convs[i] - - x_hg = hg(x_inter) - x_hg = post_conv(x_hg) - - all_heatmaps.append(x_hg) - - # between hourglasses, we insert intermediate layers - if i < self._n_stacks - 1: - inter_hg_conv1 = self.inter_hg_convs1[i] - inter_hg_conv2 = self.inter_hg_convs2[i] - res = self.res[i] - - x_inter = inter_hg_conv1(x_inter) + inter_hg_conv2(x_hg) - x_inter = self.relu(x_inter) - x_inter = res(x_inter) - - return x_hg, all_heatmaps diff --git a/centernet/modeling/backbones/centernet_backbone_test.py b/centernet/modeling/backbones/centernet_backbone_test.py deleted file mode 100644 index 193d3f06c..000000000 --- a/centernet/modeling/backbones/centernet_backbone_test.py +++ /dev/null @@ -1,39 +0,0 @@ -import tensorflow as tf -from centernet.modeling.layers.nn_blocks import HourglassBlock -from centernet.modeling.backbones.centernet_backbone import CenterNetBackbone - -from centernet.modeling.backbones.backbone_builder import buildCenterNetBackbone -from centernet.modeling.backbones.config import CENTERNET_HG104_CFG - -if __name__ == '__main__': - hg_test_input_shape = (1, 512, 512, 256) - bb_test_input_shape = (1, 512, 512, 3) - x_hg = tf.ones(shape=hg_test_input_shape) - x_bb = tf.ones(shape=bb_test_input_shape) - - tf.print("Testing Single hourglass") - hg = HourglassBlock(filter_sizes=filter_sizes, rep_sizes=rep_sizes) - - hg.build(input_shape=hg_test_input_shape) - tf.print('Made an hourglass!') - out = hg(x_hg) - tf.print('Hourglass module output shape:{} Expected shape:{}'.format( - tf.shape(out), hg_test_input_shape)) - - tf.print("Testing backbone") - - backbone = buildCenterNetBackbone(CENTERNET_HG104_CFG) - - backbone.build(input_shape=bb_test_input_shape) - - # Backbone summary shows output shape to be multiple for hg modules - # Maybe this is because the hg call method has a conditional? - backbone.summary() - - out_final, all_outs = backbone(x_bb) - tf.print('Made backbone!') - - expected_out_filters = CENTERNET_HG104_CFG[-1][1][0] # last hourglass filters - tf.print('Backbone output shape: {} Expected shape: {}'.format( - tf.shape(out_final), (bb_test_input_shape[0], bb_test_input_shape[1]//8, - bb_test_input_shape[2]//8, expected_out_filters))) diff --git a/centernet/modeling/backbones/config.py b/centernet/modeling/backbones/config.py deleted file mode 100644 index 0a202be90..000000000 --- a/centernet/modeling/backbones/config.py +++ /dev/null @@ -1,6 +0,0 @@ -# (name, filterSizes, repSizes, strides) -CENTERNET_HG104_CFG = [ - ("Downsample", None, None, None), - ("HourglassBlock", [256, 256, 384, 384, 384, 512], [2, 2, 2, 2, 2, 4], 1), - ("HourglassBlock", [256, 256, 384, 384, 384, 512], [2, 2, 2, 2, 2, 4], 1), -] \ No newline at end of file diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 0473efe9f..c655f58b8 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -1,6 +1,6 @@ import tensorflow as tf -from centernet.configs import backbones +from centernet.configs import backbones as cfg from centernet.modeling.layers import nn_blocks # from official.vision.beta.modeling.backbones import factory @@ -9,19 +9,21 @@ from typing import List + class Hourglass(tf.keras.Model): """ CenterNet Hourglass backbone """ - def __init__(self, - input_channel_dims : int, - channel_dims_per_stage: List[int], - blocks_per_stage: List[int], - num_hourglasses: int, - initial_downsample: bool = True, - input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 3]), - **kwargs): + def __init__( + self, + input_channel_dims: int, + channel_dims_per_stage: List[int], + blocks_per_stage: List[int], + num_hourglasses: int, + initial_downsample: bool = True, + input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 3]), + **kwargs): """ Args: channel_dims_per_stage: list of filter sizes for Residual blocks @@ -29,61 +31,87 @@ def __init__(self, num_hourglasses: integer, number of hourglass modules in backbone pre_layers: tf.keras layer to process input before stacked hourglasses """ - + # yapf: disable input = tf.keras.layers.Input(shape=input_specs.shape[1:]) x_inter = input # Create prelayers if downsampling input if initial_downsample: x_inter = tf.keras.Sequential([ - tf.keras.layers.Conv2D( - filters=input_channel_dims, kernel_size=(7, 7), strides=(2, 2), - padding='same', use_bias=True, activation='relu'), - official_nn_blocks.ResidualBlock( - filters=input_channel_dims*2, use_projection=True, - strides=2) - ], name="Prelayers")(x_inter) + tf.keras.layers.Conv2D( + filters=input_channel_dims, + kernel_size=(7, 7), + strides=(2, 2), + padding='same', + use_bias=True, + activation='relu'), + official_nn_blocks.ResidualBlock( + filters=input_channel_dims * 2, use_projection=True, strides=2) + ], name='Prelayers')(x_inter) all_heatmaps = [] for i in range(num_hourglasses): # Create hourglass stacks - x_hg = nn_blocks.HourglassBlock(channel_dims_per_stage=channel_dims_per_stage, blocks_per_stage=blocks_per_stage)(x_inter) + x_hg = nn_blocks.HourglassBlock( + channel_dims_per_stage=channel_dims_per_stage, + blocks_per_stage=blocks_per_stage + )(x_inter) - # Create some intermediate and postlayers to generate the heatmaps (document and make cleaner later) + # Create some intermediate and postlayers to generate the heatmaps + # (document and make cleaner later) inp_filters = channel_dims_per_stage[0] # cnvs x_hg = tf.keras.layers.Conv2D( - filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), - padding='same', use_bias=True, activation='relu')(x_hg) + filters=inp_filters, + kernel_size=(3, 3), + strides=(1, 1), + padding='same', + use_bias=True, + activation='relu' + )(x_hg) all_heatmaps.append(x_hg) # between hourglasses, we insert intermediate layers if i < num_hourglasses - 1: - #cnvs_ + # cnvs_ inter_hg_conv1 = tf.keras.layers.Conv2D( - filters=input_channel_dims*2, kernel_size=(1, 1), strides=(1, 1), - padding='same', use_bias=True, activation='linear')(x_inter) - - #inters_ + filters=input_channel_dims * 2, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + use_bias=True, + activation='linear' + )(x_inter) + + # inters_ inter_hg_conv2 = tf.keras.layers.Conv2D( - filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), - padding='same', use_bias=True, activation='linear')(x_hg) + filters=inp_filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + use_bias=True, + activation='linear' + )(x_hg) x_inter = inter_hg_conv1 + inter_hg_conv2 x_inter = tf.keras.layers.ReLU()(x_inter) # inters x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, - strides=2)(x_inter) + filters=inp_filters, use_projection=True, strides=2 + )(x_inter) + # yapf: enable + + super().__init__(inputs=input, outputs=all_heatmaps, **kwargs) - super().__init__(inputs=input, outputs=[x_hg, all_heatmaps], **kwargs) # @factory.register_backbone_builder('hourglass') -@register.backbone('hourglass', backbones.Hourglass) + + +@register.backbone('hourglass', cfg.Hourglass) def build_hourglass( input_specs: tf.keras.layers.InputSpec, model_config, diff --git a/centernet/modeling/backbones/hourglass_test.py b/centernet/modeling/backbones/hourglass_test.py new file mode 100644 index 000000000..37932377e --- /dev/null +++ b/centernet/modeling/backbones/hourglass_test.py @@ -0,0 +1,38 @@ +from absl.testing import parameterized +import tensorflow as tf +import numpy as np + +from centernet.configs import backbones as cfg +from centernet.modeling.backbones import hourglass + +import dataclasses +from official.modeling import hyperparams +from official.vision.beta.configs import backbones + +@dataclasses.dataclass +class CenterNet(hyperparams.Config): + backbone : backbones.Backbone = backbones.Backbone(type='hourglass') + +class HourglassTest(tf.test.TestCase, parameterized.TestCase): + + def test_hourglass(self): + # model = hourglass.Hourglass( + # blocks_per_stage=[2, 3, 4, 5, 6], + # input_channel_dims=4, + # channel_dims_per_stage=[6, 8, 10, 12, 14], + # num_hourglasses=2) + # outputs = model(np.zeros((2, 64, 64, 3), dtype=np.float32)) + # self.assertEqual(outputs[0].shape, (2, 16, 16, 6)) + # self.assertEqual(outputs[1].shape, (2, 16, 16, 6)) + + backbone = hourglass.build_hourglass(tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]), CenterNet()) + input = np.zeros((2, 512, 512, 3), dtype=np.float32) + outputs = backbone(input) + backbone.summary() + # print(type(outputs[0]), len(outputs[0])) + self.assertEqual(outputs[0].shape, (2, 128, 128, 256)) + self.assertEqual(outputs[1].shape, (2, 64, 64, 256)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 67f385930..4221256a9 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -2,12 +2,17 @@ from official.modeling import tf_utils from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks + class HourglassBlock(tf.keras.layers.Layer): """ Hourglass module """ - def __init__(self, channel_dims_per_stage, blocks_per_stage, strides=1, **kwargs): + def __init__(self, + channel_dims_per_stage, + blocks_per_stage, + strides=1, + **kwargs): """ Args: channel_dims_per_stage: list of filter sizes for Residual blocks @@ -19,8 +24,8 @@ def __init__(self, channel_dims_per_stage, blocks_per_stage, strides=1, **kwargs self._blocks_per_stage = blocks_per_stage self._strides = strides - assert len(channel_dims_per_stage) == len(blocks_per_stage), 'filter size and ' \ - 'residual block repetition lists must have the same length' + assert len(channel_dims_per_stage) == len(blocks_per_stage), 'filter ' \ + 'size and residual block repetition lists must have the same length' self._filters = channel_dims_per_stage[0] self._reps = blocks_per_stage[0] @@ -57,9 +62,9 @@ def build(self, input_shape): # recursively define inner hourglasses self.inner_hg = type(self)( - channel_dims_per_stage=self._channel_dims_per_stage[1:], - blocks_per_stage=self._blocks_per_stage[1:], - strides=self._strides) + channel_dims_per_stage=self._channel_dims_per_stage[1:], + blocks_per_stage=self._blocks_per_stage[1:], + strides=self._strides) # outer hourglass structures end_block = [ diff --git a/centernet/modeling/layers/nn_blocks_test.py b/centernet/modeling/layers/nn_blocks_test.py index 951193145..66060e72e 100644 --- a/centernet/modeling/layers/nn_blocks_test.py +++ b/centernet/modeling/layers/nn_blocks_test.py @@ -1,5 +1,6 @@ from absl.testing import parameterized import tensorflow as tf +import numpy as np from centernet.modeling.layers import nn_blocks @@ -53,7 +54,7 @@ def build(self, input_shape): self.max1 = self.make_pool_layer(curr_dim) self.low1 = self.make_hg_layer(3, curr_dim, next_dim, curr_mod, **kwargs) self.low2 = type(self)( - dims, modules, k=k+1, **kwargs + dims, modules, k=k + 1, **kwargs ) if self.n - k > 1 else \ self.make_low_layer( 3, next_dim, next_dim, next_mod, **kwargs @@ -113,10 +114,34 @@ class NNBlocksTest(parameterized.TestCase, tf.test.TestCase): def test_hourglass_block(self): dims = [256, 256, 384, 384, 384, 512] modules = [2, 2, 2, 2, 2, 4] - model = nn_blocks.HourglassBlock(len(dims) - 1, dims, modules) + model = nn_blocks.HourglassBlock(dims, modules) test_input = tf.keras.Input((512, 512, 256)) _ = model(test_input) + filter_sizes = [256, 256, 384, 384, 384, 512] + rep_sizes = [2, 2, 2, 2, 2, 4] + + hg_test_input_shape = (1, 512, 512, 256) + bb_test_input_shape = (1, 512, 512, 3) + x_hg = tf.ones(shape=hg_test_input_shape) + x_bb = tf.ones(shape=bb_test_input_shape) + + hg = nn_blocks.HourglassBlock( + channel_dims_per_stage=filter_sizes, blocks_per_stage=rep_sizes) + + hg.build(input_shape=hg_test_input_shape) + out = hg(x_hg) + self.assertAllEqual( + tf.shape(out), hg_test_input_shape, + 'Hourglass module output shape and expected shape differ') + + # ODAPI Test + layer = nn_blocks.HourglassBlock( + blocks_per_stage=[2, 3, 4, 5, 6], + channel_dims_per_stage=[4, 6, 8, 10, 12]) + output = layer(np.zeros((2, 64, 64, 4), dtype=np.float32)) + self.assertEqual(output.shape, (2, 64, 64, 4)) + if __name__ == '__main__': tf.test.main() From c6a23e01aef376815acbf33e2cebd44b83149346 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 11 Feb 2021 16:52:31 -0500 Subject: [PATCH 020/132] Bug fixes There was no initial convolution before when there was one in Google's code. The residual connection between hourglasses didn't match before, now they do. Before, the final residual downsampled. Now it doesn't. Co-Authored-By: David Li --- centernet/modeling/backbones/hourglass.py | 40 +++++++++++-------- .../modeling/backbones/hourglass_test.py | 20 +++++----- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index c655f58b8..56e9f1f50 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -35,19 +35,29 @@ def __init__( input = tf.keras.layers.Input(shape=input_specs.shape[1:]) x_inter = input + # Create some intermediate and postlayers to generate the heatmaps + # (document and make cleaner later) + inp_filters = channel_dims_per_stage[0] + # Create prelayers if downsampling input if initial_downsample: - x_inter = tf.keras.Sequential([ - tf.keras.layers.Conv2D( - filters=input_channel_dims, - kernel_size=(7, 7), - strides=(2, 2), - padding='same', - use_bias=True, - activation='relu'), - official_nn_blocks.ResidualBlock( - filters=input_channel_dims * 2, use_projection=True, strides=2) - ], name='Prelayers')(x_inter) + prelayer_kernel_size = 7 + prelayer_strides = 2 + else: + prelayer_kernel_size = 3 + prelayer_strides = 1 + + x_inter = tf.keras.layers.Conv2D( + filters=input_channel_dims, + kernel_size=prelayer_kernel_size, + strides=prelayer_strides, + padding='same', # TODO: Google used valid + use_bias=True, + activation='relu' + )(x_inter) + x_inter = official_nn_blocks.ResidualBlock( + filters=inp_filters, use_projection=True, strides=prelayer_strides + )(x_inter) all_heatmaps = [] @@ -58,10 +68,6 @@ def __init__( blocks_per_stage=blocks_per_stage )(x_inter) - # Create some intermediate and postlayers to generate the heatmaps - # (document and make cleaner later) - inp_filters = channel_dims_per_stage[0] - # cnvs x_hg = tf.keras.layers.Conv2D( filters=inp_filters, @@ -78,7 +84,7 @@ def __init__( if i < num_hourglasses - 1: # cnvs_ inter_hg_conv1 = tf.keras.layers.Conv2D( - filters=input_channel_dims * 2, + filters=inp_filters, # TODO: input_channel_dims * 2 was here before kernel_size=(1, 1), strides=(1, 1), padding='same', @@ -101,7 +107,7 @@ def __init__( # inters x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, strides=2 + filters=inp_filters, use_projection=True, strides=1 # TODO: strides=2 ? )(x_inter) # yapf: enable diff --git a/centernet/modeling/backbones/hourglass_test.py b/centernet/modeling/backbones/hourglass_test.py index 37932377e..f8b423961 100644 --- a/centernet/modeling/backbones/hourglass_test.py +++ b/centernet/modeling/backbones/hourglass_test.py @@ -16,22 +16,20 @@ class CenterNet(hyperparams.Config): class HourglassTest(tf.test.TestCase, parameterized.TestCase): def test_hourglass(self): - # model = hourglass.Hourglass( - # blocks_per_stage=[2, 3, 4, 5, 6], - # input_channel_dims=4, - # channel_dims_per_stage=[6, 8, 10, 12, 14], - # num_hourglasses=2) - # outputs = model(np.zeros((2, 64, 64, 3), dtype=np.float32)) - # self.assertEqual(outputs[0].shape, (2, 16, 16, 6)) - # self.assertEqual(outputs[1].shape, (2, 16, 16, 6)) + model = hourglass.Hourglass( + blocks_per_stage=[2, 3, 4, 5, 6], + input_channel_dims=4, + channel_dims_per_stage=[6, 8, 10, 12, 14], + num_hourglasses=2) + outputs = model(np.zeros((2, 64, 64, 3), dtype=np.float32)) + self.assertEqual(outputs[0].shape, (2, 16, 16, 6)) + self.assertEqual(outputs[1].shape, (2, 16, 16, 6)) backbone = hourglass.build_hourglass(tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]), CenterNet()) input = np.zeros((2, 512, 512, 3), dtype=np.float32) outputs = backbone(input) - backbone.summary() - # print(type(outputs[0]), len(outputs[0])) self.assertEqual(outputs[0].shape, (2, 128, 128, 256)) - self.assertEqual(outputs[1].shape, (2, 64, 64, 256)) + self.assertEqual(outputs[1].shape, (2, 128, 128, 256)) if __name__ == '__main__': From ef6cf215be8a91e857809347fefc8244b5bca45f Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 11 Feb 2021 17:16:39 -0500 Subject: [PATCH 021/132] Some changes to the Hourglass configs --- centernet/configs/backbones.py | 8 ++++--- centernet/modeling/backbones/hourglass.py | 23 ++++++++++++++++--- .../modeling/backbones/hourglass_test.py | 7 ++++-- centernet/modeling/layers/nn_blocks.py | 9 ++++++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/centernet/configs/backbones.py b/centernet/configs/backbones.py index 4121e8410..9338e34e3 100644 --- a/centernet/configs/backbones.py +++ b/centernet/configs/backbones.py @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================== """Backbones configurations.""" -from typing import Tuple +from typing import List # Import libraries import dataclasses @@ -26,7 +26,9 @@ class Hourglass(hyperparams.Config): """Hourglass config.""" input_channel_dims: int = 128 - channel_dims_per_stage: Tuple[int] = (256, 256, 384, 384, 384, 512) - blocks_per_stage: Tuple[int] = (2, 2, 2, 2, 2, 4) + channel_dims_per_stage: List[int] = dataclasses.field( + default_factory=lambda: [256, 256, 384, 384, 384, 512]) + blocks_per_stage: List[int] = dataclasses.field( + default_factory=lambda: [2, 2, 2, 2, 2, 4]) num_hourglasses: int = 2 initial_downsample: bool = True diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 56e9f1f50..990989f31 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -51,7 +51,7 @@ def __init__( filters=input_channel_dims, kernel_size=prelayer_kernel_size, strides=prelayer_strides, - padding='same', # TODO: Google used valid + padding='same', use_bias=True, activation='relu' )(x_inter) @@ -84,7 +84,7 @@ def __init__( if i < num_hourglasses - 1: # cnvs_ inter_hg_conv1 = tf.keras.layers.Conv2D( - filters=inp_filters, # TODO: input_channel_dims * 2 was here before + filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), padding='same', @@ -107,12 +107,29 @@ def __init__( # inters x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, strides=1 # TODO: strides=2 ? + filters=inp_filters, use_projection=True, strides=1 )(x_inter) # yapf: enable super().__init__(inputs=input, outputs=all_heatmaps, **kwargs) + self._input_channel_dims = input_channel_dims + self._channel_dims_per_stage = channel_dims_per_stage + self._blocks_per_stage = blocks_per_stage + self._num_hourglasses = num_hourglasses + self._initial_downsample = initial_downsample + + def get_config(self): + layer_config = { + 'input_channel_dims': self._input_channel_dims, + 'channel_dims_per_stage': self._channel_dims_per_stage, + 'blocks_per_stage': self._blocks_per_stage, + 'num_hourglasses': self._num_hourglasses, + 'initial_downsample': self._initial_downsample + } + layer_config.update(super().get_config()) + return layer_config + # @factory.register_backbone_builder('hourglass') diff --git a/centernet/modeling/backbones/hourglass_test.py b/centernet/modeling/backbones/hourglass_test.py index f8b423961..71be62258 100644 --- a/centernet/modeling/backbones/hourglass_test.py +++ b/centernet/modeling/backbones/hourglass_test.py @@ -9,9 +9,11 @@ from official.modeling import hyperparams from official.vision.beta.configs import backbones + @dataclasses.dataclass class CenterNet(hyperparams.Config): - backbone : backbones.Backbone = backbones.Backbone(type='hourglass') + backbone: backbones.Backbone = backbones.Backbone(type='hourglass') + class HourglassTest(tf.test.TestCase, parameterized.TestCase): @@ -25,7 +27,8 @@ def test_hourglass(self): self.assertEqual(outputs[0].shape, (2, 16, 16, 6)) self.assertEqual(outputs[1].shape, (2, 16, 16, 6)) - backbone = hourglass.build_hourglass(tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]), CenterNet()) + backbone = hourglass.build_hourglass( + tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]), CenterNet()) input = np.zeros((2, 512, 512, 3), dtype=np.float32) outputs = backbone(input) self.assertEqual(outputs[0].shape, (2, 128, 128, 256)) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 4221256a9..ed3144a07 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -89,3 +89,12 @@ def call(self, x): inner_output = self.inner_hg(x_pooled) hg_output = self.end_block(inner_output) return self.upsample_layer(hg_output) + x_side + + def get_config(self): + layer_config = { + 'channel_dims_per_stage': self._channel_dims_per_stage, + 'blocks_per_stage': self._blocks_per_stage, + 'strides': self._strides + } + layer_config.update(super().get_config()) + return layer_config From b85dbddbb9573c7faa5860d05b97287d03d5320b Mon Sep 17 00:00:00 2001 From: davidliii Date: Sat, 13 Feb 2021 22:40:56 -0500 Subject: [PATCH 022/132] I am dumb. You are right David. We need the YOLO ConvBN. --- centernet/modeling/backbones/hourglass.py | 13 +- centernet/modeling/layers/nn_blocks.py | 181 ++++++++++++++++++++++ 2 files changed, 186 insertions(+), 8 deletions(-) diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 990989f31..83cacaf28 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -10,6 +10,7 @@ from typing import List +@tf.keras.utils.register_keras_serializable(package='centernet') class Hourglass(tf.keras.Model): """ CenterNet Hourglass backbone @@ -47,12 +48,11 @@ def __init__( prelayer_kernel_size = 3 prelayer_strides = 1 - x_inter = tf.keras.layers.Conv2D( + x_inter = nn_blocks.ConvBN( filters=input_channel_dims, kernel_size=prelayer_kernel_size, strides=prelayer_strides, padding='same', - use_bias=True, activation='relu' )(x_inter) x_inter = official_nn_blocks.ResidualBlock( @@ -69,12 +69,11 @@ def __init__( )(x_inter) # cnvs - x_hg = tf.keras.layers.Conv2D( + x_hg = nn_blocks.ConvBN( filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), padding='same', - use_bias=True, activation='relu' )(x_hg) @@ -83,22 +82,20 @@ def __init__( # between hourglasses, we insert intermediate layers if i < num_hourglasses - 1: # cnvs_ - inter_hg_conv1 = tf.keras.layers.Conv2D( + inter_hg_conv1 = nn_blocks.ConvBN( filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), padding='same', - use_bias=True, activation='linear' )(x_inter) # inters_ - inter_hg_conv2 = tf.keras.layers.Conv2D( + inter_hg_conv2 = nn_blocks.ConvBN( filters=inp_filters, kernel_size=(1, 1), strides=(1, 1), padding='same', - use_bias=True, activation='linear' )(x_hg) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index ed3144a07..247b12066 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -3,6 +3,187 @@ from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks +@tf.keras.utils.register_keras_serializable(package='centernet') +class Identity(tf.keras.layers.Layer): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def call(self, input): + return input + + +@tf.keras.utils.register_keras_serializable(package='centernet') +class ConvBN(tf.keras.layers.Layer): + """ + Modified Convolution layer to match that of the DarkNet Library. The Layer is a standards combination of Conv BatchNorm Activation, + however, the use of bias in the conv is determined by the use of batch normalization. + Cross Stage Partial networks (CSPNets) were proposed in: + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang Chen, Jun-Wei Hsieh + CSPNet: A New Backbone that can Enhance Learning Capability of CNN. arXiv:1911.11929 + Args: + filters: integer for output depth, or the number of features to learn + kernel_size: integer or tuple for the shape of the weight matrix or kernel to learn + strides: integer of tuple how much to move the kernel after each kernel use + padding: string 'valid' or 'same', if same, then pad the image, else do not + dialtion_rate: tuple to indicate how much to modulate kernel weights and + how many pixels in a feature map to skip + kernel_initializer: string to indicate which function to use to initialize weights + bias_initializer: string to indicate which function to use to initialize bias + kernel_regularizer: string to indicate which function to use to regularizer weights + bias_regularizer: string to indicate which function to use to regularizer bias + use_bn: boolean for whether to use batch normalization + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics (across all input batches) + norm_moment: float for moment to use for batch normalization + norm_epsilon: float for batch normalization epsilon + activation: string or None for activation function to use in layer, + if None activation is replaced by linear + leaky_alpha: float to use as alpha if activation function is leaky + **kwargs: Keyword Arguments + """ + + def __init__( + self, + filters=1, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + dilation_rate=(1, 1), + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + bias_regularizer=None, + kernel_regularizer=None, # Specify the weight decay as the default will not work. + use_bn=True, + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + activation='leaky', + leaky_alpha=0.1, + **kwargs): + + # convolution params + self._filters = filters + self._kernel_size = kernel_size + self._strides = strides + self._padding = padding + self._dilation_rate = dilation_rate + self._kernel_initializer = kernel_initializer + self._bias_initializer = bias_initializer + self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + + # batch normalization params + self._use_bn = use_bn + self._use_sync_bn = use_sync_bn + self._norm_moment = norm_momentum + self._norm_epsilon = norm_epsilon + + if tf.keras.backend.image_data_format() == 'channels_last': + # format: (batch_size, height, width, channels) + self._bn_axis = -1 + else: + # format: (batch_size, channels, width, height) + self._bn_axis = 1 + + # activation params + self._activation = activation + self._leaky_alpha = leaky_alpha + + super().__init__(**kwargs) + + def build(self, input_shape): + kernel_size = self._kernel_size if isinstance(self._kernel_size, + int) else self._kernel_size[0] + dilation_rate = self._dilation_rate if isinstance( + self._dilation_rate, int) else self._dilation_rate[0] + if self._padding == 'same' and kernel_size != 1: + padding = (dilation_rate * (kernel_size - 1)) + left_shift = padding // 2 + self._zeropad = tf.keras.layers.ZeroPadding2D([[left_shift, left_shift], + [left_shift, left_shift]]) + else: + self._zeropad = Identity() + + use_bias = not self._use_bn + + # self.conv = tf.keras.layers.Conv2D( + # filters=self._filters, + # kernel_size=self._kernel_size, + # strides=self._strides, + # padding= self._padding,# 'valid', + # dilation_rate=self._dilation_rate, + # use_bias=use_bias, + # kernel_initializer=self._kernel_initializer, + # bias_initializer=self._bias_initializer, + # kernel_regularizer=self._kernel_regularizer, + # bias_regularizer=self._bias_regularizer) + + self.conv = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=self._kernel_size, + strides=self._strides, + padding='valid', + dilation_rate=self._dilation_rate, + use_bias=use_bias, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + + if self._use_bn: + if self._use_sync_bn: + self.bn = tf.keras.layers.experimental.SyncBatchNormalization( + momentum=self._norm_moment, + epsilon=self._norm_epsilon, + axis=self._bn_axis) + else: + self.bn = tf.keras.layers.BatchNormalization( + momentum=self._norm_moment, + epsilon=self._norm_epsilon, + axis=self._bn_axis) + else: + self.bn = Identity() + + if self._activation == 'leaky': + self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) + elif self._activation == 'mish': + self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) + else: + self._activation_fn = tf_utils.get_activation( + self._activation) # tf.keras.layers.Activation(self._activation) + + def call(self, x): + x = self._zeropad(x) + x = self.conv(x) + x = self.bn(x) + x = self._activation_fn(x) + return x + + def get_config(self): + # used to store/share parameters to reconstruct the model + layer_config = { + 'filters': self._filters, + 'kernel_size': self._kernel_size, + 'strides': self._strides, + 'padding': self._padding, + 'dilation_rate': self._dilation_rate, + 'kernel_initializer': self._kernel_initializer, + 'bias_initializer': self._bias_initializer, + 'bias_regularizer': self._bias_regularizer, + 'kernel_regularizer': self._kernel_regularizer, + 'use_bn': self._use_bn, + 'use_sync_bn': self._use_sync_bn, + 'norm_moment': self._norm_moment, + 'norm_epsilon': self._norm_epsilon, + 'activation': self._activation, + 'leaky_alpha': self._leaky_alpha + } + layer_config.update(super().get_config()) + return layer_config + + +@tf.keras.utils.register_keras_serializable(package='centernet') class HourglassBlock(tf.keras.layers.Layer): """ Hourglass module From 6fbef32821dacdac74b500c82490a691cdc3e5c2 Mon Sep 17 00:00:00 2001 From: David Li Date: Sun, 14 Feb 2021 01:21:01 -0500 Subject: [PATCH 023/132] wrote head and conv nn_block + tests (adding more) --- centernet/modeling/heads/centernet_head.py | 67 +++++++++++++++++++ .../modeling/heads/centernet_head_test.py | 20 ++++++ centernet/modeling/layers/nn_blocks.py | 35 ++++++++++ 3 files changed, 122 insertions(+) create mode 100644 centernet/modeling/heads/centernet_head.py create mode 100644 centernet/modeling/heads/centernet_head_test.py diff --git a/centernet/modeling/heads/centernet_head.py b/centernet/modeling/heads/centernet_head.py new file mode 100644 index 000000000..34f931140 --- /dev/null +++ b/centernet/modeling/heads/centernet_head.py @@ -0,0 +1,67 @@ +import tensorflow as tf +from centernet.modeling.layers.nn_blocks import CenterNetHeadConv + +class CenterNetHead(tf.keras.layers.Layer): + """ + CenterNet Head + """ + def __init__(self, + classes: int = 91, + joints: int = 17, + task: str = "2D", + **kwargs): + """ + Args: + classes: int, number of possible class predictions for the network + joints: int, number of possible joint location predictions for pose + estimation + task: string, indicating the prediction task. Valid values are + "2D", "3D", and "pose". + + call Returns: + dictionary where the keys-value pairs are the output names and output + tensors + """ + self._classes = classes + self._joints = joints + self._task = task + + # These specify the layers required for each type of task + # Each spec is a dictionary of the output name and its + # respective channel depth + task_output_specs = { + "2D" : {'heatmaps': self._classes, + 'local_offset': 2, + 'object_size': 2 + }, + "3D": {'heatmaps': self._classes, + 'local_offset': 2, + 'object_size': 3, + 'depth': 1, + 'orientation': 8 + }, + "pose": {'heatmaps': self._classes, + 'joint_locs': self._joints * 2, + 'joint_heatmap': self._joints, + 'joint_offset': 2 + } + } + + self.task_outputs = task_output_specs[self._task] + + super().__init__(**kwargs) + + def build(self, input_shape): + self.layers = {} + for key in self.task_outputs: + num_filters = self.task_outputs[key] + self.layers[key] = CenterNetHeadConv(output_filters=num_filters, name=key) + + super().build(input_shape) + + def call(self, x): + outputs = {} + for key in self.layers: + outputs[key] = self.layers[key](x) + + return outputs \ No newline at end of file diff --git a/centernet/modeling/heads/centernet_head_test.py b/centernet/modeling/heads/centernet_head_test.py new file mode 100644 index 000000000..3d564d8a2 --- /dev/null +++ b/centernet/modeling/heads/centernet_head_test.py @@ -0,0 +1,20 @@ +from absl.testing import parameterized +import tensorflow as tf +import numpy as np + +from centernet.modeling.heads import centernet_head + + +class CenterNetHeadTest(tf.test.TestCase, parameterized.TestCase): + + def test_head(self): + head = centernet_head.CenterNetHead(classes=91, task="2D") + + outputs = head(np.zeros((2, 128, 128, 256), dtype=np.float32)) + self.assertEqual(len(outputs), 3) + self.assertEqual(outputs['heatmaps'].shape, (2, 128, 128, 91)) + self.assertEqual(outputs['local_offset'].shape, (2, 128, 128, 2)) + self.assertEqual(outputs['object_size'].shape, (2, 128, 128, 2)) + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index ed3144a07..b712fff00 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -98,3 +98,38 @@ def get_config(self): } layer_config.update(super().get_config()) return layer_config + +class CenterNetHeadConv(tf.keras.layers.Layer): + """ + Convolution block for the CenterNet head. This is used to generate + both the confidence heatmaps and other regressed predictions such as + center offsets, object size, etc. + """ + def __init__(self, + output_filters: int, + name: str, + **kwargs): + """ + Args: + output_filters: int, channel depth of layer output + name: string, layer name + """ + self._output_filters = output_filters + super().__init__(name=name, **kwargs) + + def build(self, input_shape): + n_channels = input_shape[-1] + + self.conv1 = tf.keras.layers.Conv2D(filters=n_channels, + kernel_size=(3, 3), padding='same') + + self.relu = tf.keras.layers.ReLU() + + self.conv2 = tf.keras.layers.Conv2D(filters=self._output_filters, + kernel_size=(1, 1), padding='valid') + + def call(self, x): + x = self.conv1(x) + x = self.relu(x) + x = self.conv2(x) + return x \ No newline at end of file From 5415f9ade999f47f9e7f58dc71a2fd53e40f6a3c Mon Sep 17 00:00:00 2001 From: anivegesana Date: Sun, 14 Feb 2021 11:15:59 -0500 Subject: [PATCH 024/132] Add test cases for gaussian_radius --- centernet/ops/preprocessing_ops.py | 2 + centernet/ops/preprocessing_ops_test.py | 57 +++++++++++++++++++++++++ centernet/utils/groundtruth_test.py | 3 ++ 3 files changed, 62 insertions(+) create mode 100644 centernet/ops/preprocessing_ops.py create mode 100644 centernet/ops/preprocessing_ops_test.py create mode 100644 centernet/utils/groundtruth_test.py diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py new file mode 100644 index 000000000..0dd42e503 --- /dev/null +++ b/centernet/ops/preprocessing_ops.py @@ -0,0 +1,2 @@ +# TODO: Move groundtruth.py to preprocessing_ops.py when possible +from centernet.utils.groundtruth import * diff --git a/centernet/ops/preprocessing_ops_test.py b/centernet/ops/preprocessing_ops_test.py new file mode 100644 index 000000000..ac97da7a0 --- /dev/null +++ b/centernet/ops/preprocessing_ops_test.py @@ -0,0 +1,57 @@ +import tensorflow as tf +import numpy as np + +import centernet.utils.groundtruth as preprocessing_ops + +class CenterNetBoxTargetAssignerTest(tf.test.TestCase): + + def __init__(self, *args, **kwargs): + super(CenterNetBoxTargetAssignerTest, self).__init__(*args, **kwargs) + self._box_center = [0.0, 0.0, 1.0, 1.0] + self._box_center_small = [0.25, 0.25, 0.75, 0.75] + self._box_lower_left = [0.5, 0.0, 1.0, 0.5] + self._box_center_offset = [0.1, 0.05, 1.0, 1.0] + self._box_odd_coordinates = [0.1625, 0.2125, 0.5625, 0.9625] + + def test_max_distance_for_overlap(self): + """Test that the distance ensures the IoU with random boxes.""" + + # TODO(vighneshb) remove this after the `_smallest_positive_root` + # function if fixed. + self.skipTest(('Skipping test because we are using an incorrect version of' + 'the `max_distance_for_overlap` function to reproduce' + ' results.')) + + rng = np.random.RandomState(0) + n_samples = 100 + + width = rng.uniform(1, 100, size=n_samples) + height = rng.uniform(1, 100, size=n_samples) + min_iou = rng.uniform(0.1, 1.0, size=n_samples) + + max_dist = preprocessing_ops.gaussian_radius((height, width), min_iou) + xmin1 = np.zeros(n_samples) + ymin1 = np.zeros(n_samples) + xmax1 = np.zeros(n_samples) + width + ymax1 = np.zeros(n_samples) + height + + xmin2 = max_dist * np.cos(rng.uniform(0, 2 * np.pi)) + ymin2 = max_dist * np.sin(rng.uniform(0, 2 * np.pi)) + xmax2 = width + max_dist * np.cos(rng.uniform(0, 2 * np.pi)) + ymax2 = height + max_dist * np.sin(rng.uniform(0, 2 * np.pi)) + + boxes1 = np.vstack([ymin1, xmin1, ymax1, xmax1]).T + boxes2 = np.vstack([ymin2, xmin2, ymax2, xmax2]).T + + iou = np.diag(np_box_ops.iou(boxes1, boxes2)) + + self.assertTrue(np.all(iou >= min_iou)) + + def test_max_distance_for_overlap_centernet(self): + """Test the version of the function used in the CenterNet paper.""" + distance = preprocessing_ops.gaussian_radius((10, 5), 0.5) + self.assertAlmostEqual(2.807764064, distance.numpy()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/utils/groundtruth_test.py b/centernet/utils/groundtruth_test.py new file mode 100644 index 000000000..9d8af4660 --- /dev/null +++ b/centernet/utils/groundtruth_test.py @@ -0,0 +1,3 @@ +from centernet.ops.preprocessing_ops_test import * +if __name__ == '__main__': + tf.test.main() From 037e78c3be039e4c82c61dd3c4b7d60fd91e4167 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Sun, 14 Feb 2021 11:40:53 -0500 Subject: [PATCH 025/132] Add base code for OD task --- centernet/tasks/centernet_object_detection.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 centernet/tasks/centernet_object_detection.py diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py new file mode 100644 index 000000000..6788ee578 --- /dev/null +++ b/centernet/tasks/centernet_object_detection.py @@ -0,0 +1,24 @@ +import tensorflow as tf +from tensorflow.keras.mixed_precision import experimental as mixed_precision + +from absl import logging +from official.core import base_task +from official.core import input_reader +from official.core import task_factory + +from official.vision.beta.evaluation import coco_evaluator + + +# @task_factory.register_task_cls(exp_cfg.YoloTask) +class CenterNetObjectDetectionTask(base_task.Task): + + def __init__(self, params, logging_dir: str = None): + super().__init__(params, logging_dir) + + def build_losses(self, outputs, labels, aux_losses=None): + loss = 0.0 + metric_dict = dict() + + # TODO: Calculate loss + + return loss, metric_dict From fd4eaeda3947b91e5874c898bccb4ae1a3f3e645 Mon Sep 17 00:00:00 2001 From: patel996 Date: Sun, 14 Feb 2021 12:53:59 -0500 Subject: [PATCH 026/132] loss_utils.py for helper functions --- centernet/tasks/loss_utils.py | 246 ++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 centernet/tasks/loss_utils.py diff --git a/centernet/tasks/loss_utils.py b/centernet/tasks/loss_utils.py new file mode 100644 index 000000000..be06f70f3 --- /dev/null +++ b/centernet/tasks/loss_utils.py @@ -0,0 +1,246 @@ +class BoxListFields(object): + """Naming conventions for BoxLists. + Attributes: + boxes: bounding box coordinates. + classes: classes per bounding box. + scores: scores per bounding box. + weights: sample weights per bounding box. + objectness: objectness score per bounding box. + masks: masks per bounding box. + boundaries: boundaries per bounding box. + keypoints: keypoints per bounding box. + keypoint_heatmaps: keypoint heatmaps per bounding box. + is_crowd: is_crowd annotation per bounding box. + """ + boxes = 'boxes' + classes = 'classes' + scores = 'scores' + weights = 'weights' + objectness = 'objectness' + masks = 'masks' + boundaries = 'boundaries' + keypoints = 'keypoints' + keypoint_heatmaps = 'keypoint_heatmaps' + is_crowd = 'is_crowd' + + +class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)): + """Abstract base class for detection models. + Extends tf.Module to guarantee variable tracking. + """ + + def __init__(self, num_classes): + """Constructor. + Args: + num_classes: number of classes. Note that num_classes *does not* include + background categories that might be implicitly predicted in various + implementations. + """ + self._num_classes = num_classes + self._groundtruth_lists = {} + + super(DetectionModel, self).__init__() + + @property + def num_classes(self): + return self._num_classes + + def groundtruth_lists(self, field): + """Access list of groundtruth tensors. + Args: + field: a string key, options are + fields.BoxListFields.{boxes,classes,masks,keypoints, + keypoint_visibilities, densepose_*, track_ids, + temporal_offsets, track_match_flags} + fields.InputDataFields.is_annotated. + Returns: + a list of tensors holding groundtruth information (see also + provide_groundtruth function below), with one entry for each image in the + batch. + Raises: + RuntimeError: if the field has not been provided via provide_groundtruth. + """ + if field not in self._groundtruth_lists: + raise RuntimeError('Groundtruth tensor {} has not been provided'.format( + field)) + return self._groundtruth_lists[field] + +class InputDataFields(object): + """Names for the input tensors. + Holds the standard data field names to use for identifying input tensors. This + should be used by the decoder to identify keys for the returned tensor_dict + containing input tensors. And it should be used by the model to identify the + tensors it needs. + Attributes: + image: image. + image_additional_channels: additional channels. + original_image: image in the original input size. + key: unique key corresponding to image. + source_id: source of the original image. + filename: original filename of the dataset (without common path). + groundtruth_image_classes: image-level class labels. + groundtruth_boxes: coordinates of the ground truth boxes in the image. + groundtruth_classes: box-level class labels. + groundtruth_label_types: box-level label types (e.g. explicit negative). + groundtruth_is_crowd: [DEPRECATED, use groundtruth_group_of instead] + is the groundtruth a single object or a crowd. + groundtruth_area: area of a groundtruth segment. + groundtruth_difficult: is a `difficult` object + groundtruth_group_of: is a `group_of` objects, e.g. multiple objects of the + same class, forming a connected group, where instances are heavily + occluding each other. + proposal_boxes: coordinates of object proposal boxes. + proposal_objectness: objectness score of each proposal. + groundtruth_instance_masks: ground truth instance masks. + groundtruth_instance_boundaries: ground truth instance boundaries. + groundtruth_instance_classes: instance mask-level class labels. + groundtruth_keypoints: ground truth keypoints. + groundtruth_keypoint_visibilities: ground truth keypoint visibilities. + groundtruth_label_scores: groundtruth label scores. + groundtruth_weights: groundtruth weight factor for bounding boxes. + num_groundtruth_boxes: number of groundtruth boxes. + true_image_shapes: true shapes of images in the resized images, as resized + images can be padded with zeros. + multiclass_scores: the label score per class for each box. + """ + image = 'image' + image_additional_channels = 'image_additional_channels' + original_image = 'original_image' + key = 'key' + source_id = 'source_id' + filename = 'filename' + groundtruth_image_classes = 'groundtruth_image_classes' + groundtruth_boxes = 'groundtruth_boxes' + groundtruth_classes = 'groundtruth_classes' + groundtruth_label_types = 'groundtruth_label_types' + groundtruth_is_crowd = 'groundtruth_is_crowd' + groundtruth_area = 'groundtruth_area' + groundtruth_difficult = 'groundtruth_difficult' + groundtruth_group_of = 'groundtruth_group_of' + proposal_boxes = 'proposal_boxes' + proposal_objectness = 'proposal_objectness' + groundtruth_instance_masks = 'groundtruth_instance_masks' + groundtruth_instance_boundaries = 'groundtruth_instance_boundaries' + groundtruth_instance_classes = 'groundtruth_instance_classes' + groundtruth_keypoints = 'groundtruth_keypoints' + groundtruth_keypoint_visibilities = 'groundtruth_keypoint_visibilities' + groundtruth_label_scores = 'groundtruth_label_scores' + groundtruth_weights = 'groundtruth_weights' + num_groundtruth_boxes = 'num_groundtruth_boxes' + true_image_shape = 'true_image_shape' + multiclass_scores = 'multiclass_scores' + + class CenterNetCenterHeatmapTargetAssigner(object): + """Wrapper to compute the object center heatmap.""" + + def __init__(self, stride, min_overlap=0.7, compute_heatmap_sparse=False): + """Initializes the target assigner. + Args: + stride: int, the stride of the network in output pixels. + min_overlap: The minimum IOU overlap that boxes need to have to not be + penalized. + compute_heatmap_sparse: bool, indicating whether or not to use the sparse + version of the Op that computes the heatmap. The sparse version scales + better with number of classes, but in some cases is known to cause + OOM error. See (b/170989061). + """ + + self._stride = stride + self._min_overlap = min_overlap + self._compute_heatmap_sparse = compute_heatmap_sparse + + def assign_center_targets_from_boxes(self, + height, + width, + gt_boxes_list, + gt_classes_list, + gt_weights_list=None): + """Computes the object center heatmap target. + Args: + height: int, height of input to the model. This is used to + determine the height of the output. + width: int, width of the input to the model. This is used to + determine the width of the output. + gt_boxes_list: A list of float tensors with shape [num_boxes, 4] + representing the groundtruth detection bounding boxes for each sample in + the batch. The box coordinates are expected in normalized coordinates. + gt_classes_list: A list of float tensors with shape [num_boxes, + num_classes] representing the one-hot encoded class labels for each box + in the gt_boxes_list. + gt_weights_list: A list of float tensors with shape [num_boxes] + representing the weight of each groundtruth detection box. + Returns: + heatmap: A Tensor of size [batch_size, output_height, output_width, + num_classes] representing the per class center heatmap. output_height + and output_width are computed by dividing the input height and width by + the stride specified during initialization. + """ + + out_height = tf.cast(height // self._stride, tf.float32) + out_width = tf.cast(width // self._stride, tf.float32) + # Compute the yx-grid to be used to generate the heatmap. Each returned + # tensor has shape of [out_height, out_width] + (y_grid, x_grid) = ta_utils.image_shape_to_grids(out_height, out_width) + + heatmaps = [] + if gt_weights_list is None: + gt_weights_list = [None] * len(gt_boxes_list) + # TODO(vighneshb) Replace the for loop with a batch version. + for boxes, class_targets, weights in zip(gt_boxes_list, gt_classes_list, + gt_weights_list): + boxes = box_list.BoxList(boxes) + # Convert the box coordinates to absolute output image dimension space. + boxes = box_list_ops.to_absolute_coordinates(boxes, + height // self._stride, + width // self._stride) + # Get the box center coordinates. Each returned tensors have the shape of + # [num_instances] + (y_center, x_center, boxes_height, + boxes_width) = boxes.get_center_coordinates_and_sizes() + + # Compute the sigma from box size. The tensor shape: [num_instances]. + sigma = _compute_std_dev_from_box_size(boxes_height, boxes_width, + self._min_overlap) + # Apply the Gaussian kernel to the center coordinates. Returned heatmap + # has shape of [out_height, out_width, num_classes] + heatmap = ta_utils.coordinates_to_heatmap( + y_grid=y_grid, + x_grid=x_grid, + y_coordinates=y_center, + x_coordinates=x_center, + sigma=sigma, + channel_onehot=class_targets, + channel_weights=weights, + sparse=self._compute_heatmap_sparse) + heatmaps.append(heatmap) + + # Return the stacked heatmaps over the batch. + return tf.stack(heatmaps, axis=0) + +def _to_float32(x): + return tf.cast(x, tf.float32) + +def _flatten_spatial_dimensions(batch_images): + batch_size, height, width, channels = _get_shape(batch_images, 4) + return tf.reshape(batch_images, [batch_size, height * width, + channels]) + +def get_num_instances_from_weights(groundtruth_weights_list): + """Computes the number of instances/boxes from the weights in a batch. + Args: + groundtruth_weights_list: A list of float tensors with shape + [max_num_instances] representing whether there is an actual instance in + the image (with non-zero value) or is padded to match the + max_num_instances (with value 0.0). The list represents the batch + dimension. + Returns: + A scalar integer tensor incidating how many instances/boxes are in the + images in the batch. Note that this function is usually used to normalize + the loss so the minimum return value is 1 to avoid weird behavior. + """ + num_instances = tf.reduce_sum( + [tf.math.count_nonzero(w) for w in groundtruth_weights_list]) + num_instances = tf.maximum(num_instances, 1) + return num_instances + + From 1a7cae1391fa5fa66a9e100bdd78b360b58daae0 Mon Sep 17 00:00:00 2001 From: Jack LeCroy <3073035+jacklecroy@users.noreply.github.com> Date: Mon, 15 Feb 2021 15:24:50 -0500 Subject: [PATCH 027/132] Create tensors --- centernet/dataloaders/centernet_input.py | 28 +++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index cf4f07dec..c5ff143db 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -21,6 +21,17 @@ def _parse_train_data(self, decoded_tensors): images: the image tensor. labels: a dict of Tensors that contains labels. """ + tl_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) + br_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) + ct_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) + tl_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) + br_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) + ct_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) + tl_tags = tf.zeros((max_tag_len), dtype=tf.int64) + br_tags = tf.zeros((max_tag_len), dtype=tf.int64) + ct_tags = tf.zeros((max_tag_len), dtype=tf.int64) + tag_masks = tf.zeros((max_tag_len), dtype=tf.uint8) + # TODO: input size, output size image = decoded_tensors["image"] @@ -29,11 +40,15 @@ def _parse_train_data(self, decoded_tensors): for ind, detection in enumerate(decoded_tensors["groundtruth_boxes"]): category = int(detection[-1]) - 1 - #category = 0 + # category = 0 xtl, ytl = detection[0], detection[1] xbr, ybr = detection[2], detection[3] - xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. + + xct, yct = ( + (detection[2] + detection[0]) / 2, + (detection[3] + detection[1]) / 2 + ) fxtl = (xtl * width_ratio) fytl = (ytl * height_ratio) @@ -61,24 +76,21 @@ def _parse_train_data(self, decoded_tensors): radius = max(0, int(radius)) else: radius = gaussian_rad - # TODO: implement gaussian - # draw_gaussian(tl_heatmaps[b_ind, category], [xtl, ytl], radius) - # draw_gaussian(br_heatmaps[b_ind, category], [xbr, ybr], radius) - # draw_gaussian(ct_heatmaps[b_ind, category], [xct, yct], radius, delte=5) + draw_gaussian(tl_heatmaps[category], [xtl, ytl], radius) + draw_gaussian(br_heatmaps[category], [xbr, ybr], radius) + draw_gaussian(ct_heatmaps[category], [xct, yct], radius, delte=5) else: tl_heatmaps[category, ytl, xtl] = 1 br_heatmaps[category, ybr, xbr] = 1 ct_heatmaps[category, yct, xct] = 1 - tag_ind = tag_lens tl_regrs[tag_ind, :] = [fxtl - xtl, fytl - ytl] br_regrs[tag_ind, :] = [fxbr - xbr, fybr - ybr] ct_regrs[tag_ind, :] = [fxct - xct, fyct - yct] tl_tags[tag_ind] = ytl * output_size[1] + xtl br_tags[tag_ind] = ybr * output_size[1] + xbr ct_tags[tag_ind] = yct * output_size[1] + xct - tag_lens += 1 labels = { 'tl_tags': tl_tags, From 17fd9541887a7078e9eb6903cae1c0cd6a54655e Mon Sep 17 00:00:00 2001 From: anivegesana Date: Mon, 15 Feb 2021 17:53:07 -0500 Subject: [PATCH 028/132] Correct bug in Gaussian ground truth Co-Authored-By: Josh --- centernet/utils/groundtruth.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/centernet/utils/groundtruth.py b/centernet/utils/groundtruth.py index 55d1855a9..bec14122a 100644 --- a/centernet/utils/groundtruth.py +++ b/centernet/utils/groundtruth.py @@ -11,11 +11,11 @@ def _smallest_positive_root(a, b, c) -> tf.Tensor: root1 = (-b - discriminant) / (2 * a) root2 = (-b + discriminant) / (2 * a) - return tf.where(tf.less(root1, 0), root2, root1) + return (-b + discriminant) / (2) #tf.where(tf.less(root1, 0), root2, root1) def gaussian_radius(det_size, min_overlap=0.7) -> int: """ - Given a bounding box size, returns a lower bound on how far apart the + Given a bounding box size, returns a lower bound on how far apart the corners of another bounding box can lie while still maintaining the given minimum overlap, or IoU. Modified from implementation found in https://github.com/tensorflow/models/blob/master/research/object_detection/core/target_assigner.py. @@ -24,7 +24,7 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: det_size (tuple): tuple of integers representing height and width min_overlap (tf.float32): minimum IoU desired Returns: - int representing desired gaussian radius + int representing desired gaussian radius """ height, width = det_size @@ -32,32 +32,29 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: # contains the other. a1 = 1 - b1 = (height + width) + b1 = -(height + width) c1 = width * height * (1 - min_overlap) / (1 + min_overlap) - distance_detection_offset = _smallest_positive_root(a1, b1, c1) - r1 = (b1 + distance_detection_offset) / 2 + r1 = _smallest_positive_root(a1, b1, c1) # Case where detection is smaller than ground truth and completely contained # in it. a2 = 4 - b2 = 2 * (height + width) + b2 = -2 * (height + width) c2 = (1 - min_overlap) * width * height - distance_detection_in_gt = _smallest_positive_root(a2, b2, c2) - r2 = (b2 + distance_detection_in_gt) / 2 + r2 = _smallest_positive_root(a2, b2, c2) # Case where ground truth is smaller than detection and completely contained # in it. a3 = 4 * min_overlap - b3 = -2 * min_overlap * (height + width) + b3 = 2 * min_overlap * (height + width) c3 = (min_overlap - 1) * width * height - distance_gt_in_detection = _smallest_positive_root(a3, b3, c3) - r3 = (b3 + distance_gt_in_detection) / 2 + r3 = _smallest_positive_root(a3, b3, c3) # TODO discuss whether to return scalar or tensor # return tf.reduce_min([r1, r2, r3], axis=0) - return min(r1, r2, r3) + return tf.reduce_min([r1, r2, r3], axis=0) def gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: """ @@ -87,11 +84,11 @@ def draw_gaussian(heatmap, center, radius, k=1): diameter = 2 * radius + 1 gaussian = gaussian_penalty(radius) - + x, y = center height, width = heatmap.shape[0:2] - + left, right = min(x, radius), min(width - x, radius + 1) top, bottom = min(y, radius), min(height - y, radius + 1) From 9e5ff89074c6c4794e2530670e3e86f3329c4851 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 16 Feb 2021 12:06:37 -0500 Subject: [PATCH 029/132] bias initialization + more tests --- centernet/modeling/heads/centernet_head.py | 81 ++++++++++--------- .../modeling/heads/centernet_head_test.py | 23 +++++- centernet/modeling/layers/nn_blocks.py | 8 +- 3 files changed, 70 insertions(+), 42 deletions(-) diff --git a/centernet/modeling/heads/centernet_head.py b/centernet/modeling/heads/centernet_head.py index 34f931140..1a733934c 100644 --- a/centernet/modeling/heads/centernet_head.py +++ b/centernet/modeling/heads/centernet_head.py @@ -1,61 +1,64 @@ import tensorflow as tf from centernet.modeling.layers.nn_blocks import CenterNetHeadConv +""" + # OLD: use to reference task_outputs dictionary, maybe can go in config + # class later + task_output_specs = { + "2D" : {'heatmap': 91, + 'local_offset': 2, + 'object_size': 2 + }, + "3D": {'heatmap': 91, + 'local_offset': 2, + 'object_size': 3, + 'depth': 1, + 'orientation': 8 + }, + "pose": {'heatmap': 91, + 'joint_locs': 17 * 2, + 'joint_heatmap': 17, + 'joint_offset': 2 + } + } +""" + class CenterNetHead(tf.keras.layers.Layer): """ CenterNet Head """ def __init__(self, - classes: int = 91, - joints: int = 17, - task: str = "2D", + task_outputs: dict, + heatmap_bias: float = -2.19, **kwargs): """ Args: - classes: int, number of possible class predictions for the network - joints: int, number of possible joint location predictions for pose - estimation - task: string, indicating the prediction task. Valid values are - "2D", "3D", and "pose". + task_outputs: dict, with key-value pairs denoting the names of the outputs + and the desired channel depth of each output + heatmap_bias: float, constant value to initialize the convolution layer + bias vector if it is responsible for generating a heatmap (not for + regressed predictions) call Returns: - dictionary where the keys-value pairs are the output names and output - tensors + dictionary where the keys-value pairs denote the names of the output + and the respective output tensor """ - self._classes = classes - self._joints = joints - self._task = task - - # These specify the layers required for each type of task - # Each spec is a dictionary of the output name and its - # respective channel depth - task_output_specs = { - "2D" : {'heatmaps': self._classes, - 'local_offset': 2, - 'object_size': 2 - }, - "3D": {'heatmaps': self._classes, - 'local_offset': 2, - 'object_size': 3, - 'depth': 1, - 'orientation': 8 - }, - "pose": {'heatmaps': self._classes, - 'joint_locs': self._joints * 2, - 'joint_heatmap': self._joints, - 'joint_offset': 2 - } - } - - self.task_outputs = task_output_specs[self._task] + + self._task_outputs = task_outputs + self._heatmap_bias = heatmap_bias super().__init__(**kwargs) def build(self, input_shape): self.layers = {} - for key in self.task_outputs: - num_filters = self.task_outputs[key] - self.layers[key] = CenterNetHeadConv(output_filters=num_filters, name=key) + for key in self._task_outputs: + num_filters = self._task_outputs[key] + bias = 0 + if key is "heatmap": + bias = self._heatmap_bias + + self.layers[key] = CenterNetHeadConv(output_filters=num_filters, + name=key, bias_init=bias) super().build(input_shape) diff --git a/centernet/modeling/heads/centernet_head_test.py b/centernet/modeling/heads/centernet_head_test.py index 3d564d8a2..2b6758557 100644 --- a/centernet/modeling/heads/centernet_head_test.py +++ b/centernet/modeling/heads/centernet_head_test.py @@ -8,13 +8,32 @@ class CenterNetHeadTest(tf.test.TestCase, parameterized.TestCase): def test_head(self): - head = centernet_head.CenterNetHead(classes=91, task="2D") + task_outputs = {'heatmap': 91, + 'local_offset': 2, + 'object_size': 2 + } + head = centernet_head.CenterNetHead(task_outputs=task_outputs, + heatmap_bias=-2.19) + + # Output shape tests outputs = head(np.zeros((2, 128, 128, 256), dtype=np.float32)) self.assertEqual(len(outputs), 3) - self.assertEqual(outputs['heatmaps'].shape, (2, 128, 128, 91)) + self.assertEqual(outputs['heatmap'].shape, (2, 128, 128, 91)) self.assertEqual(outputs['local_offset'].shape, (2, 128, 128, 2)) self.assertEqual(outputs['object_size'].shape, (2, 128, 128, 2)) + # Weight initialization tests + hm_bias_vector = np.asarray(head.layers['heatmap'].weights[-1]) + off_bias_vector = np.asarray(head.layers['local_offset'].weights[-1]) + size_bias_vector = np.asarray(head.layers['object_size'].weights[-1]) + + self.assertArrayNear(hm_bias_vector, + np.repeat(-2.19, repeats=91), err=1.00e-6) + self.assertArrayNear(off_bias_vector, + np.repeat(0, repeats=2), err=1.00e-6) + self.assertArrayNear(size_bias_vector, + np.repeat(0, repeats=2), err=1.00e-6) + if __name__ == '__main__': tf.test.main() diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index b712fff00..5a9a7d136 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -107,14 +107,18 @@ class CenterNetHeadConv(tf.keras.layers.Layer): """ def __init__(self, output_filters: int, + bias_init : float, name: str, **kwargs): """ Args: output_filters: int, channel depth of layer output + bias_init: float, value to initialize the bias vector for the final + convolution layer name: string, layer name """ self._output_filters = output_filters + self._bias_init = bias_init super().__init__(name=name, **kwargs) def build(self, input_shape): @@ -125,8 +129,10 @@ def build(self, input_shape): self.relu = tf.keras.layers.ReLU() + # Initialize bias to the last Conv2D Layer self.conv2 = tf.keras.layers.Conv2D(filters=self._output_filters, - kernel_size=(1, 1), padding='valid') + kernel_size=(1, 1), + bias_initializer=tf.constant_initializer(self._bias_init)) def call(self, x): x = self.conv1(x) From c74e2dfad3e9f68204d1b06865b28b2ff4b397d8 Mon Sep 17 00:00:00 2001 From: patel996 Date: Tue, 16 Feb 2021 12:27:02 -0500 Subject: [PATCH 030/132] implement penalty reduced local loss --- centernet/tasks/centernet_object_detection.py | 15 +- centernet/tasks/loss_utils.py | 221 +----------------- 2 files changed, 16 insertions(+), 220 deletions(-) diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py index 6788ee578..8c52d1d89 100644 --- a/centernet/tasks/centernet_object_detection.py +++ b/centernet/tasks/centernet_object_detection.py @@ -7,7 +7,7 @@ from official.core import task_factory from official.vision.beta.evaluation import coco_evaluator - +import loss_utils as utils # @task_factory.register_task_cls(exp_cfg.YoloTask) class CenterNetObjectDetectionTask(base_task.Task): @@ -20,5 +20,18 @@ def build_losses(self, outputs, labels, aux_losses=None): metric_dict = dict() # TODO: Calculate loss + flattened_ct_heatmaps = utils._flatten_spatial_dimensions(labels['ct_heatmaps']) + num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) + + object_center_loss = self._center_params.classification_loss + # Loop through each feature output head. + for pred in outputs['ct_heatmaps']: + pred = utils._flatten_spatial_dimensions(pred) + total_loss = object_center_loss( + pred, flattened_ct_heatmaps) #removed weight parameter (weight = per_pixel_weight) + loss += tf.reduce_sum(total_loss) / ( + float(len(total_loss)) * num_boxes) return loss, metric_dict + + \ No newline at end of file diff --git a/centernet/tasks/loss_utils.py b/centernet/tasks/loss_utils.py index be06f70f3..d27fc4449 100644 --- a/centernet/tasks/loss_utils.py +++ b/centernet/tasks/loss_utils.py @@ -1,221 +1,4 @@ -class BoxListFields(object): - """Naming conventions for BoxLists. - Attributes: - boxes: bounding box coordinates. - classes: classes per bounding box. - scores: scores per bounding box. - weights: sample weights per bounding box. - objectness: objectness score per bounding box. - masks: masks per bounding box. - boundaries: boundaries per bounding box. - keypoints: keypoints per bounding box. - keypoint_heatmaps: keypoint heatmaps per bounding box. - is_crowd: is_crowd annotation per bounding box. - """ - boxes = 'boxes' - classes = 'classes' - scores = 'scores' - weights = 'weights' - objectness = 'objectness' - masks = 'masks' - boundaries = 'boundaries' - keypoints = 'keypoints' - keypoint_heatmaps = 'keypoint_heatmaps' - is_crowd = 'is_crowd' - - -class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)): - """Abstract base class for detection models. - Extends tf.Module to guarantee variable tracking. - """ - - def __init__(self, num_classes): - """Constructor. - Args: - num_classes: number of classes. Note that num_classes *does not* include - background categories that might be implicitly predicted in various - implementations. - """ - self._num_classes = num_classes - self._groundtruth_lists = {} - - super(DetectionModel, self).__init__() - - @property - def num_classes(self): - return self._num_classes - - def groundtruth_lists(self, field): - """Access list of groundtruth tensors. - Args: - field: a string key, options are - fields.BoxListFields.{boxes,classes,masks,keypoints, - keypoint_visibilities, densepose_*, track_ids, - temporal_offsets, track_match_flags} - fields.InputDataFields.is_annotated. - Returns: - a list of tensors holding groundtruth information (see also - provide_groundtruth function below), with one entry for each image in the - batch. - Raises: - RuntimeError: if the field has not been provided via provide_groundtruth. - """ - if field not in self._groundtruth_lists: - raise RuntimeError('Groundtruth tensor {} has not been provided'.format( - field)) - return self._groundtruth_lists[field] - -class InputDataFields(object): - """Names for the input tensors. - Holds the standard data field names to use for identifying input tensors. This - should be used by the decoder to identify keys for the returned tensor_dict - containing input tensors. And it should be used by the model to identify the - tensors it needs. - Attributes: - image: image. - image_additional_channels: additional channels. - original_image: image in the original input size. - key: unique key corresponding to image. - source_id: source of the original image. - filename: original filename of the dataset (without common path). - groundtruth_image_classes: image-level class labels. - groundtruth_boxes: coordinates of the ground truth boxes in the image. - groundtruth_classes: box-level class labels. - groundtruth_label_types: box-level label types (e.g. explicit negative). - groundtruth_is_crowd: [DEPRECATED, use groundtruth_group_of instead] - is the groundtruth a single object or a crowd. - groundtruth_area: area of a groundtruth segment. - groundtruth_difficult: is a `difficult` object - groundtruth_group_of: is a `group_of` objects, e.g. multiple objects of the - same class, forming a connected group, where instances are heavily - occluding each other. - proposal_boxes: coordinates of object proposal boxes. - proposal_objectness: objectness score of each proposal. - groundtruth_instance_masks: ground truth instance masks. - groundtruth_instance_boundaries: ground truth instance boundaries. - groundtruth_instance_classes: instance mask-level class labels. - groundtruth_keypoints: ground truth keypoints. - groundtruth_keypoint_visibilities: ground truth keypoint visibilities. - groundtruth_label_scores: groundtruth label scores. - groundtruth_weights: groundtruth weight factor for bounding boxes. - num_groundtruth_boxes: number of groundtruth boxes. - true_image_shapes: true shapes of images in the resized images, as resized - images can be padded with zeros. - multiclass_scores: the label score per class for each box. - """ - image = 'image' - image_additional_channels = 'image_additional_channels' - original_image = 'original_image' - key = 'key' - source_id = 'source_id' - filename = 'filename' - groundtruth_image_classes = 'groundtruth_image_classes' - groundtruth_boxes = 'groundtruth_boxes' - groundtruth_classes = 'groundtruth_classes' - groundtruth_label_types = 'groundtruth_label_types' - groundtruth_is_crowd = 'groundtruth_is_crowd' - groundtruth_area = 'groundtruth_area' - groundtruth_difficult = 'groundtruth_difficult' - groundtruth_group_of = 'groundtruth_group_of' - proposal_boxes = 'proposal_boxes' - proposal_objectness = 'proposal_objectness' - groundtruth_instance_masks = 'groundtruth_instance_masks' - groundtruth_instance_boundaries = 'groundtruth_instance_boundaries' - groundtruth_instance_classes = 'groundtruth_instance_classes' - groundtruth_keypoints = 'groundtruth_keypoints' - groundtruth_keypoint_visibilities = 'groundtruth_keypoint_visibilities' - groundtruth_label_scores = 'groundtruth_label_scores' - groundtruth_weights = 'groundtruth_weights' - num_groundtruth_boxes = 'num_groundtruth_boxes' - true_image_shape = 'true_image_shape' - multiclass_scores = 'multiclass_scores' - - class CenterNetCenterHeatmapTargetAssigner(object): - """Wrapper to compute the object center heatmap.""" - - def __init__(self, stride, min_overlap=0.7, compute_heatmap_sparse=False): - """Initializes the target assigner. - Args: - stride: int, the stride of the network in output pixels. - min_overlap: The minimum IOU overlap that boxes need to have to not be - penalized. - compute_heatmap_sparse: bool, indicating whether or not to use the sparse - version of the Op that computes the heatmap. The sparse version scales - better with number of classes, but in some cases is known to cause - OOM error. See (b/170989061). - """ - - self._stride = stride - self._min_overlap = min_overlap - self._compute_heatmap_sparse = compute_heatmap_sparse - - def assign_center_targets_from_boxes(self, - height, - width, - gt_boxes_list, - gt_classes_list, - gt_weights_list=None): - """Computes the object center heatmap target. - Args: - height: int, height of input to the model. This is used to - determine the height of the output. - width: int, width of the input to the model. This is used to - determine the width of the output. - gt_boxes_list: A list of float tensors with shape [num_boxes, 4] - representing the groundtruth detection bounding boxes for each sample in - the batch. The box coordinates are expected in normalized coordinates. - gt_classes_list: A list of float tensors with shape [num_boxes, - num_classes] representing the one-hot encoded class labels for each box - in the gt_boxes_list. - gt_weights_list: A list of float tensors with shape [num_boxes] - representing the weight of each groundtruth detection box. - Returns: - heatmap: A Tensor of size [batch_size, output_height, output_width, - num_classes] representing the per class center heatmap. output_height - and output_width are computed by dividing the input height and width by - the stride specified during initialization. - """ - - out_height = tf.cast(height // self._stride, tf.float32) - out_width = tf.cast(width // self._stride, tf.float32) - # Compute the yx-grid to be used to generate the heatmap. Each returned - # tensor has shape of [out_height, out_width] - (y_grid, x_grid) = ta_utils.image_shape_to_grids(out_height, out_width) - - heatmaps = [] - if gt_weights_list is None: - gt_weights_list = [None] * len(gt_boxes_list) - # TODO(vighneshb) Replace the for loop with a batch version. - for boxes, class_targets, weights in zip(gt_boxes_list, gt_classes_list, - gt_weights_list): - boxes = box_list.BoxList(boxes) - # Convert the box coordinates to absolute output image dimension space. - boxes = box_list_ops.to_absolute_coordinates(boxes, - height // self._stride, - width // self._stride) - # Get the box center coordinates. Each returned tensors have the shape of - # [num_instances] - (y_center, x_center, boxes_height, - boxes_width) = boxes.get_center_coordinates_and_sizes() - - # Compute the sigma from box size. The tensor shape: [num_instances]. - sigma = _compute_std_dev_from_box_size(boxes_height, boxes_width, - self._min_overlap) - # Apply the Gaussian kernel to the center coordinates. Returned heatmap - # has shape of [out_height, out_width, num_classes] - heatmap = ta_utils.coordinates_to_heatmap( - y_grid=y_grid, - x_grid=x_grid, - y_coordinates=y_center, - x_coordinates=x_center, - sigma=sigma, - channel_onehot=class_targets, - channel_weights=weights, - sparse=self._compute_heatmap_sparse) - heatmaps.append(heatmap) - - # Return the stacked heatmaps over the batch. - return tf.stack(heatmaps, axis=0) +import tensorflow as tf def _to_float32(x): return tf.cast(x, tf.float32) @@ -243,4 +26,4 @@ def get_num_instances_from_weights(groundtruth_weights_list): num_instances = tf.maximum(num_instances, 1) return num_instances - + From 761d65e24aacb1ccfc1640a250906b92b5412cb9 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 16 Feb 2021 12:27:28 -0500 Subject: [PATCH 031/132] moved code to decoder directory --- .../centernet_decoder.py} | 10 +++++----- .../centernet_decoder_test.py} | 6 +++--- centernet/modeling/layers/nn_blocks.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) rename centernet/modeling/{heads/centernet_head.py => decoders/centernet_decoder.py} (86%) rename centernet/modeling/{heads/centernet_head_test.py => decoders/centernet_decoder_test.py} (84%) diff --git a/centernet/modeling/heads/centernet_head.py b/centernet/modeling/decoders/centernet_decoder.py similarity index 86% rename from centernet/modeling/heads/centernet_head.py rename to centernet/modeling/decoders/centernet_decoder.py index 1a733934c..fe171660b 100644 --- a/centernet/modeling/heads/centernet_head.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -1,5 +1,5 @@ import tensorflow as tf -from centernet.modeling.layers.nn_blocks import CenterNetHeadConv +from centernet.modeling.layers.nn_blocks import CenterNetDecoderConv """ # OLD: use to reference task_outputs dictionary, maybe can go in config @@ -23,9 +23,9 @@ } """ -class CenterNetHead(tf.keras.layers.Layer): +class CenterNetDecoder(tf.keras.layers.Layer): """ - CenterNet Head + CenterNet Decoder """ def __init__(self, task_outputs: dict, @@ -54,10 +54,10 @@ def build(self, input_shape): for key in self._task_outputs: num_filters = self._task_outputs[key] bias = 0 - if key is "heatmap": + if key == "heatmap": bias = self._heatmap_bias - self.layers[key] = CenterNetHeadConv(output_filters=num_filters, + self.layers[key] = CenterNetDecoderConv(output_filters=num_filters, name=key, bias_init=bias) super().build(input_shape) diff --git a/centernet/modeling/heads/centernet_head_test.py b/centernet/modeling/decoders/centernet_decoder_test.py similarity index 84% rename from centernet/modeling/heads/centernet_head_test.py rename to centernet/modeling/decoders/centernet_decoder_test.py index 2b6758557..a77bbecca 100644 --- a/centernet/modeling/heads/centernet_head_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -2,10 +2,10 @@ import tensorflow as tf import numpy as np -from centernet.modeling.heads import centernet_head +from centernet.modeling.decoders import centernet_decoder -class CenterNetHeadTest(tf.test.TestCase, parameterized.TestCase): +class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_head(self): task_outputs = {'heatmap': 91, @@ -13,7 +13,7 @@ def test_head(self): 'object_size': 2 } - head = centernet_head.CenterNetHead(task_outputs=task_outputs, + head = centernet_decoder.CenterNetDecoder(task_outputs=task_outputs, heatmap_bias=-2.19) # Output shape tests diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 5a9a7d136..10e80211b 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -99,7 +99,7 @@ def get_config(self): layer_config.update(super().get_config()) return layer_config -class CenterNetHeadConv(tf.keras.layers.Layer): +class CenterNetDecoderConv(tf.keras.layers.Layer): """ Convolution block for the CenterNet head. This is used to generate both the confidence heatmaps and other regressed predictions such as From c78fe9c4a81c98f2e4ed18a0e452f53233559e1b Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 16 Feb 2021 14:34:06 -0500 Subject: [PATCH 032/132] added config --- centernet/configs/centernet.py | 53 +++++++++++++++++++ .../modeling/decoders/centernet_decoder.py | 50 ++++++++--------- .../decoders/centernet_decoder_test.py | 27 +++++----- 3 files changed, 88 insertions(+), 42 deletions(-) create mode 100644 centernet/configs/centernet.py diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py new file mode 100644 index 000000000..949cc4669 --- /dev/null +++ b/centernet/configs/centernet.py @@ -0,0 +1,53 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Decoder configurations.""" +from typing import Dict + +# Import libraries +import dataclasses + +from official.modeling import hyperparams + +# Note these do not subclass from hyperparams.Config, +# for some reason Dicts are not supported in their +# immutable types + +@dataclasses.dataclass +class CenterNet2D(): + """CenterNet for 2D Object Detection Decoder.""" + task_outputs: Dict[str, int] = dataclasses.field( + default_factory=lambda: {'heatmap': 91, 'local_offset': 2, 'object_size': 2}) + heatmap_bias: float = -2.19 + +@dataclasses.dataclass +class CenterNet3D(): + """CenterNet for 3D Object Detection Decoder.""" + task_outputs: Dict[str, int] = dataclasses.field( + default_factory=lambda: {'heatmap': 91, + 'local_offset': 2, + 'object_size': 3, + 'depth' : 1, + 'orientation': 8}) + heatmap_bias: float = -2.19 + +@dataclasses.dataclass +class CenterNetPose(): + """CenterNet for Pose Estimation Decoder.""" + task_outputs: Dict[str, int] = dataclasses.field( + default_factory=lambda: {'heatmap': 17, + 'joint_locs': 17 * 2, + 'joint_offset': 2}) + heatmap_bias: float = -2.19 \ No newline at end of file diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index fe171660b..ed4acde33 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -1,29 +1,7 @@ import tensorflow as tf from centernet.modeling.layers.nn_blocks import CenterNetDecoderConv -""" - # OLD: use to reference task_outputs dictionary, maybe can go in config - # class later - task_output_specs = { - "2D" : {'heatmap': 91, - 'local_offset': 2, - 'object_size': 2 - }, - "3D": {'heatmap': 91, - 'local_offset': 2, - 'object_size': 3, - 'depth': 1, - 'orientation': 8 - }, - "pose": {'heatmap': 91, - 'joint_locs': 17 * 2, - 'joint_heatmap': 17, - 'joint_offset': 2 - } - } -""" - -class CenterNetDecoder(tf.keras.layers.Layer): +class CenterNetDecoder(tf.keras.Model): """ CenterNet Decoder """ @@ -50,21 +28,35 @@ def __init__(self, super().__init__(**kwargs) def build(self, input_shape): - self.layers = {} + self.out_layers = {} for key in self._task_outputs: num_filters = self._task_outputs[key] bias = 0 - if key == "heatmap": + if key == 'heatmap': bias = self._heatmap_bias - self.layers[key] = CenterNetDecoderConv(output_filters=num_filters, + self.out_layers[key] = CenterNetDecoderConv(output_filters=num_filters, name=key, bias_init=bias) super().build(input_shape) def call(self, x): outputs = {} - for key in self.layers: - outputs[key] = self.layers[key](x) + for key in self.out_layers: + outputs[key] = self.out_layers[key](x) + + return outputs + + def get_config(self): + layer_config = { + 'task_outputs': self._task_outputs, + 'heatmap_bias': self._heatmap_bias + } + + #layer_config.update(super().get_config()) + return layer_config - return outputs \ No newline at end of file +def build_centernet_decoder(model_config) -> tf.keras.Model: + return CenterNetDecoder( + task_outputs=model_config.task_outputs, + heatmap_bias=model_config.heatmap_bias) \ No newline at end of file diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index a77bbecca..40f1fac13 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -2,38 +2,39 @@ import tensorflow as tf import numpy as np +from centernet.configs import centernet as cfg from centernet.modeling.decoders import centernet_decoder class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): - def test_head(self): - task_outputs = {'heatmap': 91, - 'local_offset': 2, - 'object_size': 2 - } + def test_create_decoder(self): + decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNet2D()) + config = decoder.get_config() + self.assertEqual(len(config), 2) + self.assertEqual(config['heatmap_bias'], -2.19) - head = centernet_decoder.CenterNetDecoder(task_outputs=task_outputs, - heatmap_bias=-2.19) + def test_decoder_shape(self): + decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNet2D()) # Output shape tests - outputs = head(np.zeros((2, 128, 128, 256), dtype=np.float32)) + outputs = decoder(np.zeros((2, 128, 128, 256), dtype=np.float32)) self.assertEqual(len(outputs), 3) self.assertEqual(outputs['heatmap'].shape, (2, 128, 128, 91)) self.assertEqual(outputs['local_offset'].shape, (2, 128, 128, 2)) self.assertEqual(outputs['object_size'].shape, (2, 128, 128, 2)) # Weight initialization tests - hm_bias_vector = np.asarray(head.layers['heatmap'].weights[-1]) - off_bias_vector = np.asarray(head.layers['local_offset'].weights[-1]) - size_bias_vector = np.asarray(head.layers['object_size'].weights[-1]) + hm_bias_vector = np.asarray(decoder.out_layers['heatmap'].weights[-1]) + off_bias_vector = np.asarray(decoder.out_layers['local_offset'].weights[-1]) + size_bias_vector = np.asarray(decoder.out_layers['object_size'].weights[-1]) self.assertArrayNear(hm_bias_vector, np.repeat(-2.19, repeats=91), err=1.00e-6) self.assertArrayNear(off_bias_vector, np.repeat(0, repeats=2), err=1.00e-6) self.assertArrayNear(size_bias_vector, - np.repeat(0, repeats=2), err=1.00e-6) + np.repeat(0, repeats=2), err=1.00e-6) if __name__ == '__main__': - tf.test.main() + tf.test.main() \ No newline at end of file From 64e73f4d724681f62e9425d4758ecafc9dac936e Mon Sep 17 00:00:00 2001 From: patel996 Date: Tue, 16 Feb 2021 16:47:56 -0500 Subject: [PATCH 033/132] implement l1 localization loss --- centernet/tasks/centernet_object_detection.py | 42 +++++++++++++++---- centernet/tasks/loss_utils.py | 16 +++++++ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py index 8c52d1d89..0827d45d0 100644 --- a/centernet/tasks/centernet_object_detection.py +++ b/centernet/tasks/centernet_object_detection.py @@ -8,6 +8,8 @@ from official.vision.beta.evaluation import coco_evaluator import loss_utils as utils +from losses import penalty_reduced_logistic_focal_loss +from losses import l1_localization_loss # @task_factory.register_task_cls(exp_cfg.YoloTask) class CenterNetObjectDetectionTask(base_task.Task): @@ -16,22 +18,46 @@ def __init__(self, params, logging_dir: str = None): super().__init__(params, logging_dir) def build_losses(self, outputs, labels, aux_losses=None): + total_loss = 0.0 + total_scale_loss = 0.0 + total_offset_loss = 0.0 loss = 0.0 + scale_loss = 0.0 + offset_loss = 0.0 + metric_dict = dict() # TODO: Calculate loss flattened_ct_heatmaps = utils._flatten_spatial_dimensions(labels['ct_heatmaps']) - num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) + num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) #gt_weights_list here shouldn't be tag_masks here - object_center_loss = self._center_params.classification_loss + object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss() # Loop through each feature output head. for pred in outputs['ct_heatmaps']: pred = utils._flatten_spatial_dimensions(pred) - total_loss = object_center_loss( - pred, flattened_ct_heatmaps) #removed weight parameter (weight = per_pixel_weight) - loss += tf.reduce_sum(total_loss) / ( - float(len(total_loss)) * num_boxes) - - return loss, metric_dict + total_loss += object_center_loss( + flattened_ct_heatmaps, pred) #removed weight parameter (weight = per_pixel_weight) + loss += tf.reduce_sum(total_loss) / ( + float(len(outputs['ct_heatmaps'])) * num_boxes) + + #localization loss for offset and scale loss + localization_loss_fn = l1_localization_loss.L1LocalizationLoss() + for scale_pred, offset_pred in zip(outputs['ct_size'], outputs['ct_offset']): + # Compute the scale loss. + scale_pred = utils.get_batch_predictions_from_indices( + scale_pred, labels['tag_masks']) + total_scale_loss += localization_loss_fn( + labels['ct_size'], scale_pred) #removed weights=batch_weights + # Compute the offset loss. + offset_pred = utils.get_batch_predictions_from_indices( + offset_pred, labels['tag_masks']) + total_offset_loss += localization_loss_fn( + labels['ct_offset'], offset_pred) #removed weights=batch_weights + scale_loss += tf.reduce_sum(total_scale_loss) / ( + float(len(outputs['ct_size'])) * num_boxes) + offset_loss += tf.reduce_sum(total_offset_loss) / ( + float(len(outputs['ct_size'])) * num_boxes) + + return loss, scale_loss, offset_loss, metric_dict \ No newline at end of file diff --git a/centernet/tasks/loss_utils.py b/centernet/tasks/loss_utils.py index d27fc4449..a50ae8cd2 100644 --- a/centernet/tasks/loss_utils.py +++ b/centernet/tasks/loss_utils.py @@ -26,4 +26,20 @@ def get_num_instances_from_weights(groundtruth_weights_list): num_instances = tf.maximum(num_instances, 1) return num_instances +def get_batch_predictions_from_indices(batch_predictions, indices): + """Gets the values of predictions in a batch at the given indices. + The indices are expected to come from the offset targets generation functions + in this library. The returned value is intended to be used inside a loss + function. + Args: + batch_predictions: A tensor of shape [batch_size, height, width, channels] + or [batch_size, height, width, class, channels] for class-specific + features (e.g. keypoint joint offsets). + indices: A tensor of shape [num_instances, 3] for single class features or + [num_instances, 4] for multiple classes features. + Returns: + values: A tensor of shape [num_instances, channels] holding the predicted + values at the given indices. + """ + return tf.gather_nd(batch_predictions, indices) From 9882cf98275d642d94f2667fe8a630924392905e Mon Sep 17 00:00:00 2001 From: anivegesana Date: Tue, 16 Feb 2021 18:47:38 -0500 Subject: [PATCH 034/132] New config format --- centernet/configs/centernet.py | 92 ++++++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 949cc4669..dd4858553 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -20,13 +20,19 @@ import dataclasses from official.modeling import hyperparams +from official.modeling.hyperparams import config_definitions as cfg +from official.vision.beta.configs import common +from official.vision.beta.configs import backbones -# Note these do not subclass from hyperparams.Config, +from centernet.configs import backbones + +# Note these do not subclass from hyperparams.Config, # for some reason Dicts are not supported in their # immutable types + @dataclasses.dataclass -class CenterNet2D(): +class CenterNetDecoder(hyperparams.Config): """CenterNet for 2D Object Detection Decoder.""" task_outputs: Dict[str, int] = dataclasses.field( default_factory=lambda: {'heatmap': 91, 'local_offset': 2, 'object_size': 2}) @@ -37,7 +43,7 @@ class CenterNet3D(): """CenterNet for 3D Object Detection Decoder.""" task_outputs: Dict[str, int] = dataclasses.field( default_factory=lambda: {'heatmap': 91, - 'local_offset': 2, + 'local_offset': 2, 'object_size': 3, 'depth' : 1, 'orientation': 8}) @@ -48,6 +54,82 @@ class CenterNetPose(): """CenterNet for Pose Estimation Decoder.""" task_outputs: Dict[str, int] = dataclasses.field( default_factory=lambda: {'heatmap': 17, - 'joint_locs': 17 * 2, + 'joint_locs': 17 * 2, 'joint_offset': 2}) - heatmap_bias: float = -2.19 \ No newline at end of file + heatmap_bias: float = -2.19 + + +@dataclasses.dataclass +class CenterNetDecoder(hyperparams.Config): + heatmap_bias: float = -2.19 + + +@dataclasses.dataclass +class CenterNet(hyperparams.Config): + num_classes: int = 80 + decoder: CenterNetDecoder = CenterNetDecoder() + + +@dataclasses.dataclass +class CenterNetDetection(cfg.TaskConfig): + use_centers: bool = True + use_corners: bool = False + predict_3d: bool = False + + +@dataclasses.dataclass +class CenterNetSubTasks(cfg.TaskConfig): + detection: CenterNetDetection = CenterNetDetection() + # kp_detection: bool = False + segmentation: bool = False + # pose: bool = False + # reid: bool = False + # temporal: bool = False + + +@dataclasses.dataclass +class CenterNetTask(cfg.TaskConfig): + model: CenterNet = CenterNet() + subtasks: CenterNetSubTasks = CenterNetSubTasks() + def _get_output_length_dict(self): + lengths = {} + assert self.subtasks.detection is not None or self.subtasks.kp_detection \ + or self.subtasks.segmentation, "You must specify at least one " \ + "subtask to CenterNet" + + if self.subtasks.detection: + assert self.subtasks.detection.use_centers or \ + self.subtasks.detection.use_corners, "Cannot use CenterNet without " \ + "heatmaps" + if self.subtasks.detection.use_centers: + lengths.update({ + 'ct_heatmaps': self.model.num_classes, # renamed from heatmap + 'ct_offset': 2, # renamed from local_offset + 'ct_size': 2 # renamed from object_size + }) + if self.subtasks.detection.use_corners: + lengths.update({ + 'tl_heatmaps': self.model.num_classes, + 'tl_offset': 2, + 'tl_size': 2, + 'br_heatmaps': self.model.num_classes, + 'br_offset': 2, + 'br_size': 2, + }) + if self.subtasks.detection.predict_3d: + lengths.update({ + 'depth': 1, + 'orientation': 8 + }) + + if self.subtasks.segmentation: + lengths['seg_heatmaps'] = self.model.num_classes + + # if self.subtasks.pose: + # lengths.update({ + # 'pose_heatmaps': 17, + # 'joint_locs': 17 * 2, + # 'joint_offset': 2 + # }) + + return lengths From bed46cf8e1a891409e2bb06984d3f76c7ae13632 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 16 Feb 2021 19:34:22 -0500 Subject: [PATCH 035/132] updated build and test for new cfg format --- .../modeling/decoders/centernet_decoder.py | 13 ++++++++++--- .../decoders/centernet_decoder_test.py | 18 +++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index ed4acde33..094eb7663 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -32,7 +32,7 @@ def build(self, input_shape): for key in self._task_outputs: num_filters = self._task_outputs[key] bias = 0 - if key == 'heatmap': + if 'heatmaps' in key: bias = self._heatmap_bias self.out_layers[key] = CenterNetDecoderConv(output_filters=num_filters, @@ -57,6 +57,13 @@ def get_config(self): return layer_config def build_centernet_decoder(model_config) -> tf.keras.Model: + cfg = model_config() + + task_outputs = cfg._get_output_length_dict() + heatmap_bias = cfg.model.decoder.heatmap_bias + + print(task_outputs) + return CenterNetDecoder( - task_outputs=model_config.task_outputs, - heatmap_bias=model_config.heatmap_bias) \ No newline at end of file + task_outputs=task_outputs, + heatmap_bias=heatmap_bias) \ No newline at end of file diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index 40f1fac13..65cdbc213 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -9,28 +9,28 @@ class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_create_decoder(self): - decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNet2D()) + decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNetTask) config = decoder.get_config() self.assertEqual(len(config), 2) self.assertEqual(config['heatmap_bias'], -2.19) def test_decoder_shape(self): - decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNet2D()) + decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNetTask) # Output shape tests outputs = decoder(np.zeros((2, 128, 128, 256), dtype=np.float32)) self.assertEqual(len(outputs), 3) - self.assertEqual(outputs['heatmap'].shape, (2, 128, 128, 91)) - self.assertEqual(outputs['local_offset'].shape, (2, 128, 128, 2)) - self.assertEqual(outputs['object_size'].shape, (2, 128, 128, 2)) + self.assertEqual(outputs['ct_heatmaps'].shape, (2, 128, 128, 80)) + self.assertEqual(outputs['ct_offset'].shape, (2, 128, 128, 2)) + self.assertEqual(outputs['ct_size'].shape, (2, 128, 128, 2)) # Weight initialization tests - hm_bias_vector = np.asarray(decoder.out_layers['heatmap'].weights[-1]) - off_bias_vector = np.asarray(decoder.out_layers['local_offset'].weights[-1]) - size_bias_vector = np.asarray(decoder.out_layers['object_size'].weights[-1]) + hm_bias_vector = np.asarray(decoder.out_layers['ct_heatmaps'].weights[-1]) + off_bias_vector = np.asarray(decoder.out_layers['ct_offset'].weights[-1]) + size_bias_vector = np.asarray(decoder.out_layers['ct_size'].weights[-1]) self.assertArrayNear(hm_bias_vector, - np.repeat(-2.19, repeats=91), err=1.00e-6) + np.repeat(-2.19, repeats=80), err=1.00e-6) self.assertArrayNear(off_bias_vector, np.repeat(0, repeats=2), err=1.00e-6) self.assertArrayNear(size_bias_vector, From 1bb1ce359053c0eaceebe28563ae80d40d6ae912 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 17 Feb 2021 11:24:00 -0500 Subject: [PATCH 036/132] More accurate hyperparameter naming --- centernet/configs/centernet.py | 57 +++++++++---------- .../modeling/decoders/centernet_decoder.py | 30 +++++----- .../decoders/centernet_decoder_test.py | 14 ++--- 3 files changed, 49 insertions(+), 52 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index dd4858553..288f17155 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -26,37 +26,29 @@ from centernet.configs import backbones -# Note these do not subclass from hyperparams.Config, -# for some reason Dicts are not supported in their -# immutable types + +@dataclasses.dataclass +class Loss(hyperparams.Config): + pass @dataclasses.dataclass -class CenterNetDecoder(hyperparams.Config): - """CenterNet for 2D Object Detection Decoder.""" - task_outputs: Dict[str, int] = dataclasses.field( - default_factory=lambda: {'heatmap': 91, 'local_offset': 2, 'object_size': 2}) - heatmap_bias: float = -2.19 +class DetectionLoss(Loss): + detection_weight: float = 1.0 + corner_pull_weight: float = 0.1 # alpha + corner_push_weight: float = 0.1 # beta + offset_weight: float = 1 # gamma + @dataclasses.dataclass -class CenterNet3D(): - """CenterNet for 3D Object Detection Decoder.""" - task_outputs: Dict[str, int] = dataclasses.field( - default_factory=lambda: {'heatmap': 91, - 'local_offset': 2, - 'object_size': 3, - 'depth' : 1, - 'orientation': 8}) - heatmap_bias: float = -2.19 +class SegmentationLoss(Loss): + pass + @dataclasses.dataclass -class CenterNetPose(): - """CenterNet for Pose Estimation Decoder.""" - task_outputs: Dict[str, int] = dataclasses.field( - default_factory=lambda: {'heatmap': 17, - 'joint_locs': 17 * 2, - 'joint_offset': 2}) - heatmap_bias: float = -2.19 +class Losses(hyperparams.Config): + detection: DetectionLoss = DetectionLoss() + segmentation: SegmentationLoss = SegmentationLoss() @dataclasses.dataclass @@ -91,6 +83,8 @@ class CenterNetSubTasks(cfg.TaskConfig): class CenterNetTask(cfg.TaskConfig): model: CenterNet = CenterNet() subtasks: CenterNetSubTasks = CenterNetSubTasks() + losses: Losses = Losses() + def _get_output_length_dict(self): lengths = {} assert self.subtasks.detection is not None or self.subtasks.kp_detection \ @@ -98,24 +92,27 @@ def _get_output_length_dict(self): "subtask to CenterNet" if self.subtasks.detection: + # TODO: locations of the ground truths will also be passed in from the + # data pipeline which need to be mapped accordingly assert self.subtasks.detection.use_centers or \ self.subtasks.detection.use_corners, "Cannot use CenterNet without " \ "heatmaps" if self.subtasks.detection.use_centers: lengths.update({ - 'ct_heatmaps': self.model.num_classes, # renamed from heatmap - 'ct_offset': 2, # renamed from local_offset - 'ct_size': 2 # renamed from object_size + 'ct_heatmaps': self.model.num_classes, + 'ct_offset': 2, }) + if not self.subtasks.detection.use_corners: + lengths['ct_size'] = 2 + if self.subtasks.detection.use_corners: lengths.update({ 'tl_heatmaps': self.model.num_classes, 'tl_offset': 2, - 'tl_size': 2, 'br_heatmaps': self.model.num_classes, - 'br_offset': 2, - 'br_size': 2, + 'br_offset': 2 }) + if self.subtasks.detection.predict_3d: lengths.update({ 'depth': 1, diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index 094eb7663..e6ce00684 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -1,11 +1,13 @@ import tensorflow as tf from centernet.modeling.layers.nn_blocks import CenterNetDecoderConv +from centernet.configs import centernet as cfg + class CenterNetDecoder(tf.keras.Model): """ CenterNet Decoder """ - def __init__(self, + def __init__(self, task_outputs: dict, heatmap_bias: float = -2.19, **kwargs): @@ -13,20 +15,20 @@ def __init__(self, Args: task_outputs: dict, with key-value pairs denoting the names of the outputs and the desired channel depth of each output - heatmap_bias: float, constant value to initialize the convolution layer - bias vector if it is responsible for generating a heatmap (not for + heatmap_bias: float, constant value to initialize the convolution layer + bias vector if it is responsible for generating a heatmap (not for regressed predictions) call Returns: - dictionary where the keys-value pairs denote the names of the output + dictionary where the keys-value pairs denote the names of the output and the respective output tensor """ - + self._task_outputs = task_outputs self._heatmap_bias = heatmap_bias super().__init__(**kwargs) - + def build(self, input_shape): self.out_layers = {} for key in self._task_outputs: @@ -35,9 +37,9 @@ def build(self, input_shape): if 'heatmaps' in key: bias = self._heatmap_bias - self.out_layers[key] = CenterNetDecoderConv(output_filters=num_filters, + self.out_layers[key] = CenterNetDecoderConv(output_filters=num_filters, name=key, bias_init=bias) - + super().build(input_shape) def call(self, x): @@ -46,7 +48,7 @@ def call(self, x): outputs[key] = self.out_layers[key](x) return outputs - + def get_config(self): layer_config = { 'task_outputs': self._task_outputs, @@ -56,14 +58,12 @@ def get_config(self): #layer_config.update(super().get_config()) return layer_config -def build_centernet_decoder(model_config) -> tf.keras.Model: - cfg = model_config() - - task_outputs = cfg._get_output_length_dict() - heatmap_bias = cfg.model.decoder.heatmap_bias +def build_centernet_decoder(task_config: cfg.CenterNetTask) -> tf.keras.Model: + task_outputs = task_config._get_output_length_dict() + heatmap_bias = task_config.model.decoder.heatmap_bias print(task_outputs) return CenterNetDecoder( task_outputs=task_outputs, - heatmap_bias=heatmap_bias) \ No newline at end of file + heatmap_bias=heatmap_bias) diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index 65cdbc213..cf46022a3 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -9,13 +9,13 @@ class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_create_decoder(self): - decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNetTask) + decoder = centernet_decoder.build_centernet_decoder(task_config=cfg.CenterNetTask()) config = decoder.get_config() self.assertEqual(len(config), 2) self.assertEqual(config['heatmap_bias'], -2.19) def test_decoder_shape(self): - decoder = centernet_decoder.build_centernet_decoder(model_config=cfg.CenterNetTask) + decoder = centernet_decoder.build_centernet_decoder(task_config=cfg.CenterNetTask()) # Output shape tests outputs = decoder(np.zeros((2, 128, 128, 256), dtype=np.float32)) @@ -29,12 +29,12 @@ def test_decoder_shape(self): off_bias_vector = np.asarray(decoder.out_layers['ct_offset'].weights[-1]) size_bias_vector = np.asarray(decoder.out_layers['ct_size'].weights[-1]) - self.assertArrayNear(hm_bias_vector, + self.assertArrayNear(hm_bias_vector, np.repeat(-2.19, repeats=80), err=1.00e-6) - self.assertArrayNear(off_bias_vector, + self.assertArrayNear(off_bias_vector, + np.repeat(0, repeats=2), err=1.00e-6) + self.assertArrayNear(size_bias_vector, np.repeat(0, repeats=2), err=1.00e-6) - self.assertArrayNear(size_bias_vector, - np.repeat(0, repeats=2), err=1.00e-6) if __name__ == '__main__': - tf.test.main() \ No newline at end of file + tf.test.main() From 8ce9cbaf89504bc0c3bb0328f20d6784436a3609 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 17 Feb 2021 12:17:27 -0500 Subject: [PATCH 037/132] Add build_model function and change some basic formatting --- centernet/{tasks => ops}/loss_utils.py | 0 centernet/tasks/centernet_object_detection.py | 32 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) rename centernet/{tasks => ops}/loss_utils.py (100%) diff --git a/centernet/tasks/loss_utils.py b/centernet/ops/loss_utils.py similarity index 100% rename from centernet/tasks/loss_utils.py rename to centernet/ops/loss_utils.py diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py index 0827d45d0..e072533db 100644 --- a/centernet/tasks/centernet_object_detection.py +++ b/centernet/tasks/centernet_object_detection.py @@ -7,16 +7,23 @@ from official.core import task_factory from official.vision.beta.evaluation import coco_evaluator -import loss_utils as utils -from losses import penalty_reduced_logistic_focal_loss -from losses import l1_localization_loss +from centernet.configs import centernet as cfg +import centernet.ops.loss_utils as utils +from centernet.losses import penalty_reduced_logistic_focal_loss +from centernet.losses import l1_localization_loss -# @task_factory.register_task_cls(exp_cfg.YoloTask) -class CenterNetObjectDetectionTask(base_task.Task): +@task_factory.register_task_cls(cfg.CenterNetTask) +class CenterNetTask(base_task.Task): def __init__(self, params, logging_dir: str = None): super().__init__(params, logging_dir) + def build_inputs(self, params, input_context=None): + pass + + def build_model(self): + task_cfg = self.task_config + def build_losses(self, outputs, labels, aux_losses=None): total_loss = 0.0 total_scale_loss = 0.0 @@ -37,27 +44,32 @@ def build_losses(self, outputs, labels, aux_losses=None): pred = utils._flatten_spatial_dimensions(pred) total_loss += object_center_loss( flattened_ct_heatmaps, pred) #removed weight parameter (weight = per_pixel_weight) - loss += tf.reduce_sum(total_loss) / ( + center_loss = tf.reduce_sum(total_loss) / ( float(len(outputs['ct_heatmaps'])) * num_boxes) + loss += center_loss + metric_dict['ct_loss'] = center_loss #localization loss for offset and scale loss localization_loss_fn = l1_localization_loss.L1LocalizationLoss() for scale_pred, offset_pred in zip(outputs['ct_size'], outputs['ct_offset']): # Compute the scale loss. scale_pred = utils.get_batch_predictions_from_indices( - scale_pred, labels['tag_masks']) + scale_pred, labels['tag_locs']) total_scale_loss += localization_loss_fn( labels['ct_size'], scale_pred) #removed weights=batch_weights # Compute the offset loss. offset_pred = utils.get_batch_predictions_from_indices( - offset_pred, labels['tag_masks']) + offset_pred, labels['tag_locs']) total_offset_loss += localization_loss_fn( labels['ct_offset'], offset_pred) #removed weights=batch_weights scale_loss += tf.reduce_sum(total_scale_loss) / ( float(len(outputs['ct_size'])) * num_boxes) offset_loss += tf.reduce_sum(total_offset_loss) / ( float(len(outputs['ct_size'])) * num_boxes) + metric_dict['ct_scale_loss'] = scale_loss + metric_dict['ct_offset_loss'] = offset_loss - return loss, scale_loss, offset_loss, metric_dict + return loss, metric_dict - \ No newline at end of file + def build_metrics(self, training=True): + pass From c18a0c337c34ed861f071b899c9e9d8320339ef1 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Fri, 19 Feb 2021 17:09:59 -0500 Subject: [PATCH 038/132] Adapt TF1 version of absolute_difference to TF2 --- centernet/losses/l1_localization_loss.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/centernet/losses/l1_localization_loss.py b/centernet/losses/l1_localization_loss.py index fe189e1dc..d9fb5fefe 100644 --- a/centernet/losses/l1_localization_loss.py +++ b/centernet/losses/l1_localization_loss.py @@ -10,7 +10,7 @@ def absolute_difference( labels, predictions, weights=1.0, - reduction=tf.keras.losses.Reduction.SUM_BY_NONZERO_WEIGHTS): + reduction=tf.keras.losses.Reduction.SUM): """Adds an Absolute Difference loss to the training procedure. `weights` acts as a coefficient for the loss. If a scalar is provided, then the loss is simply scaled by the given value. If `weights` is a `Tensor` of @@ -39,8 +39,7 @@ def absolute_difference( raise ValueError("labels must not be None.") if predictions is None: raise ValueError("predictions must not be None.") - with tf.name_scope(scope, "absolute_difference", - (predictions, labels, weights)) as scope: + with tf.name_scope("absolute_difference") as scope: predictions = tf.cast(predictions, dtype=tf.float32) labels = tf.cast(labels, dtype=tf.float32) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) From e3ec21534321050b77fa3d3d462bcee44fce259f Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 23 Feb 2021 00:58:02 -0500 Subject: [PATCH 039/132] finished build_model, passes base tests --- centernet/configs/backbones.py | 5 + centernet/configs/centernet.py | 28 +++-- centernet/modeling/CenterNet.py | 106 ++++++++++++++++++ centernet/modeling/backbones/hourglass.py | 4 + .../modeling/decoders/centernet_decoder.py | 17 ++- .../decoders/centernet_decoder_test.py | 9 +- centernet/modeling/layers/CenterNet_test.py | 27 +++++ centernet/tasks/centernet.py | 72 ++++++++++++ centernet/tasks/centernet_test.py | 24 ++++ 9 files changed, 272 insertions(+), 20 deletions(-) create mode 100644 centernet/modeling/CenterNet.py create mode 100644 centernet/modeling/layers/CenterNet_test.py create mode 100644 centernet/tasks/centernet.py create mode 100644 centernet/tasks/centernet_test.py diff --git a/centernet/configs/backbones.py b/centernet/configs/backbones.py index 9338e34e3..c7bb53388 100644 --- a/centernet/configs/backbones.py +++ b/centernet/configs/backbones.py @@ -20,6 +20,7 @@ import dataclasses from official.modeling import hyperparams +from official.vision.beta.configs import backbones @dataclasses.dataclass @@ -32,3 +33,7 @@ class Hourglass(hyperparams.Config): default_factory=lambda: [2, 2, 2, 2, 2, 4]) num_hourglasses: int = 2 initial_downsample: bool = True + +@dataclasses.dataclass +class Backbone(backbones.Backbone): + hourglass: Hourglass = Hourglass() \ No newline at end of file diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 288f17155..9cc0fbe91 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Decoder configurations.""" -from typing import Dict +"""CenterNet configuration definition.""" +from typing import ClassVar, Dict, List, Optional, Tuple, Union # Import libraries import dataclasses @@ -22,11 +22,8 @@ from official.modeling import hyperparams from official.modeling.hyperparams import config_definitions as cfg from official.vision.beta.configs import common -from official.vision.beta.configs import backbones - from centernet.configs import backbones - @dataclasses.dataclass class Loss(hyperparams.Config): pass @@ -56,12 +53,6 @@ class CenterNetDecoder(hyperparams.Config): heatmap_bias: float = -2.19 -@dataclasses.dataclass -class CenterNet(hyperparams.Config): - num_classes: int = 80 - decoder: CenterNetDecoder = CenterNetDecoder() - - @dataclasses.dataclass class CenterNetDetection(cfg.TaskConfig): use_centers: bool = True @@ -78,6 +69,17 @@ class CenterNetSubTasks(cfg.TaskConfig): # reid: bool = False # temporal: bool = False +@dataclasses.dataclass +class CenterNetBase(hyperparams.OneOfConfig): + backbone: backbones.Backbone = backbones.Backbone(type='hourglass') + decoder: CenterNetDecoder = CenterNetDecoder() + +@dataclasses.dataclass +class CenterNet(hyperparams.Config): + num_classes: int = 80 + input_size: Optional[List[int]] = dataclasses.field( + default_factory=lambda: [None, None, 3]) + base: Union[str, CenterNetBase] = CenterNetBase() @dataclasses.dataclass class CenterNetTask(cfg.TaskConfig): @@ -85,6 +87,8 @@ class CenterNetTask(cfg.TaskConfig): subtasks: CenterNetSubTasks = CenterNetSubTasks() losses: Losses = Losses() + weight_decay: float = 5e-4 + def _get_output_length_dict(self): lengths = {} assert self.subtasks.detection is not None or self.subtasks.kp_detection \ @@ -129,4 +133,4 @@ def _get_output_length_dict(self): # 'joint_offset': 2 # }) - return lengths + return lengths \ No newline at end of file diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py new file mode 100644 index 000000000..a19324663 --- /dev/null +++ b/centernet/modeling/CenterNet.py @@ -0,0 +1,106 @@ +from official.core import registry +from official.vision.beta.modeling.backbones import factory +import tensorflow as tf +import tensorflow.keras as ks + +from centernet.modeling.backbones.hourglass import Hourglass +from centernet.modeling.backbones.hourglass import build_hourglass +from centernet.modeling.decoders.centernet_decoder import CenterNetDecoder + +# TODO: import prediction and filtering layers when made + +class CenterNet(ks.Model): + + def __init__(self, + backbone=None, + decoder=None, + head=None, + filter=None, + **kwargs): + super().__init__(**kwargs) + # model components + self._backbone = backbone + self._decoder = decoder + self._head = head + self._filter = filter + return + + def build(self, input_shape): + self._backbone.build(input_shape) + nshape = self._backbone.output_specs + self._decoder.build(nshape) + super().build(input_shape) + + def call(self, inputs, training=False): + features = self._backbone(inputs) + final_backbone_output = features[-1] + decoded_maps = self._decoder(final_backbone_output) + + # TODO: head + filters + + if training: + return {"raw_output": decoded_maps} + else: + # TODO: uncomment when filter is implemented + # predictions = self._filter(raw_predictions) + # predictions.update({"raw_output": raw_predictions}) + # return predictions + return {"raw_output": decoded_maps} + + @property + def backbone(self): + return self._backbone + + @property + def decoder(self): + return self._decoder + + @property + def head(self): + return self._head + + @property + def filter(self): + return self._filter + +def build_centernet_decoder(input_specs, task_config): + # NOTE: For now just support the default config + # model specific + heatmap_bias = task_config.model.base.decoder.heatmap_bias + + # task specific + task_outputs = task_config._get_output_length_dict() + model = CenterNetDecoder( + task_outputs=task_outputs, + heatmap_bias=heatmap_bias) + + model.build(input_specs) + return model + +def build_centernet_filter(model_config): + return None + +def build_centernet_head(model_config): + return None + +def build_centernet(input_specs, task_config, l2_regularization): + print(task_config.as_dict()) + print(input_specs) + print(l2_regularization) + model_config = task_config.model + backbone = factory.build_backbone(input_specs, model_config.base, + l2_regularization) + + decoder = build_centernet_decoder(backbone.output_specs.as_list(), task_config) + head = build_centernet_head(model_config) + filter = build_centernet_filter(model_config) + + model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) + + model.build(input_specs.shape) + + # TODO: uncommend when filter is implemented + # losses = filter.losses + losses = None + return model, losses + diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 83cacaf28..28373478d 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -115,6 +115,7 @@ def __init__( self._blocks_per_stage = blocks_per_stage self._num_hourglasses = num_hourglasses self._initial_downsample = initial_downsample + self._output_specs = all_heatmaps[-1].get_shape() def get_config(self): layer_config = { @@ -127,6 +128,9 @@ def get_config(self): layer_config.update(super().get_config()) return layer_config + @property + def output_specs(self): + return self._output_specs # @factory.register_backbone_builder('hourglass') diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index e6ce00684..94166a9e6 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -58,12 +58,17 @@ def get_config(self): #layer_config.update(super().get_config()) return layer_config -def build_centernet_decoder(task_config: cfg.CenterNetTask) -> tf.keras.Model: +def build_centernet_decoder(input_specs, task_config): + # NOTE: For now just support the default config + + # model specific + heatmap_bias = task_config.model.base.decoder.heatmap_bias + + # task specific task_outputs = task_config._get_output_length_dict() - heatmap_bias = task_config.model.decoder.heatmap_bias - - print(task_outputs) - - return CenterNetDecoder( + model = CenterNetDecoder( task_outputs=task_outputs, heatmap_bias=heatmap_bias) + + model.build(input_specs) + return model \ No newline at end of file diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index cf46022a3..16466d6cc 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -9,13 +9,18 @@ class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_create_decoder(self): - decoder = centernet_decoder.build_centernet_decoder(task_config=cfg.CenterNetTask()) + decoder = centernet_decoder.build_centernet_decoder( + task_config=cfg.CenterNetTask(), + input_specs=(2, 128, 128, 256)) + config = decoder.get_config() self.assertEqual(len(config), 2) self.assertEqual(config['heatmap_bias'], -2.19) def test_decoder_shape(self): - decoder = centernet_decoder.build_centernet_decoder(task_config=cfg.CenterNetTask()) + decoder = centernet_decoder.build_centernet_decoder( + task_config=cfg.CenterNetTask(), + input_specs=(2, 128, 128, 256)) # Output shape tests outputs = decoder(np.zeros((2, 128, 128, 256), dtype=np.float32)) diff --git a/centernet/modeling/layers/CenterNet_test.py b/centernet/modeling/layers/CenterNet_test.py new file mode 100644 index 000000000..40ed698e4 --- /dev/null +++ b/centernet/modeling/layers/CenterNet_test.py @@ -0,0 +1,27 @@ +from absl.testing import parameterized +import tensorflow as tf +import numpy as np +import dataclasses + +from official.modeling import hyperparams +from official.vision.beta.configs import backbones + +from centernet.modeling.CenterNet import build_centernet +from centernet.configs import centernet + + +class CenterNetTest(parameterized.TestCase, tf.test.TestCase): + + def testBuildCenterNet(self): + input_specs = tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]) + + config = centernet.CenterNetTask() + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + + # TODO: add some call tests + + + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py new file mode 100644 index 000000000..dfa49ef9b --- /dev/null +++ b/centernet/tasks/centernet.py @@ -0,0 +1,72 @@ +import tensorflow as tf +from tensorflow.keras.mixed_precision import experimental as mixed_precision + +from absl import logging +from official.core import base_task +from official.core import input_reader +from official.core import task_factory +from centernet.configs import centernet as exp_cfg + +from official.vision.beta.evaluation import coco_evaluator + + +@task_factory.register_task_cls(exp_cfg.CenterNetTask) +class CenterNetTask(base_task.Task): + """A single-replica view of training procedure. + RetinaNet task provides artifacts for training/evalution procedures, including + loading/iterating over Datasets, initializing the model, calculating the loss, + post-processing, and customized metrics with reduction. + """ + + def __init__(self, params, logging_dir: str = None): + super().__init__(params, logging_dir) + self._loss_dict = None + + self.coco_metric = None + self._metric_names = [] + self._metrics = [] + return + + def build_model(self): + """get an instance of CenterNet""" + from centernet.modeling.CenterNet import build_centernet + params = self.task_config.train_data + model_base_cfg = self.task_config.model + l2_weight_decay = self.task_config.weight_decay / 2.0 + + input_specs = tf.keras.layers.InputSpec(shape=[None] + + model_base_cfg.input_size) + l2_regularizer = ( + tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) + + model, losses = build_centernet(input_specs, self.task_config, l2_regularizer) + self._loss_dict = losses + return model + + # Everything below was from YOLO + def build_inputs(self, params, input_context=None): + pass + + def build_losses(self, outputs, labels, aux_losses=None): + pass + + def build_metrics(self, training=True): + pass + + def train_step(self, inputs, model, optimizer, metrics=None): + pass + + def aggregate_logs(self, state=None, step_outputs=None): + pass + + def reduce_aggregated_logs(self, aggregated_logs): + pass + + def _get_masks(self, + xy_exponential=True, + exp_base=2, + xy_scale_base='default_value'): + pass + + def initialize(self, model: tf.keras.Model): + pass \ No newline at end of file diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py new file mode 100644 index 000000000..caa01c6a3 --- /dev/null +++ b/centernet/tasks/centernet_test.py @@ -0,0 +1,24 @@ +from absl.testing import parameterized +import tensorflow as tf +import numpy as np +import dataclasses + +from official.modeling import hyperparams +from official.vision.beta.configs import backbones + +from centernet.tasks.centernet import CenterNetTask +from centernet.configs import centernet as exp_cfg + + +class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): + + def testCenterNetTask(self): + config = exp_cfg.CenterNetTask() + task = CenterNetTask(config) + model = task.build_model() + model.summary() + + + +if __name__ == '__main__': + tf.test.main() From d789b16474dfb27d4ba1d0b566ae6a496a21f2aa Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 23 Feb 2021 20:54:51 -0500 Subject: [PATCH 040/132] moved CenterNet_test out of layers directory --- centernet/modeling/{layers => }/CenterNet_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename centernet/modeling/{layers => }/CenterNet_test.py (100%) diff --git a/centernet/modeling/layers/CenterNet_test.py b/centernet/modeling/CenterNet_test.py similarity index 100% rename from centernet/modeling/layers/CenterNet_test.py rename to centernet/modeling/CenterNet_test.py From 2521ec74774175dea6b57c501289d96cc20c4fcc Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 24 Feb 2021 00:25:20 -0500 Subject: [PATCH 041/132] Another test case --- centernet/ops/preprocessing_ops_test.py | 45 +++++++++++++++++++++++-- centernet/utils/groundtruth.py | 6 ++-- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/centernet/ops/preprocessing_ops_test.py b/centernet/ops/preprocessing_ops_test.py index ac97da7a0..bd97ac345 100644 --- a/centernet/ops/preprocessing_ops_test.py +++ b/centernet/ops/preprocessing_ops_test.py @@ -1,9 +1,31 @@ +from absl.testing import parameterized import tensorflow as tf import numpy as np import centernet.utils.groundtruth as preprocessing_ops +from yolo.ops import box_ops -class CenterNetBoxTargetAssignerTest(tf.test.TestCase): +def image_shape_to_grids(height, width): + """Computes xy-grids given the shape of the image. + Args: + height: The height of the image. + width: The width of the image. + Returns: + A tuple of two tensors: + y_grid: A float tensor with shape [height, width] representing the + y-coordinate of each pixel grid. + x_grid: A float tensor with shape [height, width] representing the + x-coordinate of each pixel grid. + """ + out_height = tf.cast(height, tf.float32) + out_width = tf.cast(width, tf.float32) + x_range = tf.range(out_width, dtype=tf.float32) + y_range = tf.range(out_height, dtype=tf.float32) + x_grid, y_grid = tf.meshgrid(x_range, y_range, indexing='xy') + return (y_grid, x_grid) + + +class CenterNetBoxTargetAssignerTest(parameterized.TestCase, tf.test.TestCase): def __init__(self, *args, **kwargs): super(CenterNetBoxTargetAssignerTest, self).__init__(*args, **kwargs) @@ -43,7 +65,7 @@ def test_max_distance_for_overlap(self): boxes1 = np.vstack([ymin1, xmin1, ymax1, xmax1]).T boxes2 = np.vstack([ymin2, xmin2, ymax2, xmax2]).T - iou = np.diag(np_box_ops.iou(boxes1, boxes2)) + iou = box_ops.compute_iou(boxes1, boxes2) self.assertTrue(np.all(iou >= min_iou)) @@ -52,6 +74,25 @@ def test_max_distance_for_overlap_centernet(self): distance = preprocessing_ops.gaussian_radius((10, 5), 0.5) self.assertAlmostEqual(2.807764064, distance.numpy()) + @parameterized.parameters((False,), (True,)) + def test_coordinates_to_heatmap(self, sparse): + self.skipTest('Not yet functioning.') + + (y_grid, x_grid) = image_shape_to_grids(height=3, width=5) + y_coordinates = tf.constant([1.5, 0.5], dtype=tf.float32) + x_coordinates = tf.constant([2.5, 4.5], dtype=tf.float32) + sigma = tf.constant([0.1, 0.5], dtype=tf.float32) + channel_onehot = tf.constant([[1, 0, 0], [0, 1, 0]], dtype=tf.float32) + channel_weights = tf.constant([1, 1], dtype=tf.float32) + heatmap = ta_utils.coordinates_to_heatmap(y_grid, x_grid, y_coordinates, + x_coordinates, sigma, + channel_onehot, + channel_weights, sparse=sparse) + + # Peak at (1, 2) for the first class. + self.assertAlmostEqual(1.0, heatmap[1, 2, 0]) + # Peak at (0, 4) for the second class. + self.assertAlmostEqual(1.0, heatmap[0, 4, 1]) if __name__ == '__main__': tf.test.main() diff --git a/centernet/utils/groundtruth.py b/centernet/utils/groundtruth.py index bec14122a..91f20d10f 100644 --- a/centernet/utils/groundtruth.py +++ b/centernet/utils/groundtruth.py @@ -11,7 +11,7 @@ def _smallest_positive_root(a, b, c) -> tf.Tensor: root1 = (-b - discriminant) / (2 * a) root2 = (-b + discriminant) / (2 * a) - return (-b + discriminant) / (2) #tf.where(tf.less(root1, 0), root2, root1) + return (-b + discriminant) / (2) # tf.where(tf.less(root1, 0), root2, root1) def gaussian_radius(det_size, min_overlap=0.7) -> int: """ @@ -56,7 +56,7 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: return tf.reduce_min([r1, r2, r3], axis=0) -def gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: +def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: """ This represents the penalty reduction around a point. Params: @@ -83,7 +83,7 @@ def draw_gaussian(heatmap, center, radius, k=1): """ diameter = 2 * radius + 1 - gaussian = gaussian_penalty(radius) + gaussian = _gaussian_penalty(radius) x, y = center From f77b23c9382b49dfa76074a09e3358e2c4837b91 Mon Sep 17 00:00:00 2001 From: Feny Patel Date: Wed, 24 Feb 2021 19:50:20 -0500 Subject: [PATCH 042/132] update comments --- centernet/losses/l1_localization_loss.py | 150 +++++++++--------- .../penalty_reduced_logistic_focal_loss.py | 122 +++++++------- 2 files changed, 136 insertions(+), 136 deletions(-) diff --git a/centernet/losses/l1_localization_loss.py b/centernet/losses/l1_localization_loss.py index fe189e1dc..8e8c48db3 100644 --- a/centernet/losses/l1_localization_loss.py +++ b/centernet/losses/l1_localization_loss.py @@ -1,75 +1,75 @@ -import tensorflow as tf - -try: - # Try to get TF 1.x version if possible - import tensorflow.compat.v1 as tf_v1 - absolute_difference = tf_v1.losses.absolute_difference -except (ImportError, AttributeError): - # The following code was adapted from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/losses/losses_impl.py - from tensorflow.python.keras.utils import losses_utils - - def absolute_difference( - labels, predictions, weights=1.0, - reduction=tf.keras.losses.Reduction.SUM_BY_NONZERO_WEIGHTS): - """Adds an Absolute Difference loss to the training procedure. - `weights` acts as a coefficient for the loss. If a scalar is provided, then - the loss is simply scaled by the given value. If `weights` is a `Tensor` of - shape `[batch_size]`, then the total loss for each sample of the batch is - rescaled by the corresponding element in the `weights` vector. If the shape - of `weights` matches the shape of `predictions`, then the loss of each - measurable element of `predictions` is scaled by the corresponding value of - `weights`. - Args: - labels: The ground truth output tensor, same dimensions as 'predictions'. - predictions: The predicted outputs. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions - must be either `1`, or the same as the corresponding `losses` - dimension). - reduction: Type of reduction to apply to loss. - Returns: - Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same - shape as `labels`; otherwise, it is scalar. - Raises: - ValueError: If the shape of `predictions` doesn't match that of - `labels` or if the shape of `weights` is invalid or if `labels` - or `predictions` is None. - """ - if labels is None: - raise ValueError("labels must not be None.") - if predictions is None: - raise ValueError("predictions must not be None.") - with tf.name_scope(scope, "absolute_difference", - (predictions, labels, weights)) as scope: - predictions = tf.cast(predictions, dtype=tf.float32) - labels = tf.cast(labels, dtype=tf.float32) - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - losses = tf.abs(tf.subtract(predictions, labels)) - return losses_utils.compute_weighted_loss( - losses, weights, reduction=reduction) - -class L1LocalizationLoss(tf.keras.losses.Loss): - """L1 loss or absolute difference. - When used in a per-pixel manner, each pixel should be given as an anchor. - """ - - def __call__(self, y_true, y_pred, sample_weight=None): - """Compute loss function. - Args: - y_true: A float tensor of shape [batch_size, num_anchors] - representing the regression targets - y_pred: A float tensor of shape [batch_size, num_anchors] - representing the (encoded) predicted locations of objects. - sample_weight: a float tensor of shape [batch_size, num_anchors] - Returns: - loss: a float tensor of shape [batch_size, num_anchors] tensor - representing the value of the loss function. - """ - return absolute_difference( - y_true, - y_pred, - weights=sample_weight, - reduction=tf.keras.losses.Reduction.NONE - ) - - call = __call__ +import tensorflow as tf + +try: + # Try to get TF 1.x version if possible + import tensorflow.compat.v1 as tf_v1 + absolute_difference = tf_v1.losses.absolute_difference +except (ImportError, AttributeError): + # The following code was adapted from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/losses/losses_impl.py + from tensorflow.python.keras.utils import losses_utils + + def absolute_difference( + labels, predictions, weights=1.0, + reduction=tf.keras.losses.Reduction.SUM_BY_NONZERO_WEIGHTS): + """Adds an Absolute Difference loss to the training procedure. + `weights` acts as a coefficient for the loss. If a scalar is provided, then + the loss is simply scaled by the given value. If `weights` is a `Tensor` of + shape `[batch_size]`, then the total loss for each sample of the batch is + rescaled by the corresponding element in the `weights` vector. If the shape + of `weights` matches the shape of `predictions`, then the loss of each + measurable element of `predictions` is scaled by the corresponding value of + `weights`. + Args: + labels: The ground truth output tensor, same dimensions as 'predictions'. + predictions: The predicted outputs. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `labels`, and must be broadcastable to `labels` (i.e., all dimensions + must be either `1`, or the same as the corresponding `losses` + dimension). + reduction: Type of reduction to apply to loss. + Returns: + Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same + shape as `labels`; otherwise, it is scalar. + Raises: + ValueError: If the shape of `predictions` doesn't match that of + `labels` or if the shape of `weights` is invalid or if `labels` + or `predictions` is None. + """ + if labels is None: + raise ValueError("labels must not be None.") + if predictions is None: + raise ValueError("predictions must not be None.") + with tf.name_scope(scope, "absolute_difference", + (predictions, labels, weights)) as scope: + predictions = tf.cast(predictions, dtype=tf.float32) + labels = tf.cast(labels, dtype=tf.float32) + predictions.get_shape().assert_is_compatible_with(labels.get_shape()) + losses = tf.abs(tf.subtract(predictions, labels)) + return losses_utils.compute_weighted_loss( + losses, weights, reduction=reduction) + +class L1LocalizationLoss(tf.keras.losses.Loss): + """L1 loss or absolute difference. + When used in a per-pixel manner, each pixel should be given as an anchor. + """ + + def __call__(self, y_true, y_pred, sample_weight=None): + """Compute loss function. + Args: + y_true: A float tensor of shape [batch_size, num_anchors] + representing the regression targets + y_pred: A float tensor of shape [batch_size, num_anchors] + representing the (encoded) predicted locations of objects. + sample_weight: a float tensor of shape [batch_size, num_anchors] + Returns: + loss: a float tensor of shape [batch_size, num_anchors] tensor + representing the value of the loss function. + """ + return absolute_difference( + y_true, + y_pred, + weights=sample_weight, + reduction=tf.keras.losses.Reduction.NONE + ) + + call = __call__ diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss.py b/centernet/losses/penalty_reduced_logistic_focal_loss.py index 2f5cb5bad..7706912d6 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss.py @@ -1,61 +1,61 @@ -# may be a dupe of retinanet_losses. need to look later -import tensorflow as tf - -class PenaltyReducedLogisticFocalLoss(tf.keras.losses.Loss): - """Penalty-reduced pixelwise logistic regression with focal loss. - The loss is defined in Equation (1) of the Objects as Points[1] paper. - Although the loss is defined per-pixel in the output space, this class - assumes that each pixel is an anchor to be compatible with the base class. - [1]: https://arxiv.org/abs/1904.07850 - """ - - def __init__(self, alpha=2.0, beta=4.0, sigmoid_clip_value=1e-4): - """Constructor. - Args: - alpha: Focussing parameter of the focal loss. Increasing this will - decrease the loss contribution of the well classified examples. - beta: The local penalty reduction factor. Increasing this will decrease - the contribution of loss due to negative pixels near the keypoint. - sigmoid_clip_value: The sigmoid operation used internally will be clipped - between [sigmoid_clip_value, 1 - sigmoid_clip_value) - """ - self._alpha = alpha - self._beta = beta - self._sigmoid_clip_value = sigmoid_clip_value - super(PenaltyReducedLogisticFocalLoss, self).__init__() - - def call(self, y_true, y_pred): - """Compute loss function. - In all input tensors, `num_anchors` is the total number of pixels in the - the output space. - Args: - prediction_tensor: A float tensor of shape [batch_size, num_anchors, - num_classes] representing the predicted unscaled logits for each class. - The function will compute sigmoid on this tensor internally. - target_tensor: A float tensor of shape [batch_size, num_anchors, - num_classes] representing a tensor with the 'splatted' keypoints, - possibly using a gaussian kernel. This function assumes that - the target is bounded between [0, 1]. - weights: a float tensor of shape, either [batch_size, num_anchors, - num_classes] or [batch_size, num_anchors, 1]. If the shape is - [batch_size, num_anchors, 1], all the classses are equally weighted. - Returns: - loss: a float tensor of shape [batch_size, num_anchors, num_classes] - representing the value of the loss function. - """ - - target_tensor = y_true - - is_present_tensor = tf.math.equal(target_tensor, 1.0) - prediction_tensor = tf.clip_by_value(tf.sigmoid(y_pred), - self._sigmoid_clip_value, - 1 - self._sigmoid_clip_value) - - positive_loss = (tf.math.pow((1 - prediction_tensor), self._alpha)* - tf.math.log(prediction_tensor)) - negative_loss = (tf.math.pow((1 - target_tensor), self._beta)* - tf.math.pow(prediction_tensor, self._alpha)* - tf.math.log(1 - prediction_tensor)) - - loss = -tf.where(is_present_tensor, positive_loss, negative_loss) - return loss +# may be a dupe of retinanet_losses. need to look later +import tensorflow as tf + +class PenaltyReducedLogisticFocalLoss(tf.keras.losses.Loss): + """Penalty-reduced pixelwise logistic regression with focal loss. + The loss is defined in Equation (1) of the Objects as Points[1] paper. + Although the loss is defined per-pixel in the output space, this class + assumes that each pixel is an anchor to be compatible with the base class. + [1]: https://arxiv.org/abs/1904.07850 + """ + + def __init__(self, alpha=2.0, beta=4.0, sigmoid_clip_value=1e-4): + """Constructor. + Args: + alpha: Focussing parameter of the focal loss. Increasing this will + decrease the loss contribution of the well classified examples. + beta: The local penalty reduction factor. Increasing this will decrease + the contribution of loss due to negative pixels near the keypoint. + sigmoid_clip_value: The sigmoid operation used internally will be clipped + between [sigmoid_clip_value, 1 - sigmoid_clip_value) + """ + self._alpha = alpha + self._beta = beta + self._sigmoid_clip_value = sigmoid_clip_value + super(PenaltyReducedLogisticFocalLoss, self).__init__() + + def call(self, y_true, y_pred): + """Compute loss function. + In all input tensors, `num_anchors` is the total number of pixels in the + the output space. + Args: + prediction_tensor: A float tensor of shape [batch_size, num_anchors, + num_classes] representing the predicted unscaled logits for each class. + The function will compute sigmoid on this tensor internally. + target_tensor: A float tensor of shape [batch_size, num_anchors, + num_classes] representing a tensor with the 'splatted' keypoints, + possibly using a gaussian kernel. This function assumes that + the target is bounded between [0, 1]. + weights: a float tensor of shape, either [batch_size, num_anchors, + num_classes] or [batch_size, num_anchors, 1]. If the shape is + [batch_size, num_anchors, 1], all the classses are equally weighted. + Returns: + loss: a float tensor of shape [batch_size, num_anchors, num_classes] + representing the value of the loss function. + """ + + target_tensor = y_true + + is_present_tensor = tf.math.equal(target_tensor, 1.0) + prediction_tensor = tf.clip_by_value(tf.sigmoid(y_pred), + self._sigmoid_clip_value, + 1 - self._sigmoid_clip_value) + + positive_loss = (tf.math.pow((1 - prediction_tensor), self._alpha)* + tf.math.log(prediction_tensor)) + negative_loss = (tf.math.pow((1 - target_tensor), self._beta)* + tf.math.pow(prediction_tensor, self._alpha)* + tf.math.log(1 - prediction_tensor)) + + loss = -tf.where(is_present_tensor, positive_loss, negative_loss) + return loss From 6c628f1579fc60ac802df444d679a2f792d36d67 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 25 Feb 2021 11:55:28 -0500 Subject: [PATCH 043/132] Add log2 and log3 --- centernet/losses/penalty_reduced_logistic_focal_loss_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py index 7c16a904f..58bbe404f 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py @@ -3,6 +3,9 @@ from centernet import losses +LOG_2 = np.log(2) +LOG_3 = np.log(3) + class PenaltyReducedLogisticFocalLossTest(tf.test.TestCase): """Testing loss function from Equation (1) in [1]. [1]: https://arxiv.org/abs/1904.07850 From ac142858ff30cc0166dbf0437712ecbdc13105b1 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 25 Feb 2021 12:20:56 -0500 Subject: [PATCH 044/132] Temporary fix to different calling conventions in ODAPI loss and Keras loss --- .../penalty_reduced_logistic_focal_loss_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py index 58bbe404f..2f9da5954 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py @@ -11,8 +11,8 @@ class PenaltyReducedLogisticFocalLossTest(tf.test.TestCase): [1]: https://arxiv.org/abs/1904.07850 """ - def setUp(self): - super(PenaltyReducedLogisticFocalLossTest, self).setUp() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self._prediction = np.array([ # First batch [[1 / 2, 1 / 4, 3 / 4], @@ -37,7 +37,7 @@ def graph_fn(prediction, target): [[1.0], [1.0]], ]) loss = losses.PenaltyReducedLogisticFocalLoss(alpha=2.0, beta=0.5) - computed_value = loss(prediction, target, weights) + computed_value = loss.call(target, prediction) * weights return computed_value computed_value = graph_fn(self._prediction, self._target) expected_value = np.array([ @@ -55,7 +55,7 @@ def graph_fn(prediction, target): [0.2 * 9 / 16 * 2 * LOG_2, 1 / 9 * (LOG_3 - LOG_2), 4 / 9 * LOG_3]]]) - self.assertAllClose(computed_value, expected_value, rtol=1e-3, atol=1e-3) + self.assertAllClose(expected_value, computed_value, rtol=1e-3, atol=1e-3) def test_returns_correct_loss_weighted(self): def graph_fn(prediction, target): @@ -66,7 +66,7 @@ def graph_fn(prediction, target): loss = losses.PenaltyReducedLogisticFocalLoss(alpha=2.0, beta=0.5) - computed_value = loss(prediction, target, weights) + computed_value = loss.call(target, prediction) * weights return computed_value computed_value = graph_fn(self._prediction, self._target) expected_value = np.array([ @@ -85,7 +85,7 @@ def graph_fn(prediction, target): 0.0, 0.0]]]) - self.assertAllClose(computed_value, expected_value, rtol=1e-3, atol=1e-3) + self.assertAllClose(expected_value, computed_value, rtol=1e-3, atol=1e-3) if __name__ == '__main__': tf.test.main() From 4182d55cab8806111290dbea9a84fa1ae930f5f0 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 25 Feb 2021 12:33:37 -0500 Subject: [PATCH 045/132] More closely follow TFVision conventions --- centernet/losses/l1_localization_loss.py | 2 +- centernet/losses/l1_localization_loss_test.py | 2 +- .../losses/penalty_reduced_logistic_focal_loss.py | 4 ++-- .../losses/penalty_reduced_logistic_focal_loss_test.py | 10 ++++++---- centernet/tasks/centernet_object_detection.py | 4 ++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/centernet/losses/l1_localization_loss.py b/centernet/losses/l1_localization_loss.py index d9fb5fefe..62dacad8e 100644 --- a/centernet/losses/l1_localization_loss.py +++ b/centernet/losses/l1_localization_loss.py @@ -68,7 +68,7 @@ def __call__(self, y_true, y_pred, sample_weight=None): y_true, y_pred, weights=sample_weight, - reduction=tf.keras.losses.Reduction.NONE + reduction=self.reduction ) call = __call__ diff --git a/centernet/losses/l1_localization_loss_test.py b/centernet/losses/l1_localization_loss_test.py index eeed410ce..e85943857 100644 --- a/centernet/losses/l1_localization_loss_test.py +++ b/centernet/losses/l1_localization_loss_test.py @@ -7,7 +7,7 @@ class L1LocalizationLossTest(tf.test.TestCase): def test_returns_correct_loss(self): def graph_fn(): - loss = losses.L1LocalizationLoss() + loss = losses.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) pred = [[0.1, 0.2], [0.7, 0.5]] target = [[0.9, 1.0], [0.1, 0.4]] diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss.py b/centernet/losses/penalty_reduced_logistic_focal_loss.py index 2f5cb5bad..b17c66733 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss.py @@ -9,7 +9,7 @@ class PenaltyReducedLogisticFocalLoss(tf.keras.losses.Loss): [1]: https://arxiv.org/abs/1904.07850 """ - def __init__(self, alpha=2.0, beta=4.0, sigmoid_clip_value=1e-4): + def __init__(self, *args, alpha=2.0, beta=4.0, sigmoid_clip_value=1e-4, **kwargs): """Constructor. Args: alpha: Focussing parameter of the focal loss. Increasing this will @@ -22,7 +22,7 @@ def __init__(self, alpha=2.0, beta=4.0, sigmoid_clip_value=1e-4): self._alpha = alpha self._beta = beta self._sigmoid_clip_value = sigmoid_clip_value - super(PenaltyReducedLogisticFocalLoss, self).__init__() + super(PenaltyReducedLogisticFocalLoss, self).__init__(*args, **kwargs) def call(self, y_true, y_pred): """Compute loss function. diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py index 2f9da5954..93d2a4681 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py @@ -36,8 +36,9 @@ def graph_fn(prediction, target): [[1.0], [1.0]], [[1.0], [1.0]], ]) - loss = losses.PenaltyReducedLogisticFocalLoss(alpha=2.0, beta=0.5) - computed_value = loss.call(target, prediction) * weights + loss = losses.PenaltyReducedLogisticFocalLoss( + alpha=2.0, beta=0.5, reduction=tf.keras.losses.Reduction.NONE) + computed_value = loss(target, prediction, weights) return computed_value computed_value = graph_fn(self._prediction, self._target) expected_value = np.array([ @@ -64,9 +65,10 @@ def graph_fn(prediction, target): [[1.0, 1.0, 1.0], [0.0, 0.0, 0.0]], ]) - loss = losses.PenaltyReducedLogisticFocalLoss(alpha=2.0, beta=0.5) + loss = losses.PenaltyReducedLogisticFocalLoss( + alpha=2.0, beta=0.5, reduction=tf.keras.losses.Reduction.NONE) - computed_value = loss.call(target, prediction) * weights + computed_value = loss(target, prediction, weights) return computed_value computed_value = graph_fn(self._prediction, self._target) expected_value = np.array([ diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py index e072533db..6818bb1bf 100644 --- a/centernet/tasks/centernet_object_detection.py +++ b/centernet/tasks/centernet_object_detection.py @@ -38,7 +38,7 @@ def build_losses(self, outputs, labels, aux_losses=None): flattened_ct_heatmaps = utils._flatten_spatial_dimensions(labels['ct_heatmaps']) num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) #gt_weights_list here shouldn't be tag_masks here - object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss() + object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) # Loop through each feature output head. for pred in outputs['ct_heatmaps']: pred = utils._flatten_spatial_dimensions(pred) @@ -50,7 +50,7 @@ def build_losses(self, outputs, labels, aux_losses=None): metric_dict['ct_loss'] = center_loss #localization loss for offset and scale loss - localization_loss_fn = l1_localization_loss.L1LocalizationLoss() + localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) for scale_pred, offset_pred in zip(outputs['ct_size'], outputs['ct_offset']): # Compute the scale loss. scale_pred = utils.get_batch_predictions_from_indices( From c38bc40257442f7ddd2e837ea47e2fbd4f27947b Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 25 Feb 2021 12:39:19 -0500 Subject: [PATCH 046/132] get_config in penalty_reduced_logistic_focal_loss --- centernet/losses/l1_localization_loss.py | 2 +- centernet/losses/penalty_reduced_logistic_focal_loss.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/centernet/losses/l1_localization_loss.py b/centernet/losses/l1_localization_loss.py index 62dacad8e..bb9e772dc 100644 --- a/centernet/losses/l1_localization_loss.py +++ b/centernet/losses/l1_localization_loss.py @@ -68,7 +68,7 @@ def __call__(self, y_true, y_pred, sample_weight=None): y_true, y_pred, weights=sample_weight, - reduction=self.reduction + reduction=self._get_reduction() ) call = __call__ diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss.py b/centernet/losses/penalty_reduced_logistic_focal_loss.py index b17c66733..8dca75efb 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss.py @@ -59,3 +59,12 @@ def call(self, y_true, y_pred): loss = -tf.where(is_present_tensor, positive_loss, negative_loss) return loss + + def get_config(self): + """Returns the config dictionary for a `Loss` instance.""" + return { + 'alpha': self._alpha, + 'beta': self._beta, + 'sigmoid_clip_value': self._sigmoid_clip_value, + **super().get_config() + } From 6011df2113d1576d352f785e79d605656e103aa9 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 25 Feb 2021 13:04:58 -0500 Subject: [PATCH 047/132] Start using tf.Variable --- centernet/dataloaders/centernet_input.py | 34 +++++++++++++----------- centernet/utils/groundtruth.py | 10 ++++--- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index c5ff143db..cebec0d9d 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -6,10 +6,12 @@ class CenterNetParser(parser.Parser): def __init__( self, num_classes: int, + max_num_instances: int, gaussian_iou: float ): - self.num_classes = num_classes - self.gaussian_io = gaussian_iou + self._num_classes = num_classes + self._max_num_instances = max_num_instances + self._gaussian_iou = gaussian_iou def _parse_train_data(self, decoded_tensors): """Generates images and labels that are usable for model training. @@ -21,16 +23,16 @@ def _parse_train_data(self, decoded_tensors): images: the image tensor. labels: a dict of Tensors that contains labels. """ - tl_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) - br_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) - ct_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) - tl_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) - br_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) - ct_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) - tl_tags = tf.zeros((max_tag_len), dtype=tf.int64) - br_tags = tf.zeros((max_tag_len), dtype=tf.int64) - ct_tags = tf.zeros((max_tag_len), dtype=tf.int64) - tag_masks = tf.zeros((max_tag_len), dtype=tf.uint8) + tl_heatmaps = tf.Variable(tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32)) + br_heatmaps = tf.Variable(tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32)) + ct_heatmaps = tf.Variable(tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32)) + tl_regrs = tf.Variable(tf.zeros((self._max_num_instances, 2), dtype=tf.float32)) + br_regrs = tf.Variable(tf.zeros((self._max_num_instances, 2), dtype=tf.float32)) + ct_regrs = tf.Variable(tf.zeros((self._max_num_instances, 2), dtype=tf.float32)) + tl_tags = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.int64)) + br_tags = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.int64)) + ct_tags = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.int64)) + tag_masks = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.uint8)) # TODO: input size, output size image = decoded_tensors["image"] @@ -72,13 +74,13 @@ def _parse_train_data(self, decoded_tensors): height = math.ceil(height * height_ratio) if gaussian_rad == -1: - radius = gaussian_radius((height, width), self.gaussian_iou) + radius = gaussian_radius((height, width), self._gaussian_iou) radius = max(0, int(radius)) else: radius = gaussian_rad - draw_gaussian(tl_heatmaps[category], [xtl, ytl], radius) - draw_gaussian(br_heatmaps[category], [xbr, ybr], radius) - draw_gaussian(ct_heatmaps[category], [xct, yct], radius, delte=5) + draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) + draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) + draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) else: tl_heatmaps[category, ytl, xtl] = 1 diff --git a/centernet/utils/groundtruth.py b/centernet/utils/groundtruth.py index 91f20d10f..1e74e105b 100644 --- a/centernet/utils/groundtruth.py +++ b/centernet/utils/groundtruth.py @@ -72,14 +72,15 @@ def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) return tf.math.exp(exponent) -def draw_gaussian(heatmap, center, radius, k=1): +def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): """ Draws a gaussian heatmap around a center point given a radius. Params: heatmap (tf.Tensor): heatmap placeholder to fill + category (int): class ID of the object being drawn center (int): integer for center of gaussian radius (int): integer for radius of gaussian - k (int): scaling factor for gaussian + scaling_factor (int): scaling factor for gaussian """ diameter = 2 * radius + 1 @@ -92,8 +93,9 @@ def draw_gaussian(heatmap, center, radius, k=1): left, right = min(x, radius), min(width - x, radius + 1) top, bottom = min(y, radius), min(height - y, radius + 1) - masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + masked_heatmap = heatmap[category, y - top:y + bottom, x - left:x + right] masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] # TODO: make sure this replicates original functionality # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * k) + masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * scaling_factor) + heatmap.assign(masked_heatmap) From 6e796a3c495748d6132ef027ecb1b9fdbbd258fe Mon Sep 17 00:00:00 2001 From: anivegesana Date: Thu, 25 Feb 2021 13:48:52 -0500 Subject: [PATCH 048/132] Remove variable --- centernet/dataloaders/centernet_input.py | 55 ++++++++++++++---------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index cebec0d9d..ff1d408d3 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -23,16 +23,16 @@ def _parse_train_data(self, decoded_tensors): images: the image tensor. labels: a dict of Tensors that contains labels. """ - tl_heatmaps = tf.Variable(tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32)) - br_heatmaps = tf.Variable(tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32)) - ct_heatmaps = tf.Variable(tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32)) - tl_regrs = tf.Variable(tf.zeros((self._max_num_instances, 2), dtype=tf.float32)) - br_regrs = tf.Variable(tf.zeros((self._max_num_instances, 2), dtype=tf.float32)) - ct_regrs = tf.Variable(tf.zeros((self._max_num_instances, 2), dtype=tf.float32)) - tl_tags = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.int64)) - br_tags = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.int64)) - ct_tags = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.int64)) - tag_masks = tf.Variable(tf.zeros((self._max_num_instances), dtype=tf.uint8)) + tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + tl_regrs = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + br_regrs = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + ct_regrs = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + tl_tags = tf.zeros((self._max_num_instances), dtype=tf.int64) + br_tags = tf.zeros((self._max_num_instances), dtype=tf.int64) + ct_tags = tf.zeros((self._max_num_instances), dtype=tf.int64) + tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) # TODO: input size, output size image = decoded_tensors["image"] @@ -78,21 +78,30 @@ def _parse_train_data(self, decoded_tensors): radius = max(0, int(radius)) else: radius = gaussian_rad - draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) - draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) - draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) + tl_heatmaps = draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) + br_heatmaps = draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) + ct_heatmaps = draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) else: - tl_heatmaps[category, ytl, xtl] = 1 - br_heatmaps[category, ybr, xbr] = 1 - ct_heatmaps[category, yct, xct] = 1 - - tl_regrs[tag_ind, :] = [fxtl - xtl, fytl - ytl] - br_regrs[tag_ind, :] = [fxbr - xbr, fybr - ybr] - ct_regrs[tag_ind, :] = [fxct - xct, fyct - yct] - tl_tags[tag_ind] = ytl * output_size[1] + xtl - br_tags[tag_ind] = ybr * output_size[1] + xbr - ct_tags[tag_ind] = yct * output_size[1] + xct + # tl_heatmaps[category, ytl, xtl] = 1 + # br_heatmaps[category, ybr, xbr] = 1 + # ct_heatmaps[category, yct, xct] = 1 + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ytl, xtl]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, ytl, xtl]], [1]) + + # tl_regrs[tag_ind, :] = [fxtl - xtl, fytl - ytl] + # br_regrs[tag_ind, :] = [fxbr - xbr, fybr - ybr] + # ct_regrs[tag_ind, :] = [fxct - xct, fyct - yct] + # tl_tags[tag_ind] = ytl * output_size[1] + xtl + # br_tags[tag_ind] = ybr * output_size[1] + xbr + # ct_tags[tag_ind] = yct * output_size[1] + xct + tl_regrs = tf.tensor_scatter_nd_update(tl_regrs, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + br_regrs = tf.tensor_scatter_nd_update(br_regrs, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + ct_regrs = tf.tensor_scatter_nd_update(ct_regrs, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) + tl_tags = tf.tensor_scatter_nd_update(tl_tags, [[tag_ind]], [ytl * output_size[1] + xtl]) + br_tags = tf.tensor_scatter_nd_update(br_tags, [[tag_ind]], [ybr * output_size[1] + xbr]) + ct_tags = tf.tensor_scatter_nd_update(ct_tags, [[tag_ind]], [yct * output_size[1] + xct]) labels = { 'tl_tags': tl_tags, From ab247a4f3e143c0e3ada11ab87e5a6c9c6609431 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 25 Feb 2021 16:25:06 -0500 Subject: [PATCH 049/132] restructured decoder to functional style --- .../modeling/decoders/centernet_decoder.py | 30 ++++++++----------- centernet/tasks/centernet_test.py | 1 + 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index 94166a9e6..f09bad0d0 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -10,6 +10,7 @@ class CenterNetDecoder(tf.keras.Model): def __init__(self, task_outputs: dict, heatmap_bias: float = -2.19, + input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 256]), **kwargs): """ Args: @@ -24,30 +25,23 @@ def __init__(self, and the respective output tensor """ - self._task_outputs = task_outputs - self._heatmap_bias = heatmap_bias + input = tf.keras.layers.Input(shape=input_specs.shape[1:]) - super().__init__(**kwargs) + outputs = {} - def build(self, input_shape): - self.out_layers = {} - for key in self._task_outputs: - num_filters = self._task_outputs[key] + for key in task_outputs: + num_filters = task_outputs[key] bias = 0 if 'heatmaps' in key: - bias = self._heatmap_bias - - self.out_layers[key] = CenterNetDecoderConv(output_filters=num_filters, - name=key, bias_init=bias) + bias = heatmap_bias - super().build(input_shape) + outputs[key] = CenterNetDecoderConv(output_filters=num_filters, + name=key, bias_init=bias)(input) + + super().__init__(inputs=input, outputs=outputs, **kwargs) - def call(self, x): - outputs = {} - for key in self.out_layers: - outputs[key] = self.out_layers[key](x) - - return outputs + self._task_outputs = task_outputs + self._heatmap_bias = heatmap_bias def get_config(self): layer_config = { diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index caa01c6a3..84125a79f 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -16,6 +16,7 @@ def testCenterNetTask(self): config = exp_cfg.CenterNetTask() task = CenterNetTask(config) model = task.build_model() + out = model(tf.zeros((3, 512, 512, 3))) model.summary() From da94c3e2c3c74b48169c59f0e612d4e9ac3bf58e Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 25 Feb 2021 16:30:06 -0500 Subject: [PATCH 050/132] updated centernet task test --- centernet/tasks/centernet_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index 84125a79f..141b5ca25 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -13,7 +13,8 @@ class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): def testCenterNetTask(self): - config = exp_cfg.CenterNetTask() + model_config = exp_cfg.CenterNet(input_size=[512, 512, 3]) + config = exp_cfg.CenterNetTask(model=model_config) task = CenterNetTask(config) model = task.build_model() out = model(tf.zeros((3, 512, 512, 3))) From e5d7d87b982d225cde2028ba3ecedf0293da67a2 Mon Sep 17 00:00:00 2001 From: Feny Patel <54478610+patel996@users.noreply.github.com> Date: Fri, 26 Feb 2021 09:54:19 -0600 Subject: [PATCH 051/132] Test object detection losses --- .../tasks/centernet_object_detection_test.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 centernet/tasks/centernet_object_detection_test.py diff --git a/centernet/tasks/centernet_object_detection_test.py b/centernet/tasks/centernet_object_detection_test.py new file mode 100644 index 000000000..fbb0998a4 --- /dev/null +++ b/centernet/tasks/centernet_object_detection_test.py @@ -0,0 +1,47 @@ +import tensorflow as tf +import centernet_object_detection as centernet_object_detection +import centernet.tasks as tasks +import centernet.utils as utils + +class ObjectDetectionTest(tf.test.TestCase): + + def generate_heatmaps(self, dectections): + + for ind, detection in enumerate(detections): + category = int(detection[-1]) - 1 + #category = 0 + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. + + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + xtl = int(fxtl) + ytl = int(fytl) + xbr = int(fxbr) + ybr = int(fybr) + xct = int(fxct) + yct = int(fyct) + + if gaussian_bump: + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + width = math.ceil(width * width_ratio) + height = math.ceil(height * height_ratio) + + if gaussian_rad == -1: + radius = utils.gaussian_radius((height, width), gaussian_iou) + radius = max(0, int(radius)) + else: + radius = gaussian_rad + + utils.draw_gaussian(tl_heatmaps[category], [xtl, ytl], radius) + utils.draw_gaussian(br_heatmaps[category], [xbr, ybr], radius) + utils.draw_gaussian(ct_heatmaps[category], [xct, yct], radius, delte = 5) \ No newline at end of file From 25721d52eda675fa558ca302bd256bae493cf103 Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Fri, 26 Feb 2021 11:47:04 -0800 Subject: [PATCH 052/132] mine --- centernet/ops/preprocessing_ops_test.py | 85 +++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/centernet/ops/preprocessing_ops_test.py b/centernet/ops/preprocessing_ops_test.py index ac97da7a0..e3a8ed304 100644 --- a/centernet/ops/preprocessing_ops_test.py +++ b/centernet/ops/preprocessing_ops_test.py @@ -30,6 +30,7 @@ def test_max_distance_for_overlap(self): min_iou = rng.uniform(0.1, 1.0, size=n_samples) max_dist = preprocessing_ops.gaussian_radius((height, width), min_iou) + xmin1 = np.zeros(n_samples) ymin1 = np.zeros(n_samples) xmax1 = np.zeros(n_samples) + width @@ -53,5 +54,89 @@ def test_max_distance_for_overlap_centernet(self): self.assertAlmostEqual(2.807764064, distance.numpy()) + + + + def coordinates_to_heatmap(y_grid, + x_grid, + y_coordinates, + x_coordinates, + sigma, + channel_onehot, + channel_weights=None, + sparse=False): + """Returns the heatmap targets from a set of point coordinates. + This function maps a set of point coordinates to the output heatmap image + applied using a Gaussian kernel. Note that this function be can used by both + object detection and keypoint estimation tasks. For object detection, the + "channel" refers to the object class. For keypoint estimation, the "channel" + refers to the number of keypoint types. + Args: + y_grid: A 2D tensor with shape [height, width] which contains the grid + y-coordinates given in the (output) image dimensions. + x_grid: A 2D tensor with shape [height, width] which contains the grid + x-coordinates given in the (output) image dimensions. + y_coordinates: A 1D tensor with shape [num_instances] representing the + y-coordinates of the instances in the output space coordinates. + x_coordinates: A 1D tensor with shape [num_instances] representing the + x-coordinates of the instances in the output space coordinates. + sigma: A 1D tensor with shape [num_instances] representing the standard + deviation of the Gaussian kernel to be applied to the point. + channel_onehot: A 2D tensor with shape [num_instances, num_channels] + representing the one-hot encoded channel labels for each point. + channel_weights: A 1D tensor with shape [num_instances] corresponding to the + weight of each instance. + sparse: bool, indicating whether or not to use the sparse implementation + of the function. The sparse version scales better with number of channels, + but in some cases is known to cause OOM error. See (b/170989061). + Returns: + heatmap: A tensor of size [height, width, num_channels] representing the + heatmap. Output (height, width) match the dimensions of the input grids. + """ + + # if sparse: + # return _coordinates_to_heatmap_sparse( + # y_grid, x_grid, y_coordinates, x_coordinates, sigma, channel_onehot, + # channel_weights) + # else: + return _coordinates_to_heatmap_dense( + y_grid, x_grid, y_coordinates, x_coordinates, sigma, channel_onehot, + channel_weights) + + + def _coordinates_to_heatmap_dense(y_grid, x_grid, y_coordinates, x_coordinates, + sigma, channel_onehot, channel_weights=None): + """Dense version of coordinates to heatmap that uses an outer product.""" + num_instances, num_channels = ( + shape_utils.combined_static_and_dynamic_shape(channel_onehot)) + + x_grid = tf.expand_dims(x_grid, 2) + y_grid = tf.expand_dims(y_grid, 2) + # The raw center coordinates in the output space. + x_diff = x_grid - tf.math.floor(x_coordinates) + y_diff = y_grid - tf.math.floor(y_coordinates) + squared_distance = x_diff**2 + y_diff**2 + + gaussian_map = tf.exp(-squared_distance / (2 * sigma * sigma)) + + reshaped_gaussian_map = tf.expand_dims(gaussian_map, axis=-1) + reshaped_channel_onehot = tf.reshape(channel_onehot, + (1, 1, num_instances, num_channels)) + gaussian_per_box_per_class_map = ( + reshaped_gaussian_map * reshaped_channel_onehot) + + if channel_weights is not None: + reshaped_weights = tf.reshape(channel_weights, (1, 1, num_instances, 1)) + gaussian_per_box_per_class_map *= reshaped_weights + + # Take maximum along the "instance" dimension so that all per-instance + # heatmaps of the same class are merged together. + heatmap = tf.reduce_max(gaussian_per_box_per_class_map, axis=2) + + # Maximum of an empty tensor is -inf, the following is to avoid that. + heatmap = tf.maximum(heatmap, 0) + + return tf.stop_gradient(heatmap) + if __name__ == '__main__': tf.test.main() From 95389241e84c2e2204e7a2a788f356cc03f856bb Mon Sep 17 00:00:00 2001 From: anivegesana Date: Mon, 1 Mar 2021 11:49:40 -0500 Subject: [PATCH 053/132] Visually show heatmaps --- .../tasks/centernet_object_detection_test.py | 140 +++++++++++++----- 1 file changed, 100 insertions(+), 40 deletions(-) diff --git a/centernet/tasks/centernet_object_detection_test.py b/centernet/tasks/centernet_object_detection_test.py index fbb0998a4..69a5f23e3 100644 --- a/centernet/tasks/centernet_object_detection_test.py +++ b/centernet/tasks/centernet_object_detection_test.py @@ -1,47 +1,107 @@ import tensorflow as tf -import centernet_object_detection as centernet_object_detection +import numpy as np import centernet.tasks as tasks import centernet.utils as utils -class ObjectDetectionTest(tf.test.TestCase): +def gaussian2D(shape, sigma=1): + m, n = [(ss - 1.) / 2. for ss in shape] + y, x = np.ogrid[-m:m+1,-n:n+1] + + h = np.exp(-(x * x + y * y) / (2 * sigma * sigma)) + h[h < np.finfo(h.dtype).eps * h.max()] = 0 + return h + +def draw_gaussian(heatmap, center, radius, k=1, delte=6): + diameter = 2 * radius + 1 + gaussian = gaussian2D((diameter, diameter), sigma=diameter / delte) + + x, y = center + + height, width = heatmap.shape[0:2] + + left, right = min(x, radius), min(width - x, radius + 1) + top, bottom = min(y, radius), min(height - y, radius + 1) + + masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] + np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) + +def gaussian_radius(det_size, min_overlap): + height, width = det_size + + a1 = 1 + b1 = (height + width) + c1 = width * height * (1 - min_overlap) / (1 + min_overlap) + sq1 = np.sqrt(b1 ** 2 - 4 * a1 * c1) + r1 = (b1 + sq1) / 2 + + a2 = 4 + b2 = 2 * (height + width) + c2 = (1 - min_overlap) * width * height + sq2 = np.sqrt(b2 ** 2 - 4 * a2 * c2) + r2 = (b2 + sq2) / 2 + a3 = 4 * min_overlap + b3 = -2 * min_overlap * (height + width) + c3 = (min_overlap - 1) * width * height + sq3 = np.sqrt(b3 ** 2 - 4 * a3 * c3) + r3 = (b3 + sq3) / 2 + return min(r1, r2, r3) + +def generate_heatmaps(batch_size, categories, output_size, detections, gaussian_iou=0.7): + tl_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) + br_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) + ct_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) + + for b_ind, detection_batches in enumerate(detections): + for ind, detection in enumerate(detection_batches): + category = int(detection[-1]) + #category = 0 + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. + + xtl = int(xtl) + ytl = int(ytl) + xbr = int(xbr) + ybr = int(ybr) + xct = int(xct) + yct = int(yct) + + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + radius = gaussian_radius((height, width), gaussian_iou) + radius = max(0, int(radius)) + + draw_gaussian(tl_heatmaps[b_ind, category], [xtl, ytl], radius) + draw_gaussian(br_heatmaps[b_ind, category], [xbr, ybr], radius) + draw_gaussian(ct_heatmaps[b_ind, category], [xct, yct], radius, delte = 5) + + return tl_heatmaps, br_heatmaps, ct_heatmaps + +class ObjectDetectionTest(tf.test.TestCase): def generate_heatmaps(self, dectections): + detections = [[ + (10, 30, 15, 17, 0) + ]] + tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) + pass - for ind, detection in enumerate(detections): - category = int(detection[-1]) - 1 - #category = 0 - - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] - xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. - - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - xtl = int(fxtl) - ytl = int(fytl) - xbr = int(fxbr) - ybr = int(fybr) - xct = int(fxct) - yct = int(fyct) - - if gaussian_bump: - width = detection[2] - detection[0] - height = detection[3] - detection[1] - - width = math.ceil(width * width_ratio) - height = math.ceil(height * height_ratio) - - if gaussian_rad == -1: - radius = utils.gaussian_radius((height, width), gaussian_iou) - radius = max(0, int(radius)) - else: - radius = gaussian_rad - - utils.draw_gaussian(tl_heatmaps[category], [xtl, ytl], radius) - utils.draw_gaussian(br_heatmaps[category], [xbr, ybr], radius) - utils.draw_gaussian(ct_heatmaps[category], [xct, yct], radius, delte = 5) \ No newline at end of file +if __name__ == '__main__': + # This code is for visualization + import matplotlib.pyplot as plt + detections = [[ + (10, 300, 15, 370, 0), + (100, 300, 150, 370, 0), + (200, 100, 15, 170, 0), + ], + # more images can go here if you like + ] + tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) + # ct_heatmaps[batch_id, class_id, ...] + plt.imshow(ct_heatmaps[0, 0, ...]) + plt.show() + # This is to run the test + # tf.test.main() From 8e52e3a93d5ce1f7ca09b4e57f43eae55692be86 Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Mon, 1 Mar 2021 12:24:48 -0800 Subject: [PATCH 054/132] added build_inputs template and fixed centernet parser --- centernet/dataloaders/centernet_input.py | 2 +- centernet/tasks/centernet.py | 373 +++++++++++++++++++++++ 2 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 centernet/tasks/centernet.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index c5ff143db..f7611819a 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -101,7 +101,7 @@ def _parse_train_data(self, decoded_tensors): 'ct_heatmaps': ct_heatmaps, 'tag_masks': tag_masks, 'tl_regrs': tl_regrs, - 'br_regrs', br_regrs, + 'br_regrs': br_regrs, 'ct_regrs': ct_regrs, } return image, labels diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py new file mode 100644 index 000000000..708020355 --- /dev/null +++ b/centernet/tasks/centernet.py @@ -0,0 +1,373 @@ +import tensorflow as tf +from tensorflow.keras.mixed_precision import experimental as mixed_precision + +from absl import logging +from official.core import base_task +from official.core import input_reader +from official.core import task_factory +from yolo.configs import yolo as exp_cfg + +from official.vision.beta.evaluation import coco_evaluator + +from yolo.dataloaders import yolo_input +from yolo.dataloaders.decoders import tfds_coco_decoder +from yolo.ops.kmeans_anchors import BoxGenInputReader +from yolo.ops.box_ops import xcycwh_to_yxyx + +from centernet.dataloaders import centernet_input + + +@task_factory.register_task_cls(exp_cfg.YoloTask) +class YoloTask(base_task.Task): + """A single-replica view of training procedure. + RetinaNet task provides artifacts for training/evalution procedures, including + loading/iterating over Datasets, initializing the model, calculating the loss, + post-processing, and customized metrics with reduction. + """ + + def __init__(self, params, logging_dir: str = None): + super().__init__(params, logging_dir) + self._loss_dict = None + self._num_boxes = None + self._anchors_built = False + + self._masks = None + self._path_scales = None + self._x_y_scales = None + self.coco_metric = None + return + + def build_model(self): + """get an instance of Yolo v3 or v4""" + from yolo.modeling.Yolo import build_yolo + params = self.task_config.train_data + model_base_cfg = self.task_config.model + l2_weight_decay = self.task_config.weight_decay / 2.0 + + masks, path_scales, xy_scales = self._get_masks() + self._get_boxes(gen_boxes=params.is_training) + + input_specs = tf.keras.layers.InputSpec(shape=[None] + + model_base_cfg.input_size) + l2_regularizer = ( + tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) + + model, losses = build_yolo(input_specs, model_base_cfg, l2_regularizer, + masks, xy_scales, path_scales) + self._loss_dict = losses + return model + + def build_inputs(self, params, input_context=None): + """Build input dataset.""" + decoder = tfds_coco_decoder.MSCOCODecoder() + """ + decoder_cfg = params.decoder.get() + if params.decoder.type == 'simple_decoder': + decoder = tf_example_decoder.TfExampleDecoder( + regenerate_source_id=decoder_cfg.regenerate_source_id) + elif params.decoder.type == 'label_map_decoder': + decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap( + label_map=decoder_cfg.label_map, + regenerate_source_id=decoder_cfg.regenerate_source_id) + else: + raise ValueError('Unknown decoder type: {}!'.format(params.decoder.type)) + """ + + model = self.task_config.model + + masks, path_scales, xy_scales = self._get_masks() + anchors = self._get_boxes(gen_boxes=params.is_training) + + print(masks, path_scales, xy_scales) + parser = centernet_input.CenterNetParser( + + num_classes=model.num_classes, + gaussian_iou=model.gaussian_iou + ) + + if params.is_training: + post_process_fn = parser.postprocess_fn() + else: + post_process_fn = None + + reader = input_reader.InputReader( + params, + dataset_fn=tf.data.TFRecordDataset, + decoder_fn=decoder.decode, + parser_fn=parser.parse_fn(params.is_training), + postprocess_fn=post_process_fn) + dataset = reader.read(input_context=input_context) + return dataset + + def build_losses(self, outputs, labels, aux_losses=None): + loss = 0.0 + loss_box = 0.0 + loss_conf = 0.0 + loss_class = 0.0 + metric_dict = dict() + + grid = labels['grid_form'] + for key in outputs.keys(): + # _loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](labels, outputs[key]) + _loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](grid[key], outputs[key]) + #_loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](labels[key], outputs[key]) + loss += _loss + loss_box += _loss_box + loss_conf += _loss_conf + loss_class += _loss_class + metric_dict[f"recall50_{key}"] = tf.stop_gradient(_recall50) + metric_dict[f"avg_iou_{key}"] = tf.stop_gradient(_avg_iou) + + metric_dict['box_loss'] = loss_box + metric_dict['conf_loss'] = loss_conf + metric_dict['class_loss'] = loss_class + + return loss, metric_dict + + def build_metrics(self, training=True): + #return super().build_metrics(training=training) + if not training: + self.coco_metric = coco_evaluator.COCOEvaluator( + annotation_file=self.task_config.annotation_file, + include_mask=False, + need_rescale_bboxes=False, + per_category_metrics=self._task_config.per_category_metrics) + return [] + + def train_step(self, inputs, model, optimizer, metrics=None): + #get the data point + image, label = inputs + num_replicas = tf.distribute.get_strategy().num_replicas_in_sync + with tf.GradientTape() as tape: + # compute a prediction + # cast to float32 + y_pred = model(image, training=True) + loss, metrics = self.build_losses(y_pred['raw_output'], label) + scaled_loss = loss / num_replicas + + # scale the loss for numerical stability + if isinstance(optimizer, mixed_precision.LossScaleOptimizer): + scaled_loss = optimizer.get_scaled_loss(scaled_loss) + # compute the gradient + train_vars = model.trainable_variables + gradients = tape.gradient(scaled_loss, train_vars) + # get unscaled loss if the scaled_loss was used + if isinstance(optimizer, mixed_precision.LossScaleOptimizer): + gradients = optimizer.get_unscaled_gradients(gradients) + if self.task_config.gradient_clip_norm > 0.0: + gradients, _ = tf.clip_by_global_norm(gradients, + self.task_config.gradient_clip_norm) + optimizer.apply_gradients(zip(gradients, train_vars)) + + #custom metrics + logs = {'loss': loss} + logs.update(metrics) + + #tf.print("loss: ", logs["loss"], end = "\n") + tf.print(logs, end='\n') + + ret = '\033[F' * (len(logs.keys()) + 1) + tf.print(ret, end='\n') + + return logs + + def validation_step(self, inputs, model, metrics=None): + #get the data point + image, label = inputs + + # computer detivative and apply gradients + y_pred = model(image, training=False) + loss, metrics = self.build_losses(y_pred['raw_output'], label) + + # #custom metrics + loss_metrics = {'loss': loss} + loss_metrics.update(metrics) + label['boxes'] = xcycwh_to_yxyx(label['bbox']) + del label['bbox'] + + coco_model_outputs = { + 'detection_boxes': y_pred['bbox'], + 'detection_scores': y_pred['confidence'], + 'detection_classes': y_pred['classes'], + 'num_detections': tf.shape(y_pred['bbox'])[:-1], + 'source_id': label['source_id'], + } + + loss_metrics.update({self.coco_metric.name: (label, coco_model_outputs)}) + return loss_metrics + + def aggregate_logs(self, state=None, step_outputs=None): + #return super().aggregate_logs(state=state, step_outputs=step_outputs) + if not state: + self.coco_metric.reset_states() + state = self.coco_metric + self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], + step_outputs[self.coco_metric.name][1]) + return state + + def reduce_aggregated_logs(self, aggregated_logs): + #return super().reduce_aggregated_logsI(aggregated_logs) + return self.coco_metric.result() + + @property + def anchors(self): + return self.task_config.model.boxes + + def _get_boxes(self, gen_boxes=True): + # gen_boxes = params.is_training + if gen_boxes and self.task_config.model.boxes == None and not self._anchors_built: + # must save the boxes! + model_base_cfg = self.task_config.model + self._num_boxes = (model_base_cfg.max_level - model_base_cfg.min_level + + 1) * model_base_cfg.boxes_per_scale + decoder = tfds_coco_decoder.MSCOCODecoder() + reader = BoxGenInputReader( + params, + dataset_fn=tf.data.TFRecordDataset, + decoder_fn=decoder.decode, + parser_fn=None) + anchors = reader.read( + k=9, image_width=params.parser.image_w, input_context=input_context) + self.task_config.model.set_boxes(anchors) + self._anchors_built = True + del reader + + return self.task_config.model.boxes + + def _get_masks(self, + xy_exponential=False, + exp_base=2, + xy_scale_base='default_value'): + start = 0 + boxes = {} + path_scales = {} + scale_x_y = {} + + if xy_scale_base == 'default_base': + xy_scale_base = 0.05 + xy_scale_base = xy_scale_base / ( + self._boxes_per_level * (self._max_level - self._min_level + 1) - 1) + elif xy_scale_base == 'default_value': + xy_scale_base = 0.00625 + + params = self.task_config.model + + if self._masks == None or self._path_scales == None or self._x_y_scales == None: + for i in range(params.min_level, params.max_level + 1): + boxes[str(i)] = list(range(start, params.boxes_per_scale + start)) + path_scales[str(i)] = 2**i + if xy_exponential: + scale_x_y[str(i)] = 1.0 + xy_scale_base * (exp_base**i) + else: + scale_x_y[str(i)] = 1.0 + start += params.boxes_per_scale + + self._masks = boxes + self._path_scales = path_scales + self._x_y_scales = scale_x_y + + return self._masks, self._path_scales, self._x_y_scales + + def initialize(self, model: tf.keras.Model): + if self.task_config.load_darknet_weights: + from yolo.utils import DarkNetConverter + from yolo.utils._darknet2tf.load_weights import split_converter + from yolo.utils._darknet2tf.load_weights2 import load_weights_backbone + from yolo.utils._darknet2tf.load_weights2 import load_weights_neck + from yolo.utils._darknet2tf.load_weights2 import load_head + from yolo.utils._darknet2tf.load_weights2 import load_weights_prediction_layers + from yolo.utils.downloads.file_manager import download + + weights_file = self.task_config.model.darknet_weights_file + config_file = self.task_config.model.darknet_weights_cfg + + if ('cache' not in weights_file and 'cache' not in config_file): + list_encdec = DarkNetConverter.read(config_file, weights_file) + else: + import os + path = os.path.abspath('cache') + if (not os.path.isdir(path)): + os.mkdir(path) + + cfg = f"{path}/cfg/{config_file.split('/')[-1]}" + if not os.path.isfile(cfg): + download(config_file.split('/')[-1]) + + wgt = f"{path}/weights/{weights_file.split('/')[-1]}" + if not os.path.isfile(wgt): + download(weights_file.split('/')[-1]) + + list_encdec = DarkNetConverter.read(cfg, wgt) + + splits = model.backbone._splits + if 'neck_split' in splits.keys(): + encoder, neck, decoder = split_converter(list_encdec, + splits['backbone_split'], + splits['neck_split']) + else: + encoder, decoder = split_converter(list_encdec, + splits['backbone_split']) + neck = None + + load_weights_backbone(model.backbone, encoder) + model.backbone.trainable = False + + if self.task_config.darknet_load_decoder: + if neck != None: + load_weights_neck(model.decoder.neck, neck) + model.decoder.neck.trainable = False + cfgheads = load_head(model.decoder.head, decoder) + model.decoder.head.trainable = False + load_weights_prediction_layers(cfgheads, model.head) + model.head.trainable = False + else: + """Loading pretrained checkpoint.""" + if not self.task_config.init_checkpoint: + return + + ckpt_dir_or_file = self.task_config.init_checkpoint + if tf.io.gfile.isdir(ckpt_dir_or_file): + ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) + + # Restoring checkpoint. + if self.task_config.init_checkpoint_modules == 'all': + ckpt = tf.train.Checkpoint(**model.checkpoint_items) + status = ckpt.restore(ckpt_dir_or_file) + status.assert_consumed() + elif self.task_config.init_checkpoint_modules == 'backbone': + ckpt = tf.train.Checkpoint(backbone=model.backbone) + status = ckpt.restore(ckpt_dir_or_file) + status.expect_partial().assert_existing_objects_matched() + else: + assert "Only 'all' or 'backbone' can be used to initialize the model." + + logging.info('Finished loading pretrained checkpoint from %s', + ckpt_dir_or_file) + + +if __name__ == '__main__': + import matplotlib.pyplot as plt + from yolo.utils.run_utils import prep_gpu + prep_gpu() + + config = exp_cfg.YoloTask(model=exp_cfg.Yolo(base='v3')) + task = YoloTask(config) + model = task.build_model() + model.summary() + task.initialize(model) + + train_data = task.build_inputs(config.train_data) + # test_data = task.build_inputs(config.task.validation_data) + + for l, (i, j) in enumerate(train_data): + preds = model(i, training=False) + boxes = xcycwh_to_yxyx(j['bbox']) + + i = tf.image.draw_bounding_boxes(i, boxes, [[1.0, 0.0, 0.0]]) + + i = tf.image.draw_bounding_boxes(i, preds['bbox'], [[0.0, 1.0, 0.0]]) + plt.imshow(i[0].numpy()) + plt.show() + + if l > 2: + break From 7b845bbbf31f4c0a2736848e1d8387da9f3ce703 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 1 Mar 2021 18:04:33 -0500 Subject: [PATCH 055/132] fixed tests for merge --- centernet/modeling/CenterNet_test.py | 8 +++++++- centernet/modeling/backbones/hourglass.py | 2 +- .../modeling/decoders/centernet_decoder_test.py | 8 ++++---- centernet/tasks/centernet_test.py | 12 +++++++++--- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/centernet/modeling/CenterNet_test.py b/centernet/modeling/CenterNet_test.py index 40ed698e4..1cfedb773 100644 --- a/centernet/modeling/CenterNet_test.py +++ b/centernet/modeling/CenterNet_test.py @@ -19,7 +19,13 @@ def testBuildCenterNet(self): model, loss = build_centernet(input_specs=input_specs, task_config=config, l2_regularization=0) - # TODO: add some call tests + outputs = model(tf.zeros((5, 512, 512, 3))) + self.assertEqual(len(outputs['raw_output']), 3) + self.assertEqual(outputs['raw_output']['ct_heatmaps'].shape, (5, 128, 128, 80)) + self.assertEqual(outputs['raw_output']['ct_offset'].shape, (5, 128, 128, 2)) + self.assertEqual(outputs['raw_output']['ct_size'].shape, (5, 128, 128, 2)) + + model.summary() diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 28373478d..29fc4c009 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -99,7 +99,7 @@ def __init__( activation='linear' )(x_hg) - x_inter = inter_hg_conv1 + inter_hg_conv2 + x_inter = tf.keras.layers.Add()([inter_hg_conv1, inter_hg_conv2]) x_inter = tf.keras.layers.ReLU()(x_inter) # inters diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index 16466d6cc..c218e9768 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -11,7 +11,7 @@ class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_create_decoder(self): decoder = centernet_decoder.build_centernet_decoder( task_config=cfg.CenterNetTask(), - input_specs=(2, 128, 128, 256)) + input_specs=(None, 128, 128, 256)) config = decoder.get_config() self.assertEqual(len(config), 2) @@ -30,9 +30,9 @@ def test_decoder_shape(self): self.assertEqual(outputs['ct_size'].shape, (2, 128, 128, 2)) # Weight initialization tests - hm_bias_vector = np.asarray(decoder.out_layers['ct_heatmaps'].weights[-1]) - off_bias_vector = np.asarray(decoder.out_layers['ct_offset'].weights[-1]) - size_bias_vector = np.asarray(decoder.out_layers['ct_size'].weights[-1]) + hm_bias_vector = np.asarray(decoder.layers[1].weights[-1]) + off_bias_vector = np.asarray(decoder.layers[2].weights[-1]) + size_bias_vector = np.asarray(decoder.layers[3].weights[-1]) self.assertArrayNear(hm_bias_vector, np.repeat(-2.19, repeats=80), err=1.00e-6) diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index 141b5ca25..f28999c6a 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -16,10 +16,16 @@ def testCenterNetTask(self): model_config = exp_cfg.CenterNet(input_size=[512, 512, 3]) config = exp_cfg.CenterNetTask(model=model_config) task = CenterNetTask(config) + model = task.build_model() - out = model(tf.zeros((3, 512, 512, 3))) - model.summary() - + outputs = model(tf.zeros((3, 512, 512, 3))) + + self.assertEqual(len(outputs['raw_output']), 3) + self.assertEqual(outputs['raw_output']['ct_heatmaps'].shape, (3, 128, 128, 80)) + self.assertEqual(outputs['raw_output']['ct_offset'].shape, (3, 128, 128, 2)) + self.assertEqual(outputs['raw_output']['ct_size'].shape, (3, 128, 128, 2)) + + model.summary() if __name__ == '__main__': From 07ae782d49423e835c0bc8e04b6953a82ba3eaaa Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 5 Mar 2021 04:01:30 -0500 Subject: [PATCH 056/132] weightloading + minor changes to model components --- centernet/configs/centernet.py | 2 +- centernet/modeling/CenterNet.py | 11 +- centernet/modeling/CenterNet_test.py | 9 +- centernet/modeling/backbones/hourglass.py | 12 +- .../modeling/decoders/centernet_decoder.py | 36 ++-- .../decoders/centernet_decoder_test.py | 15 +- centernet/modeling/layers/nn_blocks.py | 93 +++++---- centernet/utils/__init__.py | 0 centernet/utils/weight_utils/MODEL_VARS.txt | Bin 0 -> 89602 bytes centernet/utils/weight_utils/__init__.py | 0 .../utils/weight_utils/config_classes.py | 188 ++++++++++++++++++ centernet/utils/weight_utils/load_weights.py | 152 ++++++++++++++ 12 files changed, 447 insertions(+), 71 deletions(-) create mode 100644 centernet/utils/__init__.py create mode 100644 centernet/utils/weight_utils/MODEL_VARS.txt create mode 100644 centernet/utils/weight_utils/__init__.py create mode 100644 centernet/utils/weight_utils/config_classes.py create mode 100644 centernet/utils/weight_utils/load_weights.py diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 9cc0fbe91..b94bcffcd 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -76,7 +76,7 @@ class CenterNetBase(hyperparams.OneOfConfig): @dataclasses.dataclass class CenterNet(hyperparams.Config): - num_classes: int = 80 + num_classes: int = 90 input_size: Optional[List[int]] = dataclasses.field( default_factory=lambda: [None, None, 3]) base: Union[str, CenterNetBase] = CenterNetBase() diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index a19324663..81dad60a5 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -33,8 +33,8 @@ def build(self, input_shape): def call(self, inputs, training=False): features = self._backbone(inputs) - final_backbone_output = features[-1] - decoded_maps = self._decoder(final_backbone_output) + # final_backbone_output = features[-1] + decoded_maps = self._decoder(features) # TODO: head + filters @@ -63,7 +63,7 @@ def head(self): def filter(self): return self._filter -def build_centernet_decoder(input_specs, task_config): +def build_centernet_decoder(input_specs, task_config, num_inputs): # NOTE: For now just support the default config # model specific heatmap_bias = task_config.model.base.decoder.heatmap_bias @@ -72,7 +72,8 @@ def build_centernet_decoder(input_specs, task_config): task_outputs = task_config._get_output_length_dict() model = CenterNetDecoder( task_outputs=task_outputs, - heatmap_bias=heatmap_bias) + heatmap_bias=heatmap_bias, + num_inputs=num_inputs) model.build(input_specs) return model @@ -91,7 +92,7 @@ def build_centernet(input_specs, task_config, l2_regularization): backbone = factory.build_backbone(input_specs, model_config.base, l2_regularization) - decoder = build_centernet_decoder(backbone.output_specs.as_list(), task_config) + decoder = build_centernet_decoder(backbone.output_specs.as_list(), task_config, backbone._num_hourglasses) head = build_centernet_head(model_config) filter = build_centernet_filter(model_config) diff --git a/centernet/modeling/CenterNet_test.py b/centernet/modeling/CenterNet_test.py index 1cfedb773..cee180ac4 100644 --- a/centernet/modeling/CenterNet_test.py +++ b/centernet/modeling/CenterNet_test.py @@ -21,9 +21,12 @@ def testBuildCenterNet(self): outputs = model(tf.zeros((5, 512, 512, 3))) self.assertEqual(len(outputs['raw_output']), 3) - self.assertEqual(outputs['raw_output']['ct_heatmaps'].shape, (5, 128, 128, 80)) - self.assertEqual(outputs['raw_output']['ct_offset'].shape, (5, 128, 128, 2)) - self.assertEqual(outputs['raw_output']['ct_size'].shape, (5, 128, 128, 2)) + self.assertEqual(len(outputs['raw_output']['ct_heatmaps']), 2) + self.assertEqual(len(outputs['raw_output']['ct_offset']), 2) + self.assertEqual(len(outputs['raw_output']['ct_size']), 2) + self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (5, 128, 128, 80)) + self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (5, 128, 128, 2)) + self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (5, 128, 128, 2)) model.summary() diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 29fc4c009..a8afaddd9 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -33,7 +33,7 @@ def __init__( pre_layers: tf.keras layer to process input before stacked hourglasses """ # yapf: disable - input = tf.keras.layers.Input(shape=input_specs.shape[1:]) + input = tf.keras.layers.Input(shape=input_specs.shape[1:], name='input') x_inter = input # Create some intermediate and postlayers to generate the heatmaps @@ -53,10 +53,10 @@ def __init__( kernel_size=prelayer_kernel_size, strides=prelayer_strides, padding='same', - activation='relu' + activation='relu', )(x_inter) x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, strides=prelayer_strides + filters=inp_filters, use_projection=True, strides=prelayer_strides, )(x_inter) all_heatmaps = [] @@ -65,7 +65,7 @@ def __init__( # Create hourglass stacks x_hg = nn_blocks.HourglassBlock( channel_dims_per_stage=channel_dims_per_stage, - blocks_per_stage=blocks_per_stage + blocks_per_stage=blocks_per_stage, )(x_inter) # cnvs @@ -74,7 +74,7 @@ def __init__( kernel_size=(3, 3), strides=(1, 1), padding='same', - activation='relu' + activation='relu', )(x_hg) all_heatmaps.append(x_hg) @@ -104,7 +104,7 @@ def __init__( # inters x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, strides=1 + filters=inp_filters, use_projection=False, strides=1 )(x_inter) # yapf: enable diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index f09bad0d0..aa9978a61 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -10,7 +10,7 @@ class CenterNetDecoder(tf.keras.Model): def __init__(self, task_outputs: dict, heatmap_bias: float = -2.19, - input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 256]), + num_inputs: int = 2, **kwargs): """ Args: @@ -24,24 +24,31 @@ def __init__(self, dictionary where the keys-value pairs denote the names of the output and the respective output tensor """ + super().__init__(**kwargs) - input = tf.keras.layers.Input(shape=input_specs.shape[1:]) + self._task_outputs = task_outputs + self._heatmap_bias = heatmap_bias + self._num_inputs = num_inputs - outputs = {} + self.outputs_layers = {} - for key in task_outputs: - num_filters = task_outputs[key] + def build(self, inputs): + for key in self._task_outputs: + num_filters = self._task_outputs[key] bias = 0 if 'heatmaps' in key: - bias = heatmap_bias + bias = self._heatmap_bias - outputs[key] = CenterNetDecoderConv(output_filters=num_filters, - name=key, bias_init=bias)(input) - - super().__init__(inputs=input, outputs=outputs, **kwargs) + self.outputs_layers[key] = [CenterNetDecoderConv(output_filters=num_filters, + name=key, bias_init=bias) for _ in range(self._num_inputs)] - self._task_outputs = task_outputs - self._heatmap_bias = heatmap_bias + def call(self, inputs): + outputs = {} + + for key in self._task_outputs: + outputs[key] = [self.outputs_layers[key][i](inputs[i]) for i in range(self._num_inputs)] + + return outputs def get_config(self): layer_config = { @@ -52,7 +59,7 @@ def get_config(self): #layer_config.update(super().get_config()) return layer_config -def build_centernet_decoder(input_specs, task_config): +def build_centernet_decoder(input_specs, task_config, num_inputs): # NOTE: For now just support the default config # model specific @@ -62,7 +69,8 @@ def build_centernet_decoder(input_specs, task_config): task_outputs = task_config._get_output_length_dict() model = CenterNetDecoder( task_outputs=task_outputs, - heatmap_bias=heatmap_bias) + heatmap_bias=heatmap_bias, + num_inputs=num_inputs) model.build(input_specs) return model \ No newline at end of file diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index c218e9768..2dc40ccaa 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -11,7 +11,8 @@ class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_create_decoder(self): decoder = centernet_decoder.build_centernet_decoder( task_config=cfg.CenterNetTask(), - input_specs=(None, 128, 128, 256)) + input_specs=(None, 128, 128, 256), + num_inputs=2) config = decoder.get_config() self.assertEqual(len(config), 2) @@ -20,14 +21,16 @@ def test_create_decoder(self): def test_decoder_shape(self): decoder = centernet_decoder.build_centernet_decoder( task_config=cfg.CenterNetTask(), - input_specs=(2, 128, 128, 256)) + input_specs=(2, 128, 128, 256), + num_inputs=2) # Output shape tests - outputs = decoder(np.zeros((2, 128, 128, 256), dtype=np.float32)) + outputs = decoder([np.zeros((2, 128, 128, 256), dtype=np.float32), + np.zeros((2, 128, 128, 256), dtype=np.float32)]) self.assertEqual(len(outputs), 3) - self.assertEqual(outputs['ct_heatmaps'].shape, (2, 128, 128, 80)) - self.assertEqual(outputs['ct_offset'].shape, (2, 128, 128, 2)) - self.assertEqual(outputs['ct_size'].shape, (2, 128, 128, 2)) + self.assertEqual(outputs['ct_heatmaps'][0].shape, (2, 128, 128, 80)) + self.assertEqual(outputs['ct_offset'][0].shape, (2, 128, 128, 2)) + self.assertEqual(outputs['ct_size'][0].shape, (2, 128, 128, 2)) # Weight initialization tests hm_bias_vector = np.asarray(decoder.layers[1].weights[-1]) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 567bdeb52..db8bcf79f 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -209,37 +209,56 @@ def __init__(self, 'size and residual block repetition lists must have the same length' self._filters = channel_dims_per_stage[0] + if self._order > 0: + self._filters_downsampled = channel_dims_per_stage[1] self._reps = blocks_per_stage[0] - super().__init__() + super().__init__(**kwargs) + + def make_repeated_residual_blocks(self, reps, out_channels, + residual_channels=None, + initial_stride=1, initial_skip=False): + blocks = [] + + if residual_channels is None: + residual_channels = out_channels + + for i in range(reps - 1): + stride = initial_stride if i == 0 else 1 + skip_conv = stride > 1 + + blocks.append(official_nn_blocks.ResidualBlock( + filters=residual_channels, strides=stride, + use_projection=skip_conv)) + + if reps == 1: + stride = initial_stride + skip_conv = stride > 1 + else: + stride = 1 + skip_conv = residual_channels != out_channels + + blocks.append(official_nn_blocks.ResidualBlock( + filters=out_channels, strides=stride, + use_projection=skip_conv)) + + return tf.keras.Sequential(blocks) def build(self, input_shape): - if self._order == 1: + if self._order == 0: # base case, residual block repetitions in most inner part of hourglass - blocks = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.blocks = tf.keras.Sequential(blocks) + self.blocks = self.make_repeated_residual_blocks(reps=self._reps, + out_channels=self._filters) else: # outer hourglass structures - main_block = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.main_block = tf.keras.Sequential(main_block, name='Main_Block') - - side_block = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.side_block = tf.keras.Sequential(side_block, name='Side_Block') - - self.pool = tf.keras.layers.MaxPool2D(pool_size=2) + self.encoder_block1 = self.make_repeated_residual_blocks(reps=self._reps, + out_channels=self._filters) + + self.encoder_block2 = self.make_repeated_residual_blocks(reps=self._reps, + out_channels=self._filters_downsampled, initial_stride=2, initial_skip=self._filters != self._filters_downsampled) + + # self.pool = tf.keras.layers.MaxPool2D(pool_size=2) # recursively define inner hourglasses self.inner_hg = type(self)( @@ -248,12 +267,8 @@ def build(self, input_shape): strides=self._strides) # outer hourglass structures - end_block = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.end_block = tf.keras.Sequential(end_block, name='End_Block') + self.decoder_block = self.make_repeated_residual_blocks(reps=self._reps, + residual_channels=self._filters_downsampled, out_channels=self._filters) self.upsample_layer = tf.keras.layers.UpSampling2D( size=2, interpolation='nearest') @@ -261,15 +276,21 @@ def build(self, input_shape): super().build(input_shape) def call(self, x): - if self._order == 1: + if self._order == 0: return self.blocks(x) else: - x_pre_pooled = self.main_block(x) - x_side = self.side_block(x_pre_pooled) - x_pooled = self.pool(x_pre_pooled) - inner_output = self.inner_hg(x_pooled) - hg_output = self.end_block(inner_output) - return self.upsample_layer(hg_output) + x_side + encoded_outputs = self.encoder_block1(x) + encoded_downsampled_outputs = self.encoder_block2(x) + inner_outputs = self.inner_hg(encoded_downsampled_outputs) + hg_output = self.decoder_block(inner_outputs) + return self.upsample_layer(hg_output) + encoded_outputs + + # x_pre_pooled = self.encoder_block1(x) + # x_side = self.encoder_block2(x_pre_xpooled) + # x_pooled = self.pool(x_pre_pooled) + # inner_output = self.inner_hg(x_pooled) + # hg_output = self.decoder_block(inner_output) + # return self.upsample_layer(hg_output) + x_side def get_config(self): layer_config = { diff --git a/centernet/utils/__init__.py b/centernet/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/centernet/utils/weight_utils/MODEL_VARS.txt b/centernet/utils/weight_utils/MODEL_VARS.txt new file mode 100644 index 0000000000000000000000000000000000000000..3ebd36228e6e72a2d01e5ad8297a958438dea777 GIT binary patch literal 89602 zcmeHQZE_pO5#8Ue${linq~iD}IRJ~2XqmAjsz^$5E}vu`So)&BXT{dC{nd*AMN z?G?v9kN?s}JPtpee)GHe9e)3zz5iqQGCZ{TpNFq)CB(dK?|f}5JWj@A4%U5WV?MMI z*KNkt@UH!N*VdoL$9P=t+wZ>&-)G}-%=+6Bzdxbzv32=#i+|m+*@|yxeQ$H_TMW#6 z-y$r{_Ec#jd-Czs>gk+a+gla6;Nd8+;rOL4VGOPdci0Jb%+ujG$GkWr8xMao53Z5 ziCxXx-AHG>U0v21d+#}yaBKu0%!y%y8Idkye6G{1$Ii~MB7zxK_2P!Hj2-nX&g-F2 zy~|iN%8WSoY@S}#M!I2F$nY)Fg;)n{Zr{Hfb>yTnHy`h*n5E)_j-i$VUjJU|6W_N{Un{c| ztgxL@^k2a-4Zmz{Di$cbrRpo&pBzvLhj(Vbw>_^9R;*sbCqz@Q8*yyC3f&ReJ?wdk zT&fyDmlY1FKwQH=J!fphszcVFCaBl^nO^;LQO}4G6z4$3Qt+Cwa941A98bgdaYQXs z$C1ac?J5daW^{!kG8XO%cVs*zo3zL)dD#zJ8@cMDM1^-UM|EOjTqWv=X}h4*4x%fW zfOuM-PU}~*04tCQ)e|0K;Zk~tGB;IT`KKgyh*h1J_Ihv+9XU4*Dm{oyxM+{s*uE=w zn0~v9b|+A3#MZ1*weBg64ESa9dn_Eebf>y5K8p@1IfQsxh9jb7=EY~1c)k3->WqUG zls>EEo|bq0IAEeuI^n9-`&gkpY%cy>w zvyqj~nE!;Ib@y^c#@*wS9%@(a_K2t5r4jLbJQdmYYrLF9imU))h?m+Fk7>7*F)Xp+}=OyP^uSo+pekoXi3XcM6~s*S%4MD zgp1cMyW!z)l;2p}bN^Ad^305Q=cYNe=k!zbem^<7aLD)HOF8tJrJwt8h%4^dm6W+D zPT=gBlI8soR{F5^++%C6`tS1h-fPb|I=5Avyp%kqzSPeKjH{>@np4({cuM^uo>F0m zr&SpuYL$kF}caZFU~aX zMjy^t7FL}}{pnxb_?-f(})jzQ&jeTXf8~k*b^{-%jlemEM9C zwkuhmq!Md9ndVbP2@#3ZTW|)=_rKz&4)*xeYS<(HrmU{2tfF}$bqY09cA!elI*+UI z%jQw_whEHjr`>Zpb!fW{6`!7ir@(x?hgSW)l0Wju@)mP(RU8QOD-6K=ntvdIWp#BY zvC*QARQ;60f6mC@mg{u}VcW$FCN@G82k-(H)sm-E$cWoTl1`Y4Zfab&|cY!yl`c_D}!f_JvCoPeKtbItNgp| zsg!AJ%({98RdhIqtOgrI!NtzMhJA!JU6jdAtlcx;?l>k}kb4G6%saIrvrc0DDC)V( zpIi98FYG@_y=LyzFgR4)xiK-LxJ-??z0YEkQJIYpRT$Bw4_=-m*pc9ozw%YsN}ylv-xJV~s+REu=ik6pExxew)-en@)Ja*0K7Oy@?lB2zy zB*rb&9H|%M^Vm#hdpSvLaAzma#Yy(M%XoKQ_CCs!I1??i_Qs98e`P9(bUW5*+hs2c zBC6ZkW0w_i+xFZqq5sd>_P!I3Nycx-=E(dSjNo63GqTTvoPT&vSe% zr}W6~!}Amjs5n#o%m*Ih%KMux2j=4YHU>w}-4**Zly<9<|ImDuB zR68Q`Q=*QTs(P89MM$;3F}jwgh-&lg!6-7Jx`VO@6)$BB-j%tj^2)n3t5Rx2%eek* zQ|j4ftJ3lwIS1x2FSTPc?jH4zx|M1{JS77WPpd#g?89qBo<>*hDEUtgnVS-8CC*xm zExt+w*hjm9+8Najq(r?h!`a{IDyklH1tohCQ_J8z?-|P6RA)@gcp znU=XWUzKXfxxG81wwLe~C71fUsztvCTgil~-nkE`90>80xhe8e|IIafR)0cgr}nOA z%V(9Fv5z)-54B(=wS#Cqj!f92o|R_;i?xzTT0E3&u{Epttz?eEZK1XL6FRGU+s>$D zA#l-Z1QGjk8!M0r7iUB_GMAD?S_buHkCHhGXIwsq*8Ud!;Btf@btD>nTCq1-gNf zUx?Xz|5CD6={~4kWPbbP6 zk)N_o#8dh>;;9v_RRc1#DpsHGY;@(0^Ln4$wHWt90PN|l56r^B9l@a+#aH;>Ljvza4Q8X zG?>{wt2_YxRxrV4S7m{M0TpLbe{U!6a*EGt6dVRmG(1)@4so=sL&RxRrP36(@7CyLXVf|))1toj_FW25Ht)VQaSWmPY$vZRlutGugo&em1o8st?~Xa&(~ZamGW>PeOE)@sr9wSutCBHPX76KTvZUdxFz+eh$wd|7m=+NmUG0dVVH+Y% zqbamhVF)tHav5_g=vI*3gMZY~te!Xg=Zp+a8GPzm$sudk&8mt@%6eX=%V1HiX3WfB zH05n}2i0^P`LOfRr|XP&W!8$WuFh+$a#{@ zIm)UGA`xXS@3+4H;-r7S?Oh2_#l>~b(>-z=%Q71CEI%unV(0hj9+{It4iDSYsOEL3 zWd4Tfm?)ys%sx=zdk)blCK*(xQ8Ie-pQJLqeS0b)_10kG@Zjb!z{b=Rk&dzMctNYXEn|F$6zLukU!z4t-pwXE5niL z>RFy-P>N_8nfiCz%xNo&yQc3^RXt;_V_g<^1?*UGGuVMB%qIoGH=hK-O# zpUA&smsKB35 zre4gk9gbi#ot?plo%EgvBmN(W?Tx$rM8Q>~J{sjl4Y`Oea!j=O?Ts6eZr3Y`bUUVb zo10g^k^MLO&4YQKFZl5La+7$4yR#@pUU_<-3FPGEeD7t{Qr+=z6;b|SjcG}lNkBYpJu zF!I?=q*Jw8r?WG?MwCd`Fy7xejP9w_NNkU1EuR3^_xSbSj=RwN1Q5d}gTB>wM4ed8 z&dxA4f;m?7;Er*)BI=E(re1XnSG%4Uqey4HUG<*;JSRHus=$)f^j2Ji2goaiBxdB$ z#LVgJb5>lw4-t%F_}HAbx_dW#8y<(RPk&zh$l8Z%h5lyRfHcr>Z=!(Q40b+kW>r+zj{4VQz*G!|ibW^z&7F=cfICCvV$(-`abh zhB;K08Opr8&g((P&&_;4$+H-9Im@F7c4n@hjD!0Y(=o==HBV0QYJEzvakre2)q6U$ z)}3WeGxEGkEVJ3uXItx**P&KB!`mIq^mTs=d3IxsiM2k@e@DF5=dG&z{jvS;hgi1H zSM8Jl7ltdpZ?1eh+%?5t4ZpSbuG_m;fBt5S;JY^JkKymbzuNuRc0IJwj}!Nv$KX?o z`=2)czQz90)_ZJg;nhuh{jt4%Y|;KU%+E#N-?eeqZOk|yyw7WmO&H@_e_p+qUs-=X zsxPDcc4VvLOI>?m(L>`N_Wzp+s(m_^&6phXAvT};Qa2bI_qF`ydEm=ve(u(a%xsPM zzV-DTxodj-cjFDb3r{-uWbOH^dFT)GYo97__SCojGHz>_Y~eZ|wVKfycUkpX*1q-U Z(M0^U*mE6uXwLJ!@#gtC`J3U-{{e3L{eA!d literal 0 HcmV?d00001 diff --git a/centernet/utils/weight_utils/__init__.py b/centernet/utils/weight_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/centernet/utils/weight_utils/config_classes.py b/centernet/utils/weight_utils/config_classes.py new file mode 100644 index 000000000..12bfa98b2 --- /dev/null +++ b/centernet/utils/weight_utils/config_classes.py @@ -0,0 +1,188 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +import numpy as np + +from typing import Tuple, List, Dict, ClassVar + +class Config(ABC): + def get_weights(self): + return None + + def load_weights(self, layer): + weights = self.get_weights() + layer.set_weights(weights) + +@dataclass +class convBnCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + weights: np.array = field(repr=False, default=None) + beta: np.array = field(repr=False, default=None) + gamma: np.array = field(repr=False, default=None) + moving_mean: np.array = field(repr=False, default=None) + moving_variance: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_weights_dict = self.weights_dict['conv'] + norm_weights_dict = self.weights_dict['norm'] + + self.weights = conv_weights_dict['kernel'] + self.beta = norm_weights_dict['beta'] + self.gamma = norm_weights_dict['gamma'] + self.moving_mean = norm_weights_dict['moving_mean'] + self.moving_variance = norm_weights_dict['moving_variance'] + + def get_weights(self): + return [ + self.weights, + self.gamma, + self.beta, + self.moving_mean, + self.moving_variance + ] + +@dataclass +class residualBlockCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + + skip_weights: np.array = field(repr=False, default=None) + skip_beta: np.array = field(repr=False, default=None) + skip_gamma: np.array = field(repr=False, default=None) + skip_moving_mean: np.array = field(repr=False, default=None) + skip_moving_variance: np.array = field(repr=False, default=None) + + conv_weights: np.array = field(repr=False, default=None) + norm_beta: np.array = field(repr=False, default=None) + norm_gamma: np.array = field(repr=False, default=None) + norm_moving_mean: np.array = field(repr=False, default=None) + norm_moving_variance: np.array = field(repr=False, default=None) + + conv_block_weights: np.array = field(repr=False, default=None) + conv_block_beta: np.array = field(repr=False, default=None) + conv_block_gamma: np.array = field(repr=False, default=None) + conv_block_moving_mean: np.array = field(repr=False, default=None) + conv_block_moving_variance: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_weights_dict = self.weights_dict['conv'] + norm_weights_dict = self.weights_dict['norm'] + conv_block_weights_dict = self.weights_dict['conv_block'] + + if 'skip' in self.weights_dict: + skip_weights_dict = self.weights_dict['skip'] + self.skip_weights = skip_weights_dict['conv']['kernel'] + self.skip_beta = skip_weights_dict['norm']['beta'] + self.skip_gamma = skip_weights_dict['norm']['gamma'] + self.skip_moving_mean = skip_weights_dict['norm']['moving_mean'] + self.skip_moving_variance = skip_weights_dict['norm']['moving_variance'] + + self.conv_weights = conv_weights_dict['kernel'] + self.norm_beta = norm_weights_dict['beta'] + self.norm_gamma = norm_weights_dict['gamma'] + self.norm_moving_mean = norm_weights_dict['moving_mean'] + self.norm_moving_variance = norm_weights_dict['moving_variance'] + + self.conv_block_weights = conv_block_weights_dict['conv']['kernel'] + self.conv_block_beta = conv_block_weights_dict['norm']['beta'] + self.conv_block_gamma = conv_block_weights_dict['norm']['gamma'] + self.conv_block_moving_mean = conv_block_weights_dict['norm']['moving_mean'] + self.conv_block_moving_variance = conv_block_weights_dict['norm']['moving_variance'] + + def get_weights(self): + weights = [ + self.skip_weights, + self.skip_gamma, + self.skip_beta, + + self.conv_block_weights, + self.conv_block_gamma, + self.conv_block_beta, + + self.conv_weights, + self.norm_gamma, + self.norm_beta, + + self.skip_moving_mean, + self.skip_moving_variance, + self.conv_block_moving_mean, + self.conv_block_moving_variance, + self.norm_moving_mean, + self.norm_moving_variance, + ] + + weights = [x for x in weights if x is not None] + return weights + +@dataclass +class hourglassCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + is_last_stage: bool = field(repr=False, default=None) + + def __post_init__(self): + self.is_last_stage = False if 'inner_block' in self.weights_dict else True + + def generate_block_weights(self, weights_dict): + reps = len(weights_dict.keys()) + weights = [] + + for i in range(reps): + res_config = residualBlockCFG(weights_dict=weights_dict[str(i)]) + weights += res_config.get_weights() + + return weights + + def load_block_weights(self, layer, weight_dict): + block_weights = self.generate_block_weights(weight_dict) + layer.set_weights(block_weights) + + def load_weights(self, layer): + if not self.is_last_stage: + enc_dec_layers = [layer.submodules[0], + layer.submodules[1], + layer.submodules[3]] + enc_dec_weight_dicts = [self.weights_dict['encoder_block1'], + self.weights_dict['encoder_block2'], + self.weights_dict['decoder_block']] + + + for l, weights_dict in zip(enc_dec_layers, enc_dec_weight_dicts): + self.load_block_weights(l, weights_dict) + + if len(self.weights_dict['inner_block']) == 1: # still in an outer hourglass + inner_weights_dict = self.weights_dict['inner_block']['0'] + else: + inner_weights_dict = self.weights_dict['inner_block'] # inner residual block chain + + inner_hg_layer = layer.submodules[2] + inner_hg_cfg = type(self)(weights_dict=inner_weights_dict) + inner_hg_cfg.load_weights(inner_hg_layer) + + else: + inner_layer = layer.submodules[0] + self.load_block_weights(inner_layer, self.weights_dict) + +@dataclass +class decoderConvCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + + conv_1_weights: np.array = field(repr=False, default=None) + conv_2_bias: np.array = field(repr=False, default=None) + + conv_2_weights: np.array = field(repr=False, default=None) + conv_1_bias: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_1_weights_dict = self.weights_dict['layer_with_weights-0'] + conv_2_weights_dict = self.weights_dict['layer_with_weights-1'] + + self.conv_1_weights = conv_1_weights_dict['kernel'] + self.conv_1_bias = conv_1_weights_dict['bias'] + self.conv_2_weights = conv_2_weights_dict['kernel'] + self.conv_2_bias = conv_2_weights_dict['bias'] + + def get_weights(self): + return [ + self.conv_1_weights, + self.conv_1_bias, + self.conv_2_weights, + self.conv_2_bias + ] \ No newline at end of file diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py new file mode 100644 index 000000000..c249a7b2a --- /dev/null +++ b/centernet/utils/weight_utils/load_weights.py @@ -0,0 +1,152 @@ +import tensorflow as tf +import numpy as np + +from centernet.modeling.CenterNet import build_centernet +from centernet.configs.centernet import CenterNetTask +from centernet.utils.weight_utils.config_classes import convBnCFG, residualBlockCFG, hourglassCFG, decoderConvCFG + +from centernet.modeling.layers.nn_blocks import ConvBN, HourglassBlock, CenterNetDecoderConv +from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock + +CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' +SAVED_MODEL_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\saved_model' + +def print_dict_as_tree(dictionary, spaces): + if type(dictionary) is not dict: + return + + else: + for key in dictionary.keys(): + print(" " * spaces + key) + print_dict_as_tree(dictionary[key], spaces + 2) + +def print_layer_weights_and_shape(layer): + weights = layer.get_weights() + variables = layer.variables + + for i in range(len(weights)): + tf.print(np.shape(weights[i]), variables[i].name) + +def update_weights_dict(weights_dict, variable_key, value): + """ Inserts weight value into a weight dictionary + Args: + weights_dict: + variable_key: + value: + """ + current_dict = weights_dict + variable_key_list = variable_key.split('/') + + key = variable_key_list.pop(0) + while len(variable_key_list): + if variable_key_list[0] == '.ATTRIBUTES': + current_dict[key] = value + return + + if key not in current_dict.keys(): + current_dict[key] = {} + current_dict = current_dict[key] + key = variable_key_list.pop(0) + +def load_weights_model(model, weights_dict): + print("Loading model weights\n") + load_weights_backbone(model.backbone, + weights_dict['model']['_feature_extractor']['_network']) + + load_weights_decoder(model.decoder, + weights_dict['model']['_prediction_head_dict']) + print("Successfully loaded model weights\n") + +def get_backbone_layer_cfgs(weights_dict): + print("Fetching backbone config classes\n") + cfgs = [ + # Downsampling Layers + convBnCFG(weights_dict=weights_dict['downsample_input']['conv_block']), + residualBlockCFG(weights_dict=weights_dict['downsample_input']['residual_block']), + # Hourglass + hourglassCFG(weights_dict=weights_dict['hourglass_network']['0']), + convBnCFG(weights_dict=weights_dict['output_conv']['0']), + # Intermediate + convBnCFG(weights_dict=weights_dict['intermediate_conv1']['0']), + convBnCFG(weights_dict=weights_dict['intermediate_conv2']['0']), + residualBlockCFG(weights_dict=weights_dict['intermediate_residual']['0']), + # Hourglass + hourglassCFG(weights_dict=weights_dict['hourglass_network']['1']), + convBnCFG(weights_dict=weights_dict['output_conv']['1']), + ] + + return cfgs + +def load_weights_backbone(backbone, weights_dict): + print("Loading backbone weights\n") + backbone_layers = backbone.layers + cfgs = get_backbone_layer_cfgs(weights_dict) + + cfg = cfgs.pop(0) + for i in range(len(backbone_layers)): + layer = backbone_layers[i] + if isinstance(layer, (ConvBN, HourglassBlock, ResidualBlock)): + print("Loading weights for: {}".format(cfg)) + cfg.load_weights(layer) + if len(cfgs) == 0: + print("Weights have been loaded for {} / {} layers\n".format(i+1, len(backbone_layers))) + return + cfg = cfgs.pop(0) + +def get_decoder_layer_cfgs(weights_dict): + print("Fetching decoder config classes\n") + cfgs = [ + decoderConvCFG(weights_dict=weights_dict['object_center']['0']), + decoderConvCFG(weights_dict=weights_dict['object_center']['1']), + decoderConvCFG(weights_dict=weights_dict['box.Soffset']['0']), + decoderConvCFG(weights_dict=weights_dict['box.Soffset']['1']), + decoderConvCFG(weights_dict=weights_dict['box.Sscale']['0']), + decoderConvCFG(weights_dict=weights_dict['box.Sscale']['1']) + ] + return cfgs + +def load_weights_decoder(decoder, weights_dict): + print("Loading decoder weights\n") + decoder_layers = decoder.layers + cfgs = get_decoder_layer_cfgs(weights_dict) + + cfg = cfgs.pop(0) + for i in range(len(decoder_layers)): + layer = decoder_layers[i] + if isinstance(layer, CenterNetDecoderConv): + print("Loading weights for: {}".format(cfg)) + cfg.load_weights(layer) + if len(cfgs) == 0: + print("Weights have been loaded for {} / {} layers\n".format(i+1, len(decoder_layers))) + return + cfg = cfgs.pop(0) + +def get_model_weights_as_dict(ckpt_path): + print("\nConverting model checkpoint from {} to weights dictionary\n".format(ckpt_path)) + reader = tf.train.load_checkpoint(ckpt_path) + shape_from_key = reader.get_variable_to_shape_map() + dtype_from_key = reader.get_variable_to_dtype_map() + + variable_keys = shape_from_key.keys() + weights_dict = {} + + for key in variable_keys: + shape = shape_from_key[key] + dtype = dtype_from_key[key] + value = reader.get_tensor(key) + update_weights_dict(weights_dict, key, value) + + print("Successfully read checkpoint weights\n") + return weights_dict + +if __name__ == '__main__': + input_specs = tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]) + config = CenterNetTask() + + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + + weights_dict = get_model_weights_as_dict(CKPT_PATH) + load_weights_model(model, weights_dict) + + # print_dict_as_tree(weights_dict, 0) \ No newline at end of file From c4af9663b907335683b222d343383d5d60da45c7 Mon Sep 17 00:00:00 2001 From: David Li Date: Sun, 7 Mar 2021 23:53:59 -0500 Subject: [PATCH 057/132] documented + number weights tracking --- centernet/utils/weight_utils/MODEL_VARS.txt | Bin 89602 -> 43414 bytes .../utils/weight_utils/config_classes.py | 28 ++- centernet/utils/weight_utils/config_data.py | 48 +++++ centernet/utils/weight_utils/load_weights.py | 182 ++++++++---------- centernet/utils/weight_utils/tf_to_dict.py | 87 +++++++++ 5 files changed, 242 insertions(+), 103 deletions(-) create mode 100644 centernet/utils/weight_utils/config_data.py create mode 100644 centernet/utils/weight_utils/tf_to_dict.py diff --git a/centernet/utils/weight_utils/MODEL_VARS.txt b/centernet/utils/weight_utils/MODEL_VARS.txt index 3ebd36228e6e72a2d01e5ad8297a958438dea777..119fb548ccb31883b1c8c18b011cd16838c5d2a8 100644 GIT binary patch literal 43414 zcmeHPS#sM*68)c3-~!Vz%Tf9O3`&A+jwMn-Qg-+4M=6S60m!!$0WKYFepnWeDCAo* zvjE9{**vw+|7e=Umv*^-+qR4LkNtM}xZiBw556B>xBH))?e~vwn&xTqbG=)>ygs*! z)%x{q|9QA+w(V~9^tOCnJUnk6kI}Kf<7WM%i;4r^+wJ<}j`2-=cHdX8y@=5z$wf`G z-fUlbmo?2pyI&3teO5&fqe6uDhwk}$31YK zX_iTjrAD#hs5QhvKzj}Zl8uv%dno)VptLHfgKCz6z*Vl00>l`mCZ%58KlMs}Z+z#J zsgZ+L>-Dj!du2TI6^aY#hp};t#iMKDIO#KS`uNFic{~@uLH{|1#Me}rGN5U4%KI^? zR{apHQ9n)>Q5m1=TLysQDnrt4Q3e9Spzp`fLbQb z%LW%gvcMzAa#n;nCXKi{>lar@l_FO9E`p2+5kVF|9&G8=xSv~}MXb>fBid|S5o&?1 z%k}0Kn$r~4TC%x~pB{tz@^dlCOv^+rmTRjk|LS!-mt0!YN^I7DYLYV`|pU74psNL+M?Wz~dPUuN6Alw8&`bDg*|?s4XA&XrES05$j7oFglX>AE3o zTGms=Yi+$3-F%WbE~SGiSTigDvg=bFJg%dj5)+UWxv&u3S zC6`jqqPmWnpmNUnYVl*?xUFSOR%l1^cHzvN=AP9cy>6Bm0y*+{Q^ca05^b4St{_>I ztY}VLMqIhrSiO~bGz->N#ih#VUQ&9whFsyVXC8C&rW{5$ZPRJfW|CFu8p*9Wd%qxE zW$kFCv#e^;Ja+pAV=L)W#b%qt%D?G$Xh5U-z{(wq%R#Jx>h2d&L~cT63rwkzNe1B&&ku=7~)e9 zB0c@!sZ$IKxm-~JMT?-<67j`LW4f$JN!AW+5!WNX*B|pv_;)T5&*Qw5Wue`3iR6|L z?&f?@=R1UZd|kXhI;q-)EXJFBu|c~WZDjCD{muz>hUwIE2l-Q_ixf{H)RUAxF$d@f zrgwGh#cA^E1U`8~bKu`rrikZASoBlP8`YB_wjSB&;=(MVtqS0FuMxGtZ6h`eSxbP9 zz;1w*q15Hc$A(liC!!y*2875(fiRuE4yV6r3?t`JYh zeZV=#RPdTqJU**r3XKFS7L%fIf1Zasz^p3I7f)qFCBz`Rh6P~+ ztTNMrekuz814p$kb*iw6{{1gB@8Ko7iEtiU51G)K_?L zWB(kaOkn{En2+EHO!LiE#$u*f4#+QwNB_iqSQ#iTxEaRAF&2-miQ{B^#p&axgm64| z{2CO1v33lJuc=05KvxyOUuA5nT^WFVS4Jrhma(bQWdQJI86r8rMk|+r6Xq0@`}lOe zBO8{-DOMsM`2BR{;cuB=k1_L^Mu51Nc(3L$6!l@sLQ}L)GmmO(VIyWQ=lWP< z{K6oPo<6d0BUDjwSk5i+h_10%#7nj4J~&n=KNe`~rm<=bZkNk$Rm<~{g|uS?8J#?W zthGdMjCGiSWU^UHW>{D&fbv(zi-4#Z*5pB`x_R2ZSz*@`beCj{)vO1r|Qmw73%xcIPjLmC4 zFGJfzb>q2~PRlT654P5l<})9BqYTx2o?qL`ieY=gvzVoasjXG;HSBm0F3>_OR-{Yu zFSvy<<8y2CS=6~RsvD5@2GyeZR_3#~@f=#$a4lA(`FbsqKOw7Ksb(F0$_a!MR0Lb;CEd(B=ylv)RL6{yF)|e%PorYLc@TR-jLMlRT8*2AUyDqldp>8Krp~Mn<<0zSPh;)uWe-2w@bIA z&=`A?1C7bo+(7SPm&mDk%oi=pVRCCCdA+(@>dY(|HoSGKGQ4uvn}aH87m3L$9LIHj zJxk=Y4EgireCCnVIiz&eV?3$(1X_D7L87sSTw%KX>@gwv)9mTpCE8gUrV>=>>|!Gj zA7ML1*yoKXU84S|?DSfCC1r@B=9nGC4*h6uPZytp(_O>?*!3CttVRVC)0s->a4Zcz zRceY3+SAzCUCtfz$~&=vQ2wsr4sHNiNt#Hy{if#j>uaI_-Hq<9gR|k7{>LFv?6BnG zq|sTY_))&7;2{!J>i=1XpAweLh`E#Eu!!<+0eL;30HUcu;yrA7}ua zJ|_qwNX6@w<@gr@v$Cwa!L-uhP-AUO6H(c-5CnRT7iQPl!2q9O&Vfd^$PgY^Nb9R= zDC<42AYqo+MJhv;-;}|j?oAzzAvw;u#^B)ZShLmxg~ec@6m2Faks(bAvz6YmD|t{o3I}j`>hUx-i5&L(dJ32_u$5%& zNAsAIf>bhfgH*UYTMXPIpF9K%OQK$t6o=-wivSkQnMm=1cFj~tZ<->>^Wfs9Sxzg7 zbV&dl&K&HcsE8CkZaka8{OGI7#vNScnaAV*kDS&f8g?8rgAy6#tU|2z7S z0=UVzC779Z9nnzv6fdxMnr~_kC14H_BT2IysWSf8=k9gq$dm$?Rh#y|E69Yo1^tIg zG_cd=ZU6eVU-Z~6cO4_+Fn^41JYqTr)4P2X%)lUJWsWjB`>sqop3!Q(Z?`Y)(`vbI z7u$BXdU{(vcR#}|`HOXuXNM7p+K?HX zdQ!e*bwnS^4Cj$$Jpf#p>gi1}O?7aSERT9LIMv1L_T8tC`_*Q>_|`6;7V*u8RUbBg z{Pz3i%a>hy_`dAH&&xmC?c(QZ|84QJU48ww-~F=$^{`s*I=hdolw5q0@x5n}i%)UM zBC}=Cv)spgKBn}+$?kFa+OV literal 89602 zcmeHQZE_pO5#8Ue${linq~iD}IRJ~2XqmAjsz^$5E}vu`So)&BXT{dC{nd*AMN z?G?v9kN?s}JPtpee)GHe9e)3zz5iqQGCZ{TpNFq)CB(dK?|f}5JWj@A4%U5WV?MMI z*KNkt@UH!N*VdoL$9P=t+wZ>&-)G}-%=+6Bzdxbzv32=#i+|m+*@|yxeQ$H_TMW#6 z-y$r{_Ec#jd-Czs>gk+a+gla6;Nd8+;rOL4VGOPdci0Jb%+ujG$GkWr8xMao53Z5 ziCxXx-AHG>U0v21d+#}yaBKu0%!y%y8Idkye6G{1$Ii~MB7zxK_2P!Hj2-nX&g-F2 zy~|iN%8WSoY@S}#M!I2F$nY)Fg;)n{Zr{Hfb>yTnHy`h*n5E)_j-i$VUjJU|6W_N{Un{c| ztgxL@^k2a-4Zmz{Di$cbrRpo&pBzvLhj(Vbw>_^9R;*sbCqz@Q8*yyC3f&ReJ?wdk zT&fyDmlY1FKwQH=J!fphszcVFCaBl^nO^;LQO}4G6z4$3Qt+Cwa941A98bgdaYQXs z$C1ac?J5daW^{!kG8XO%cVs*zo3zL)dD#zJ8@cMDM1^-UM|EOjTqWv=X}h4*4x%fW zfOuM-PU}~*04tCQ)e|0K;Zk~tGB;IT`KKgyh*h1J_Ihv+9XU4*Dm{oyxM+{s*uE=w zn0~v9b|+A3#MZ1*weBg64ESa9dn_Eebf>y5K8p@1IfQsxh9jb7=EY~1c)k3->WqUG zls>EEo|bq0IAEeuI^n9-`&gkpY%cy>w zvyqj~nE!;Ib@y^c#@*wS9%@(a_K2t5r4jLbJQdmYYrLF9imU))h?m+Fk7>7*F)Xp+}=OyP^uSo+pekoXi3XcM6~s*S%4MD zgp1cMyW!z)l;2p}bN^Ad^305Q=cYNe=k!zbem^<7aLD)HOF8tJrJwt8h%4^dm6W+D zPT=gBlI8soR{F5^++%C6`tS1h-fPb|I=5Avyp%kqzSPeKjH{>@np4({cuM^uo>F0m zr&SpuYL$kF}caZFU~aX zMjy^t7FL}}{pnxb_?-f(})jzQ&jeTXf8~k*b^{-%jlemEM9C zwkuhmq!Md9ndVbP2@#3ZTW|)=_rKz&4)*xeYS<(HrmU{2tfF}$bqY09cA!elI*+UI z%jQw_whEHjr`>Zpb!fW{6`!7ir@(x?hgSW)l0Wju@)mP(RU8QOD-6K=ntvdIWp#BY zvC*QARQ;60f6mC@mg{u}VcW$FCN@G82k-(H)sm-E$cWoTl1`Y4Zfab&|cY!yl`c_D}!f_JvCoPeKtbItNgp| zsg!AJ%({98RdhIqtOgrI!NtzMhJA!JU6jdAtlcx;?l>k}kb4G6%saIrvrc0DDC)V( zpIi98FYG@_y=LyzFgR4)xiK-LxJ-??z0YEkQJIYpRT$Bw4_=-m*pc9ozw%YsN}ylv-xJV~s+REu=ik6pExxew)-en@)Ja*0K7Oy@?lB2zy zB*rb&9H|%M^Vm#hdpSvLaAzma#Yy(M%XoKQ_CCs!I1??i_Qs98e`P9(bUW5*+hs2c zBC6ZkW0w_i+xFZqq5sd>_P!I3Nycx-=E(dSjNo63GqTTvoPT&vSe% zr}W6~!}Amjs5n#o%m*Ih%KMux2j=4YHU>w}-4**Zly<9<|ImDuB zR68Q`Q=*QTs(P89MM$;3F}jwgh-&lg!6-7Jx`VO@6)$BB-j%tj^2)n3t5Rx2%eek* zQ|j4ftJ3lwIS1x2FSTPc?jH4zx|M1{JS77WPpd#g?89qBo<>*hDEUtgnVS-8CC*xm zExt+w*hjm9+8Najq(r?h!`a{IDyklH1tohCQ_J8z?-|P6RA)@gcp znU=XWUzKXfxxG81wwLe~C71fUsztvCTgil~-nkE`90>80xhe8e|IIafR)0cgr}nOA z%V(9Fv5z)-54B(=wS#Cqj!f92o|R_;i?xzTT0E3&u{Epttz?eEZK1XL6FRGU+s>$D zA#l-Z1QGjk8!M0r7iUB_GMAD?S_buHkCHhGXIwsq*8Ud!;Btf@btD>nTCq1-gNf zUx?Xz|5CD6={~4kWPbbP6 zk)N_o#8dh>;;9v_RRc1#DpsHGY;@(0^Ln4$wHWt90PN|l56r^B9l@a+#aH;>Ljvza4Q8X zG?>{wt2_YxRxrV4S7m{M0TpLbe{U!6a*EGt6dVRmG(1)@4so=sL&RxRrP36(@7CyLXVf|))1toj_FW25Ht)VQaSWmPY$vZRlutGugo&em1o8st?~Xa&(~ZamGW>PeOE)@sr9wSutCBHPX76KTvZUdxFz+eh$wd|7m=+NmUG0dVVH+Y% zqbamhVF)tHav5_g=vI*3gMZY~te!Xg=Zp+a8GPzm$sudk&8mt@%6eX=%V1HiX3WfB zH05n}2i0^P`LOfRr|XP&W!8$WuFh+$a#{@ zIm)UGA`xXS@3+4H;-r7S?Oh2_#l>~b(>-z=%Q71CEI%unV(0hj9+{It4iDSYsOEL3 zWd4Tfm?)ys%sx=zdk)blCK*(xQ8Ie-pQJLqeS0b)_10kG@Zjb!z{b=Rk&dzMctNYXEn|F$6zLukU!z4t-pwXE5niL z>RFy-P>N_8nfiCz%xNo&yQc3^RXt;_V_g<^1?*UGGuVMB%qIoGH=hK-O# zpUA&smsKB35 zre4gk9gbi#ot?plo%EgvBmN(W?Tx$rM8Q>~J{sjl4Y`Oea!j=O?Ts6eZr3Y`bUUVb zo10g^k^MLO&4YQKFZl5La+7$4yR#@pUU_<-3FPGEeD7t{Qr+=z6;b|SjcG}lNkBYpJu zF!I?=q*Jw8r?WG?MwCd`Fy7xejP9w_NNkU1EuR3^_xSbSj=RwN1Q5d}gTB>wM4ed8 z&dxA4f;m?7;Er*)BI=E(re1XnSG%4Uqey4HUG<*;JSRHus=$)f^j2Ji2goaiBxdB$ z#LVgJb5>lw4-t%F_}HAbx_dW#8y<(RPk&zh$l8Z%h5lyRfHcr>Z=!(Q40b+kW>r+zj{4VQz*G!|ibW^z&7F=cfICCvV$(-`abh zhB;K08Opr8&g((P&&_;4$+H-9Im@F7c4n@hjD!0Y(=o==HBV0QYJEzvakre2)q6U$ z)}3WeGxEGkEVJ3uXItx**P&KB!`mIq^mTs=d3IxsiM2k@e@DF5=dG&z{jvS;hgi1H zSM8Jl7ltdpZ?1eh+%?5t4ZpSbuG_m;fBt5S;JY^JkKymbzuNuRc0IJwj}!Nv$KX?o z`=2)czQz90)_ZJg;nhuh{jt4%Y|;KU%+E#N-?eeqZOk|yyw7WmO&H@_e_p+qUs-=X zsxPDcc4VvLOI>?m(L>`N_Wzp+s(m_^&6phXAvT};Qa2bI_qF`ydEm=ve(u(a%xsPM zzV-DTxodj-cjFDb3r{-uWbOH^dFT)GYo97__SCojGHz>_Y~eZ|wVKfycUkpX*1q-U Z(M0^U*mE6uXwLJ!@#gtC`J3U-{{e3L{eA!d diff --git a/centernet/utils/weight_utils/config_classes.py b/centernet/utils/weight_utils/config_classes.py index 12bfa98b2..06a878d92 100644 --- a/centernet/utils/weight_utils/config_classes.py +++ b/centernet/utils/weight_utils/config_classes.py @@ -11,6 +11,12 @@ def get_weights(self): def load_weights(self, layer): weights = self.get_weights() layer.set_weights(weights) + n_weights = 0 + + for w in weights: + n_weights += w.size + return n_weights + @dataclass class convBnCFG(Config): @@ -123,18 +129,26 @@ def __post_init__(self): def generate_block_weights(self, weights_dict): reps = len(weights_dict.keys()) weights = [] + n_weights = 0 for i in range(reps): res_config = residualBlockCFG(weights_dict=weights_dict[str(i)]) - weights += res_config.get_weights() + res_weights = res_config.get_weights() + weights += res_weights + + for w in res_weights: + n_weights += w.size - return weights + return weights, n_weights def load_block_weights(self, layer, weight_dict): - block_weights = self.generate_block_weights(weight_dict) + block_weights, n_weights = self.generate_block_weights(weight_dict) layer.set_weights(block_weights) + return n_weights def load_weights(self, layer): + n_weights = 0 + if not self.is_last_stage: enc_dec_layers = [layer.submodules[0], layer.submodules[1], @@ -145,7 +159,7 @@ def load_weights(self, layer): for l, weights_dict in zip(enc_dec_layers, enc_dec_weight_dicts): - self.load_block_weights(l, weights_dict) + n_weights += self.load_block_weights(l, weights_dict) if len(self.weights_dict['inner_block']) == 1: # still in an outer hourglass inner_weights_dict = self.weights_dict['inner_block']['0'] @@ -154,11 +168,13 @@ def load_weights(self, layer): inner_hg_layer = layer.submodules[2] inner_hg_cfg = type(self)(weights_dict=inner_weights_dict) - inner_hg_cfg.load_weights(inner_hg_layer) + n_weights += inner_hg_cfg.load_weights(inner_hg_layer) else: inner_layer = layer.submodules[0] - self.load_block_weights(inner_layer, self.weights_dict) + n_weights += self.load_block_weights(inner_layer, self.weights_dict) + + return n_weights @dataclass class decoderConvCFG(Config): diff --git a/centernet/utils/weight_utils/config_data.py b/centernet/utils/weight_utils/config_data.py new file mode 100644 index 000000000..764091b15 --- /dev/null +++ b/centernet/utils/weight_utils/config_data.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass, field +from typing import Dict + +from centernet.utils.weight_utils.config_classes import convBnCFG, residualBlockCFG, hourglassCFG, decoderConvCFG + +@dataclass +class BackboneConfigData(): + weights_dict: Dict = field(repr=False, default=None) + + def __post_init__(self): + self.hourglass104_512 = [ + # Downsampling Layers + convBnCFG(weights_dict=self.weights_dict['downsample_input']['conv_block']), + residualBlockCFG(weights_dict=self.weights_dict['downsample_input']['residual_block']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['0']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['0']), + # Intermediate + convBnCFG(weights_dict=self.weights_dict['intermediate_conv1']['0']), + convBnCFG(weights_dict=self.weights_dict['intermediate_conv2']['0']), + residualBlockCFG(weights_dict=self.weights_dict['intermediate_residual']['0']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['1']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), + ] + + def get_cfg_list(self, name): + if name == 'hourglass104_512': + return self.hourglass104_512 + +@dataclass +class DecoderConfigData(): + weights_dict: Dict = field(repr=False, default=None) + + def __post_init__(self): + self.detection_2d = [ + decoderConvCFG(weights_dict=self.weights_dict['object_center']['0']), + decoderConvCFG(weights_dict=self.weights_dict['object_center']['1']), + decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['0']), + decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['1']), + decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['0']), + decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['1']) + ] + + def get_cfg_list(self, name): + if name == 'detection_2d': + return self.detection_2d + diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index c249a7b2a..4918c3274 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -3,150 +3,138 @@ from centernet.modeling.CenterNet import build_centernet from centernet.configs.centernet import CenterNetTask + +from centernet.utils.weight_utils.tf_to_dict import get_model_weights_as_dict, write_dict_as_tree from centernet.utils.weight_utils.config_classes import convBnCFG, residualBlockCFG, hourglassCFG, decoderConvCFG +from centernet.utils.weight_utils.config_data import BackboneConfigData, DecoderConfigData from centernet.modeling.layers.nn_blocks import ConvBN, HourglassBlock, CenterNetDecoderConv + from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' SAVED_MODEL_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\saved_model' -def print_dict_as_tree(dictionary, spaces): - if type(dictionary) is not dict: - return - - else: - for key in dictionary.keys(): - print(" " * spaces + key) - print_dict_as_tree(dictionary[key], spaces + 2) - -def print_layer_weights_and_shape(layer): - weights = layer.get_weights() - variables = layer.variables - - for i in range(len(weights)): - tf.print(np.shape(weights[i]), variables[i].name) - -def update_weights_dict(weights_dict, variable_key, value): - """ Inserts weight value into a weight dictionary - Args: - weights_dict: - variable_key: - value: - """ - current_dict = weights_dict - variable_key_list = variable_key.split('/') - - key = variable_key_list.pop(0) - while len(variable_key_list): - if variable_key_list[0] == '.ATTRIBUTES': - current_dict[key] = value - return - - if key not in current_dict.keys(): - current_dict[key] = {} - current_dict = current_dict[key] - key = variable_key_list.pop(0) - -def load_weights_model(model, weights_dict): +def load_weights_model(model, weights_dict, backbone_name, decoder_name): + """ Loads weights into the model. + + Args: + model: keras.Model to load weights into + weights_dict: Dictionary that stores the weights of the model + backbone_name: String, indicating the desired backbone configuration + decoder_name: String, indicating the desired decoder configuration + """ print("Loading model weights\n") load_weights_backbone(model.backbone, - weights_dict['model']['_feature_extractor']['_network']) + weights_dict['model']['_feature_extractor']['_network'], + backbone_name) load_weights_decoder(model.decoder, - weights_dict['model']['_prediction_head_dict']) + weights_dict['model']['_prediction_head_dict'], + decoder_name) print("Successfully loaded model weights\n") -def get_backbone_layer_cfgs(weights_dict): - print("Fetching backbone config classes\n") - cfgs = [ - # Downsampling Layers - convBnCFG(weights_dict=weights_dict['downsample_input']['conv_block']), - residualBlockCFG(weights_dict=weights_dict['downsample_input']['residual_block']), - # Hourglass - hourglassCFG(weights_dict=weights_dict['hourglass_network']['0']), - convBnCFG(weights_dict=weights_dict['output_conv']['0']), - # Intermediate - convBnCFG(weights_dict=weights_dict['intermediate_conv1']['0']), - convBnCFG(weights_dict=weights_dict['intermediate_conv2']['0']), - residualBlockCFG(weights_dict=weights_dict['intermediate_residual']['0']), - # Hourglass - hourglassCFG(weights_dict=weights_dict['hourglass_network']['1']), - convBnCFG(weights_dict=weights_dict['output_conv']['1']), - ] +def get_backbone_layer_cfgs(weights_dict, backbone_name): + """ Fetches the config classes for the backbone. + + This function generates a list of config classes corresponding to + each building block in the backbone. + + Args: + weights_dict: Dictionary that stores the backbone model weights + backbone_name: String, indicating the desired backbone configuration + Returns: + A list containing the config classe of the backbone building block + """ + print("Fetching backbone config classes for {}\n".format(backbone_name)) + cfgs = BackboneConfigData(weights_dict=weights_dict).get_cfg_list(backbone_name) return cfgs -def load_weights_backbone(backbone, weights_dict): +def load_weights_backbone(backbone, weights_dict, backbone_name): + """ Loads the weights defined in the weights_dict into the backbone. + + This function loads the backbone weights by first fetching the necesary + config classes for the backbone, then loads them in one by one for + each layer that has weights associated with it. + + Args: + backbone: keras.Model backbone + weights_dict: Dictionary that stores the backbone model weights + backbone_name: String, indicating the desired backbone configuration + """ print("Loading backbone weights\n") backbone_layers = backbone.layers - cfgs = get_backbone_layer_cfgs(weights_dict) + cfgs = get_backbone_layer_cfgs(weights_dict, backbone_name) + n_weights_total = 0 cfg = cfgs.pop(0) for i in range(len(backbone_layers)): layer = backbone_layers[i] if isinstance(layer, (ConvBN, HourglassBlock, ResidualBlock)): - print("Loading weights for: {}".format(cfg)) - cfg.load_weights(layer) + n_weights = cfg.load_weights(layer) + print("Loading weights for: {}, weights loaded: {}".format(cfg, n_weights)) + n_weights_total += n_weights if len(cfgs) == 0: - print("Weights have been loaded for {} / {} layers\n".format(i+1, len(backbone_layers))) + print("{} Weights have been loaded for {} / {} layers\n".format(n_weights_total, i+1, len(backbone_layers))) return cfg = cfgs.pop(0) -def get_decoder_layer_cfgs(weights_dict): +def get_decoder_layer_cfgs(weights_dict, decoder_name): + """ Fetches the config classes for the decoder. + + This function generates a list of config classes corresponding to + each building block in the decoder. + + Args: + weights_dict: Dictionary that stores the decoder model weights + decoder_name: String, indicating the desired decoder configuration + Returns: + A list containing the config classe of the backbone building block + """ print("Fetching decoder config classes\n") - cfgs = [ - decoderConvCFG(weights_dict=weights_dict['object_center']['0']), - decoderConvCFG(weights_dict=weights_dict['object_center']['1']), - decoderConvCFG(weights_dict=weights_dict['box.Soffset']['0']), - decoderConvCFG(weights_dict=weights_dict['box.Soffset']['1']), - decoderConvCFG(weights_dict=weights_dict['box.Sscale']['0']), - decoderConvCFG(weights_dict=weights_dict['box.Sscale']['1']) - ] + + cfgs = DecoderConfigData(weights_dict=weights_dict).get_cfg_list(decoder_name) return cfgs -def load_weights_decoder(decoder, weights_dict): +def load_weights_decoder(decoder, weights_dict, decoder_name): + """ Loads the weights defined in the weights_dict into the decoder. + + This function loads the decoder weights by first fetching the necesary + config classes for the decoder, then loads them in one by one for + each layer that has weights associated with it. + + Args: + decoder: keras.Model decoder + weights_dict: Dictionary that stores the decoder model weights + decoder_name: String, indicating the desired decoder configuration + """ print("Loading decoder weights\n") decoder_layers = decoder.layers - cfgs = get_decoder_layer_cfgs(weights_dict) + cfgs = get_decoder_layer_cfgs(weights_dict, decoder_name) + n_weights_total = 0 cfg = cfgs.pop(0) for i in range(len(decoder_layers)): layer = decoder_layers[i] if isinstance(layer, CenterNetDecoderConv): - print("Loading weights for: {}".format(cfg)) - cfg.load_weights(layer) + n_weights = cfg.load_weights(layer) + print("Loading weights for: {}, weights loaded: {}".format(cfg, n_weights)) + n_weights_total += n_weights if len(cfgs) == 0: - print("Weights have been loaded for {} / {} layers\n".format(i+1, len(decoder_layers))) + print("{} Weights have been loaded for {} / {} layers\n".format(n_weights_total, i+1, len(decoder_layers))) return cfg = cfgs.pop(0) -def get_model_weights_as_dict(ckpt_path): - print("\nConverting model checkpoint from {} to weights dictionary\n".format(ckpt_path)) - reader = tf.train.load_checkpoint(ckpt_path) - shape_from_key = reader.get_variable_to_shape_map() - dtype_from_key = reader.get_variable_to_dtype_map() - - variable_keys = shape_from_key.keys() - weights_dict = {} - - for key in variable_keys: - shape = shape_from_key[key] - dtype = dtype_from_key[key] - value = reader.get_tensor(key) - update_weights_dict(weights_dict, key, value) - - print("Successfully read checkpoint weights\n") - return weights_dict - if __name__ == '__main__': input_specs = tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]) config = CenterNetTask() model, loss = build_centernet(input_specs=input_specs, task_config=config, l2_regularization=0) - + weights_dict = get_model_weights_as_dict(CKPT_PATH) - load_weights_model(model, weights_dict) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') - # print_dict_as_tree(weights_dict, 0) \ No newline at end of file + # Uncomment line below to write weights dict key names to a file + # write_dict_as_tree(weights_dict, filename="centernet/utils/weight_utils/MODEL_VARS.txt") \ No newline at end of file diff --git a/centernet/utils/weight_utils/tf_to_dict.py b/centernet/utils/weight_utils/tf_to_dict.py new file mode 100644 index 000000000..2739e7383 --- /dev/null +++ b/centernet/utils/weight_utils/tf_to_dict.py @@ -0,0 +1,87 @@ +import tensorflow as tf + +def update_weights_dict(weights_dict, variable_key, value): + """ Inserts weight value into a weight dictionary. + + This function inserts a weight value into a weights_dict based on the + variable key. It is designed to organize TF checkpoint weights by organizing + them by submodules. + + Args: + weights_dict: Dictionary to store weights + variable_key: String, name of the variable assocaited with the value + value: An ndarray that stores the weights assocaited to the variable key + """ + current_dict = weights_dict + variable_key_list = variable_key.split('/') + + key = variable_key_list.pop(0) + while len(variable_key_list): + if variable_key_list[0] == '.ATTRIBUTES': + current_dict[key] = value + return + + if key not in current_dict.keys(): + current_dict[key] = {} + current_dict = current_dict[key] + key = variable_key_list.pop(0) + +def get_model_weights_as_dict(ckpt_path): + """ Converts a TF checkpoint into a nested dictionary of weights. + + Args: + ckpt_path: String, indicating filepath of the TF checkpoint + Returns: + Dictionary where the checkpoint weights are stored + """ + print("\nConverting model checkpoint from {} to weights dictionary\n".format(ckpt_path)) + reader = tf.train.load_checkpoint(ckpt_path) + shape_from_key = reader.get_variable_to_shape_map() + dtype_from_key = reader.get_variable_to_dtype_map() + + variable_keys = shape_from_key.keys() + weights_dict = {} + + for key in variable_keys: + shape = shape_from_key[key] + dtype = dtype_from_key[key] + value = reader.get_tensor(key) + update_weights_dict(weights_dict, key, value) + + print("Successfully read checkpoint weights\n") + return weights_dict + +def write_dict_as_tree(dictionary, filename, spaces=0): + """ Writes nested dictionary keys to a file. + + Given a dictionary that contains nested dictionaries, this function + writes the name of the keys recursively to a specified file as a tree + + Args: + dictionary: Desired dictionary to write to a file + filename: String, name of file to write dictionary to + spaces: Optional; Number of spaces to insert before writing + the dictionary key names + """ + if type(dictionary) is dict: + mode = "w" if spaces == 0 else "a" + for key in dictionary.keys(): + with open(filename, mode) as fp: + fp.write(" " * spaces + key + '\n') + mode = "a" + write_dict_as_tree(dictionary[key], filename, spaces + 2) + +def print_layer_weights_and_shape(layer): + """ Prints variables information corresponding to a Keras layer. + + This function prints the name and the shape of its associated weights + of all variables (trainable and untrainable) in a Keras layer. + + Args: + layer: A Keras.layer.Layer object + """ + weights = layer.get_weights() + variables = layer.variables + + for i in range(len(weights)): + tf.print(np.shape(weights[i]), variables[i].name) \ No newline at end of file From b6a421e544853619036478fccfcbc84c029c1358 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 8 Mar 2021 00:00:13 -0500 Subject: [PATCH 058/132] fixed up some tests --- centernet/modeling/CenterNet_test.py | 2 +- centernet/modeling/decoders/centernet_decoder_test.py | 4 ++-- centernet/tasks/centernet_test.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/centernet/modeling/CenterNet_test.py b/centernet/modeling/CenterNet_test.py index cee180ac4..2b207aa74 100644 --- a/centernet/modeling/CenterNet_test.py +++ b/centernet/modeling/CenterNet_test.py @@ -24,7 +24,7 @@ def testBuildCenterNet(self): self.assertEqual(len(outputs['raw_output']['ct_heatmaps']), 2) self.assertEqual(len(outputs['raw_output']['ct_offset']), 2) self.assertEqual(len(outputs['raw_output']['ct_size']), 2) - self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (5, 128, 128, 80)) + self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (5, 128, 128, 90)) self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (5, 128, 128, 2)) self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (5, 128, 128, 2)) diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index 2dc40ccaa..6a8d65464 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -28,7 +28,7 @@ def test_decoder_shape(self): outputs = decoder([np.zeros((2, 128, 128, 256), dtype=np.float32), np.zeros((2, 128, 128, 256), dtype=np.float32)]) self.assertEqual(len(outputs), 3) - self.assertEqual(outputs['ct_heatmaps'][0].shape, (2, 128, 128, 80)) + self.assertEqual(outputs['ct_heatmaps'][0].shape, (2, 128, 128, 90)) self.assertEqual(outputs['ct_offset'][0].shape, (2, 128, 128, 2)) self.assertEqual(outputs['ct_size'][0].shape, (2, 128, 128, 2)) @@ -38,7 +38,7 @@ def test_decoder_shape(self): size_bias_vector = np.asarray(decoder.layers[3].weights[-1]) self.assertArrayNear(hm_bias_vector, - np.repeat(-2.19, repeats=80), err=1.00e-6) + np.repeat(-2.19, repeats=90), err=1.00e-6) self.assertArrayNear(off_bias_vector, np.repeat(0, repeats=2), err=1.00e-6) self.assertArrayNear(size_bias_vector, diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index f28999c6a..b0d4e7fad 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -21,9 +21,9 @@ def testCenterNetTask(self): outputs = model(tf.zeros((3, 512, 512, 3))) self.assertEqual(len(outputs['raw_output']), 3) - self.assertEqual(outputs['raw_output']['ct_heatmaps'].shape, (3, 128, 128, 80)) - self.assertEqual(outputs['raw_output']['ct_offset'].shape, (3, 128, 128, 2)) - self.assertEqual(outputs['raw_output']['ct_size'].shape, (3, 128, 128, 2)) + self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (3, 128, 128, 90)) + self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (3, 128, 128, 2)) + self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (3, 128, 128, 2)) model.summary() From 51fdb1416cde67f66445ad5ab5eade4f2e370b9f Mon Sep 17 00:00:00 2001 From: anivegesana Date: Tue, 9 Mar 2021 01:23:18 -0500 Subject: [PATCH 059/132] Rename inputs to match loss names --- centernet/dataloaders/centernet_input.py | 48 ++++++++++++------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index ff1d408d3..d22033947 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -26,12 +26,12 @@ def _parse_train_data(self, decoded_tensors): tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - tl_regrs = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - br_regrs = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - ct_regrs = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - tl_tags = tf.zeros((self._max_num_instances), dtype=tf.int64) - br_tags = tf.zeros((self._max_num_instances), dtype=tf.int64) - ct_tags = tf.zeros((self._max_num_instances), dtype=tf.int64) + tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) # TODO: input size, output size @@ -90,30 +90,30 @@ def _parse_train_data(self, decoded_tensors): br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ytl, xtl]], [1]) ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, ytl, xtl]], [1]) - # tl_regrs[tag_ind, :] = [fxtl - xtl, fytl - ytl] - # br_regrs[tag_ind, :] = [fxbr - xbr, fybr - ybr] - # ct_regrs[tag_ind, :] = [fxct - xct, fyct - yct] - # tl_tags[tag_ind] = ytl * output_size[1] + xtl - # br_tags[tag_ind] = ybr * output_size[1] + xbr - # ct_tags[tag_ind] = yct * output_size[1] + xct - tl_regrs = tf.tensor_scatter_nd_update(tl_regrs, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - br_regrs = tf.tensor_scatter_nd_update(br_regrs, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - ct_regrs = tf.tensor_scatter_nd_update(ct_regrs, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - tl_tags = tf.tensor_scatter_nd_update(tl_tags, [[tag_ind]], [ytl * output_size[1] + xtl]) - br_tags = tf.tensor_scatter_nd_update(br_tags, [[tag_ind]], [ybr * output_size[1] + xbr]) - ct_tags = tf.tensor_scatter_nd_update(ct_tags, [[tag_ind]], [yct * output_size[1] + xct]) + # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] + # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] + # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] + # tl_size[tag_ind] = ytl * output_size[1] + xtl + # br_size[tag_ind] = ybr * output_size[1] + xbr + # ct_size[tag_ind] = yct * output_size[1] + xct + tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) + tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) + br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) + ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) labels = { - 'tl_tags': tl_tags, - 'br_tags': br_tags, - 'ct_tags': ct_tags, + 'tl_size': tl_size, + 'br_size': br_size, + 'ct_size': ct_size, 'tl_heatmaps': tl_heatmaps, 'br_heatmaps': br_heatmaps, 'ct_heatmaps': ct_heatmaps, 'tag_masks': tag_masks, - 'tl_regrs': tl_regrs, - 'br_regrs', br_regrs, - 'ct_regrs': ct_regrs, + 'tl_offset': tl_offset, + 'br_offset', br_offset, + 'ct_offset': ct_offset, } return image, labels From d65132127a79df4d571cd12d765dfd12ac3f0e8e Mon Sep 17 00:00:00 2001 From: patel996 Date: Wed, 10 Mar 2021 11:35:01 -0500 Subject: [PATCH 060/132] loss testing --- centernet/losses/l1_localization_loss.py | 2 +- ...enalty_reduced_logistic_focal_loss_test.py | 2 +- centernet/ops/loss_utils.py | 4 +- centernet/tasks/centernet_object_detection.py | 31 +++-- .../tasks/centernet_object_detection_test.py | 106 ++++++++++++++++-- 5 files changed, 114 insertions(+), 31 deletions(-) diff --git a/centernet/losses/l1_localization_loss.py b/centernet/losses/l1_localization_loss.py index 2753044fe..bc8fab799 100644 --- a/centernet/losses/l1_localization_loss.py +++ b/centernet/losses/l1_localization_loss.py @@ -68,7 +68,7 @@ def __call__(self, y_true, y_pred, sample_weight=None): return absolute_difference( y_true, y_pred, - weights=sample_weight, + # weights=sample_weight, reduction=self._get_reduction() ) diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py index 632ad56ab..96cbd96ca 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py @@ -1,7 +1,7 @@ import tensorflow as tf import numpy as np -import penalty_reduced_logistic_focal_loss as losses +import centernet.losses.penalty_reduced_logistic_focal_loss as losses LOG_2 = np.log(2) LOG_3 = np.log(3) diff --git a/centernet/ops/loss_utils.py b/centernet/ops/loss_utils.py index 2e0980cff..8aaa1f9fb 100644 --- a/centernet/ops/loss_utils.py +++ b/centernet/ops/loss_utils.py @@ -4,7 +4,7 @@ def _to_float32(x): return tf.cast(x, tf.float32) def _get_shape(tensor, num_dims): - assert len(tensor.shape) == num_dims + assert len(tensor.shape.as_list()) == num_dims return combined_static_and_dynamic_shape(tensor) def combined_static_and_dynamic_shape(tensor): @@ -16,7 +16,7 @@ def combined_static_and_dynamic_shape(tensor): Returns: A list of size tensor.shape.ndims containing integers or a scalar tensor. """ - static_tensor_shape = tensor.shape + static_tensor_shape = tensor.shape.as_list() dynamic_tensor_shape = tf.shape(tensor) combined_shape = [] for index, dim in enumerate(static_tensor_shape): diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py index 6818bb1bf..c2363a8f5 100644 --- a/centernet/tasks/centernet_object_detection.py +++ b/centernet/tasks/centernet_object_detection.py @@ -36,39 +36,36 @@ def build_losses(self, outputs, labels, aux_losses=None): # TODO: Calculate loss flattened_ct_heatmaps = utils._flatten_spatial_dimensions(labels['ct_heatmaps']) - num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) #gt_weights_list here shouldn't be tag_masks here + num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) # Loop through each feature output head. - for pred in outputs['ct_heatmaps']: - pred = utils._flatten_spatial_dimensions(pred) - total_loss += object_center_loss( - flattened_ct_heatmaps, pred) #removed weight parameter (weight = per_pixel_weight) + outputs['ct_heatmaps'] = utils._flatten_spatial_dimensions(outputs['ct_heatmaps']) + total_loss += object_center_loss( + flattened_ct_heatmaps, outputs['ct_heatmaps']) #removed weight parameter (weight = per_pixel_weight) center_loss = tf.reduce_sum(total_loss) / ( float(len(outputs['ct_heatmaps'])) * num_boxes) loss += center_loss metric_dict['ct_loss'] = center_loss - #localization loss for offset and scale loss localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) - for scale_pred, offset_pred in zip(outputs['ct_size'], outputs['ct_offset']): - # Compute the scale loss. - scale_pred = utils.get_batch_predictions_from_indices( - scale_pred, labels['tag_locs']) - total_scale_loss += localization_loss_fn( - labels['ct_size'], scale_pred) #removed weights=batch_weights - # Compute the offset loss. - offset_pred = utils.get_batch_predictions_from_indices( - offset_pred, labels['tag_locs']) - total_offset_loss += localization_loss_fn( - labels['ct_offset'], offset_pred) #removed weights=batch_weights + # Compute the scale loss. + scale_pred = outputs['ct_size'] + offset_pred = outputs['ct_offset'] + total_scale_loss += localization_loss_fn( + labels['ct_size'], scale_pred) #removed weights=batch_weights + # Compute the offset loss. + total_offset_loss += localization_loss_fn( + labels['ct_offset'], offset_pred) #removed weights=batch_weights scale_loss += tf.reduce_sum(total_scale_loss) / ( float(len(outputs['ct_size'])) * num_boxes) offset_loss += tf.reduce_sum(total_offset_loss) / ( float(len(outputs['ct_size'])) * num_boxes) + metric_dict['ct_scale_loss'] = scale_loss metric_dict['ct_offset_loss'] = offset_loss + print(metric_dict) return loss, metric_dict def build_metrics(self, training=True): diff --git a/centernet/tasks/centernet_object_detection_test.py b/centernet/tasks/centernet_object_detection_test.py index 69a5f23e3..302fe2b2c 100644 --- a/centernet/tasks/centernet_object_detection_test.py +++ b/centernet/tasks/centernet_object_detection_test.py @@ -1,7 +1,10 @@ import tensorflow as tf import numpy as np import centernet.tasks as tasks -import centernet.utils as utils +import centernet.ops.loss_utils as utils +from centernet.losses import penalty_reduced_logistic_focal_loss +from centernet.losses import l1_localization_loss +from centernet.tasks.centernet import CenterNetTask def gaussian2D(shape, sigma=1): m, n = [(ss - 1.) / 2. for ss in shape] @@ -49,19 +52,39 @@ def gaussian_radius(det_size, min_overlap): return min(r1, r2, r3) def generate_heatmaps(batch_size, categories, output_size, detections, gaussian_iou=0.7): + + max_tag_len = 1 + tl_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) br_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) ct_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) + + tl_regrs = np.zeros((max_tag_len, 2), dtype=np.float32) + br_regrs = np.zeros((max_tag_len, 2), dtype=np.float32) + ct_regrs = np.zeros((max_tag_len, 2), dtype=np.float32) + tl_tags = np.zeros((max_tag_len), dtype=np.int64) + br_tags = np.zeros((max_tag_len), dtype=np.int64) + ct_tags = np.zeros((max_tag_len), dtype=np.int64) + tag_lens = np.zeros((batch_size, ), dtype=np.int32) + + width_ratio = 1 + height_ratio = 1 for b_ind, detection_batches in enumerate(detections): for ind, detection in enumerate(detection_batches): category = int(detection[-1]) #category = 0 - xtl, ytl = detection[0], detection[1] xbr, ybr = detection[2], detection[3] xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + xtl = int(xtl) ytl = int(ytl) xbr = int(xbr) @@ -78,18 +101,76 @@ def generate_heatmaps(batch_size, categories, output_size, detections, gaussian_ draw_gaussian(tl_heatmaps[b_ind, category], [xtl, ytl], radius) draw_gaussian(br_heatmaps[b_ind, category], [xbr, ybr], radius) draw_gaussian(ct_heatmaps[b_ind, category], [xct, yct], radius, delte = 5) + + tl_regrs[ind, :] = [fxtl - xtl, fytl - ytl] + br_regrs[ind, :] = [fxbr - xbr, fybr - ybr] + ct_regrs[ind, :] = [fxct - xct, fyct - yct] + tl_tags[ind] = ytl * output_size[1] + xtl + br_tags[ind] = ybr * output_size[1] + xbr + ct_tags[ind] = yct * output_size[1] + xct + + ct_regrs = tf.convert_to_tensor(ct_regrs, dtype=np.float32) + ct_tags = tf.convert_to_tensor(ct_tags, dtype=np.int64) + ct_heatmaps = tf.convert_to_tensor(ct_heatmaps, dtype=np.float32) - return tl_heatmaps, br_heatmaps, ct_heatmaps + return tl_heatmaps, br_heatmaps, ct_heatmaps, tl_regrs, br_regrs, ct_regrs, tl_tags, br_tags, ct_tags class ObjectDetectionTest(tf.test.TestCase): - def generate_heatmaps(self, dectections): - detections = [[ - (10, 30, 15, 17, 0) - ]] - tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) - pass + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.labels_heatmaps = tf.constant([[ + (10, 25, 17, 18, 0) + ]]) + + self.predicted = tf.constant([[ + (10, 30, 15, 17, 0) + ]]) + + def generate_heatmaps(self, dectections): + labels = dict() + outputs = dict() + + tl_heatmaps, br_heatmaps, ct_heatmaps, tl_regrs, br_regrs, ct_regrs, tl_tags, br_tags, ct_tags = generate_heatmaps(1, 2, (416, 416), self.predicted) + ct_heatmaps = tf.reshape(ct_heatmaps, [1, 416, 416, 2]) + + tl_labels_heatmaps, br_labels_heatmaps, ct_labels_heatmaps, tl_regrs_labels, br_regrs_labels, ct_regrs_labels, tl_tags_labels, br_tags_labels, ct_tags_labels = generate_heatmaps(1, 2, (416, 416), self.actual) + ct_labels_heatmaps = tf.reshape(ct_labels_heatmaps, [1, 416, 416, 2]) + + tag_masks = [[[True]]] + + labels = { + 'tl_size': tl_tags_labels, + 'br_size': br_tags_labels, + 'ct_size': ct_tags_labels, + 'tl_heatmaps': tl_labels_heatmaps, + 'br_heatmaps': br_labels_heatmaps, + 'ct_heatmaps': ct_labels_heatmaps, + 'tag_masks': tag_masks, + 'tl_offset': tl_regrs_labels, + 'br_offset': br_regrs_labels, + 'ct_offset': ct_regrs_labels, + } + + outputs = { + 'tl_size': tl_tags, + 'br_size': br_tags, + 'ct_size': ct_tags, + 'tl_heatmaps': tl_heatmaps, + 'br_heatmaps': br_heatmaps, + 'ct_heatmaps': ct_heatmaps, + 'tag_masks': tag_masks, + 'tl_offset': tl_regrs, + 'br_offset': br_regrs, + 'ct_offset': ct_regrs, + } + + task = CenterNetTask(None) + loss, metric = task.build_losses(outputs, labels) + + pass if __name__ == '__main__': + ''' # This code is for visualization import matplotlib.pyplot as plt detections = [[ @@ -101,7 +182,12 @@ def generate_heatmaps(self, dectections): ] tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) # ct_heatmaps[batch_id, class_id, ...] + plt.imshow(ct_heatmaps[0, 0, ...]) plt.show() # This is to run the test - # tf.test.main() + #plt.imshow(ct_heatmaps[0, 0, ...]) + #plt.imshow(labels_heatmaps[0, 0, ...]) + #plt.show() + ''' + tf.test.main() From eb27a304be7db831b42a3b1da706ceb098ac5416 Mon Sep 17 00:00:00 2001 From: patel996 Date: Wed, 10 Mar 2021 13:05:03 -0500 Subject: [PATCH 061/132] Merge branch 'centernet-oap' into centernet-loss --- centernet/configs/backbones.py | 5 + centernet/configs/centernet.py | 28 +- centernet/dataloaders/centernet_input.py | 110 ++ centernet/modeling/CenterNet.py | 107 ++ centernet/modeling/CenterNet_test.py | 36 + centernet/modeling/backbones/hourglass.py | 18 +- .../modeling/decoders/centernet_decoder.py | 41 +- .../decoders/centernet_decoder_test.py | 28 +- centernet/modeling/layers/nn_blocks.py | 93 +- centernet/ops/{loss_utils.py => loss_ops.py} | 0 centernet/ops/preprocessing_ops.py | 2 + centernet/ops/preprocessing_ops_test.py | 183 +++ centernet/tasks/_centernet_input.py | 331 ++++ centernet/tasks/centernet.py | 130 ++ centernet/tasks/centernet_object_detection.py | 2 +- .../tasks/centernet_object_detection_test.py | 51 +- centernet/tasks/centernet_test.py | 32 + centernet/utils/__init__.py | 0 centernet/utils/groundtruth.py | 99 ++ centernet/utils/groundtruth_test.py | 3 + centernet/utils/weight_utils/MODEL_VARS.txt | 1386 +++++++++++++++++ centernet/utils/weight_utils/__init__.py | 0 .../utils/weight_utils/config_classes.py | 204 +++ centernet/utils/weight_utils/config_data.py | 48 + centernet/utils/weight_utils/load_weights.py | 140 ++ centernet/utils/weight_utils/tf_to_dict.py | 87 ++ 26 files changed, 3078 insertions(+), 86 deletions(-) create mode 100644 centernet/dataloaders/centernet_input.py create mode 100644 centernet/modeling/CenterNet.py create mode 100644 centernet/modeling/CenterNet_test.py rename centernet/ops/{loss_utils.py => loss_ops.py} (100%) create mode 100644 centernet/ops/preprocessing_ops.py create mode 100644 centernet/ops/preprocessing_ops_test.py create mode 100644 centernet/tasks/_centernet_input.py create mode 100644 centernet/tasks/centernet.py create mode 100644 centernet/tasks/centernet_test.py create mode 100644 centernet/utils/__init__.py create mode 100644 centernet/utils/groundtruth.py create mode 100644 centernet/utils/groundtruth_test.py create mode 100644 centernet/utils/weight_utils/MODEL_VARS.txt create mode 100644 centernet/utils/weight_utils/__init__.py create mode 100644 centernet/utils/weight_utils/config_classes.py create mode 100644 centernet/utils/weight_utils/config_data.py create mode 100644 centernet/utils/weight_utils/load_weights.py create mode 100644 centernet/utils/weight_utils/tf_to_dict.py diff --git a/centernet/configs/backbones.py b/centernet/configs/backbones.py index 9338e34e3..c7bb53388 100644 --- a/centernet/configs/backbones.py +++ b/centernet/configs/backbones.py @@ -20,6 +20,7 @@ import dataclasses from official.modeling import hyperparams +from official.vision.beta.configs import backbones @dataclasses.dataclass @@ -32,3 +33,7 @@ class Hourglass(hyperparams.Config): default_factory=lambda: [2, 2, 2, 2, 2, 4]) num_hourglasses: int = 2 initial_downsample: bool = True + +@dataclasses.dataclass +class Backbone(backbones.Backbone): + hourglass: Hourglass = Hourglass() \ No newline at end of file diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 288f17155..b94bcffcd 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Decoder configurations.""" -from typing import Dict +"""CenterNet configuration definition.""" +from typing import ClassVar, Dict, List, Optional, Tuple, Union # Import libraries import dataclasses @@ -22,11 +22,8 @@ from official.modeling import hyperparams from official.modeling.hyperparams import config_definitions as cfg from official.vision.beta.configs import common -from official.vision.beta.configs import backbones - from centernet.configs import backbones - @dataclasses.dataclass class Loss(hyperparams.Config): pass @@ -56,12 +53,6 @@ class CenterNetDecoder(hyperparams.Config): heatmap_bias: float = -2.19 -@dataclasses.dataclass -class CenterNet(hyperparams.Config): - num_classes: int = 80 - decoder: CenterNetDecoder = CenterNetDecoder() - - @dataclasses.dataclass class CenterNetDetection(cfg.TaskConfig): use_centers: bool = True @@ -78,6 +69,17 @@ class CenterNetSubTasks(cfg.TaskConfig): # reid: bool = False # temporal: bool = False +@dataclasses.dataclass +class CenterNetBase(hyperparams.OneOfConfig): + backbone: backbones.Backbone = backbones.Backbone(type='hourglass') + decoder: CenterNetDecoder = CenterNetDecoder() + +@dataclasses.dataclass +class CenterNet(hyperparams.Config): + num_classes: int = 90 + input_size: Optional[List[int]] = dataclasses.field( + default_factory=lambda: [None, None, 3]) + base: Union[str, CenterNetBase] = CenterNetBase() @dataclasses.dataclass class CenterNetTask(cfg.TaskConfig): @@ -85,6 +87,8 @@ class CenterNetTask(cfg.TaskConfig): subtasks: CenterNetSubTasks = CenterNetSubTasks() losses: Losses = Losses() + weight_decay: float = 5e-4 + def _get_output_length_dict(self): lengths = {} assert self.subtasks.detection is not None or self.subtasks.kp_detection \ @@ -129,4 +133,4 @@ def _get_output_length_dict(self): # 'joint_offset': 2 # }) - return lengths + return lengths \ No newline at end of file diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py new file mode 100644 index 000000000..f7611819a --- /dev/null +++ b/centernet/dataloaders/centernet_input.py @@ -0,0 +1,110 @@ +import tensorflow as tf +from official.vision.beta.dataloaders import parser + + +class CenterNetParser(parser.Parser): + def __init__( + self, + num_classes: int, + gaussian_iou: float + ): + self.num_classes = num_classes + self.gaussian_io = gaussian_iou + + def _parse_train_data(self, decoded_tensors): + """Generates images and labels that are usable for model training. + + Args: + decoded_tensors: a dict of Tensors produced by the decoder. + + Returns: + images: the image tensor. + labels: a dict of Tensors that contains labels. + """ + tl_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) + br_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) + ct_heatmaps = tf.zeros((categories, output_size[0], output_size[1]), dtype=tf.float32) + tl_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) + br_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) + ct_regrs = tf.zeros((max_tag_len, 2), dtype=tf.float32) + tl_tags = tf.zeros((max_tag_len), dtype=tf.int64) + br_tags = tf.zeros((max_tag_len), dtype=tf.int64) + ct_tags = tf.zeros((max_tag_len), dtype=tf.int64) + tag_masks = tf.zeros((max_tag_len), dtype=tf.uint8) + + # TODO: input size, output size + image = decoded_tensors["image"] + + width_ratio = output_size[1] / input_size[1] + height_ratio = output_size[0] / input_size[0] + + for ind, detection in enumerate(decoded_tensors["groundtruth_boxes"]): + category = int(detection[-1]) - 1 + # category = 0 + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + + xct, yct = ( + (detection[2] + detection[0]) / 2, + (detection[3] + detection[1]) / 2 + ) + + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + xtl = int(fxtl) + ytl = int(fytl) + xbr = int(fxbr) + ybr = int(fybr) + xct = int(fxct) + yct = int(fyct) + + if gaussian_bump: + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + width = math.ceil(width * width_ratio) + height = math.ceil(height * height_ratio) + + if gaussian_rad == -1: + radius = gaussian_radius((height, width), self.gaussian_iou) + radius = max(0, int(radius)) + else: + radius = gaussian_rad + draw_gaussian(tl_heatmaps[category], [xtl, ytl], radius) + draw_gaussian(br_heatmaps[category], [xbr, ybr], radius) + draw_gaussian(ct_heatmaps[category], [xct, yct], radius, delte=5) + + else: + tl_heatmaps[category, ytl, xtl] = 1 + br_heatmaps[category, ybr, xbr] = 1 + ct_heatmaps[category, yct, xct] = 1 + + tl_regrs[tag_ind, :] = [fxtl - xtl, fytl - ytl] + br_regrs[tag_ind, :] = [fxbr - xbr, fybr - ybr] + ct_regrs[tag_ind, :] = [fxct - xct, fyct - yct] + tl_tags[tag_ind] = ytl * output_size[1] + xtl + br_tags[tag_ind] = ybr * output_size[1] + xbr + ct_tags[tag_ind] = yct * output_size[1] + xct + + labels = { + 'tl_tags': tl_tags, + 'br_tags': br_tags, + 'ct_tags': ct_tags, + 'tl_heatmaps': tl_heatmaps, + 'br_heatmaps': br_heatmaps, + 'ct_heatmaps': ct_heatmaps, + 'tag_masks': tag_masks, + 'tl_regrs': tl_regrs, + 'br_regrs': br_regrs, + 'ct_regrs': ct_regrs, + } + return image, labels + + def _parse_eval_data(self, data): + pass diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py new file mode 100644 index 000000000..81dad60a5 --- /dev/null +++ b/centernet/modeling/CenterNet.py @@ -0,0 +1,107 @@ +from official.core import registry +from official.vision.beta.modeling.backbones import factory +import tensorflow as tf +import tensorflow.keras as ks + +from centernet.modeling.backbones.hourglass import Hourglass +from centernet.modeling.backbones.hourglass import build_hourglass +from centernet.modeling.decoders.centernet_decoder import CenterNetDecoder + +# TODO: import prediction and filtering layers when made + +class CenterNet(ks.Model): + + def __init__(self, + backbone=None, + decoder=None, + head=None, + filter=None, + **kwargs): + super().__init__(**kwargs) + # model components + self._backbone = backbone + self._decoder = decoder + self._head = head + self._filter = filter + return + + def build(self, input_shape): + self._backbone.build(input_shape) + nshape = self._backbone.output_specs + self._decoder.build(nshape) + super().build(input_shape) + + def call(self, inputs, training=False): + features = self._backbone(inputs) + # final_backbone_output = features[-1] + decoded_maps = self._decoder(features) + + # TODO: head + filters + + if training: + return {"raw_output": decoded_maps} + else: + # TODO: uncomment when filter is implemented + # predictions = self._filter(raw_predictions) + # predictions.update({"raw_output": raw_predictions}) + # return predictions + return {"raw_output": decoded_maps} + + @property + def backbone(self): + return self._backbone + + @property + def decoder(self): + return self._decoder + + @property + def head(self): + return self._head + + @property + def filter(self): + return self._filter + +def build_centernet_decoder(input_specs, task_config, num_inputs): + # NOTE: For now just support the default config + # model specific + heatmap_bias = task_config.model.base.decoder.heatmap_bias + + # task specific + task_outputs = task_config._get_output_length_dict() + model = CenterNetDecoder( + task_outputs=task_outputs, + heatmap_bias=heatmap_bias, + num_inputs=num_inputs) + + model.build(input_specs) + return model + +def build_centernet_filter(model_config): + return None + +def build_centernet_head(model_config): + return None + +def build_centernet(input_specs, task_config, l2_regularization): + print(task_config.as_dict()) + print(input_specs) + print(l2_regularization) + model_config = task_config.model + backbone = factory.build_backbone(input_specs, model_config.base, + l2_regularization) + + decoder = build_centernet_decoder(backbone.output_specs.as_list(), task_config, backbone._num_hourglasses) + head = build_centernet_head(model_config) + filter = build_centernet_filter(model_config) + + model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) + + model.build(input_specs.shape) + + # TODO: uncommend when filter is implemented + # losses = filter.losses + losses = None + return model, losses + diff --git a/centernet/modeling/CenterNet_test.py b/centernet/modeling/CenterNet_test.py new file mode 100644 index 000000000..2b207aa74 --- /dev/null +++ b/centernet/modeling/CenterNet_test.py @@ -0,0 +1,36 @@ +from absl.testing import parameterized +import tensorflow as tf +import numpy as np +import dataclasses + +from official.modeling import hyperparams +from official.vision.beta.configs import backbones + +from centernet.modeling.CenterNet import build_centernet +from centernet.configs import centernet + + +class CenterNetTest(parameterized.TestCase, tf.test.TestCase): + + def testBuildCenterNet(self): + input_specs = tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]) + + config = centernet.CenterNetTask() + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + + outputs = model(tf.zeros((5, 512, 512, 3))) + self.assertEqual(len(outputs['raw_output']), 3) + self.assertEqual(len(outputs['raw_output']['ct_heatmaps']), 2) + self.assertEqual(len(outputs['raw_output']['ct_offset']), 2) + self.assertEqual(len(outputs['raw_output']['ct_size']), 2) + self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (5, 128, 128, 90)) + self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (5, 128, 128, 2)) + self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (5, 128, 128, 2)) + + model.summary() + + + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 83cacaf28..a8afaddd9 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -33,7 +33,7 @@ def __init__( pre_layers: tf.keras layer to process input before stacked hourglasses """ # yapf: disable - input = tf.keras.layers.Input(shape=input_specs.shape[1:]) + input = tf.keras.layers.Input(shape=input_specs.shape[1:], name='input') x_inter = input # Create some intermediate and postlayers to generate the heatmaps @@ -53,10 +53,10 @@ def __init__( kernel_size=prelayer_kernel_size, strides=prelayer_strides, padding='same', - activation='relu' + activation='relu', )(x_inter) x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, strides=prelayer_strides + filters=inp_filters, use_projection=True, strides=prelayer_strides, )(x_inter) all_heatmaps = [] @@ -65,7 +65,7 @@ def __init__( # Create hourglass stacks x_hg = nn_blocks.HourglassBlock( channel_dims_per_stage=channel_dims_per_stage, - blocks_per_stage=blocks_per_stage + blocks_per_stage=blocks_per_stage, )(x_inter) # cnvs @@ -74,7 +74,7 @@ def __init__( kernel_size=(3, 3), strides=(1, 1), padding='same', - activation='relu' + activation='relu', )(x_hg) all_heatmaps.append(x_hg) @@ -99,12 +99,12 @@ def __init__( activation='linear' )(x_hg) - x_inter = inter_hg_conv1 + inter_hg_conv2 + x_inter = tf.keras.layers.Add()([inter_hg_conv1, inter_hg_conv2]) x_inter = tf.keras.layers.ReLU()(x_inter) # inters x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, strides=1 + filters=inp_filters, use_projection=False, strides=1 )(x_inter) # yapf: enable @@ -115,6 +115,7 @@ def __init__( self._blocks_per_stage = blocks_per_stage self._num_hourglasses = num_hourglasses self._initial_downsample = initial_downsample + self._output_specs = all_heatmaps[-1].get_shape() def get_config(self): layer_config = { @@ -127,6 +128,9 @@ def get_config(self): layer_config.update(super().get_config()) return layer_config + @property + def output_specs(self): + return self._output_specs # @factory.register_backbone_builder('hourglass') diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index e6ce00684..aa9978a61 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -10,6 +10,7 @@ class CenterNetDecoder(tf.keras.Model): def __init__(self, task_outputs: dict, heatmap_bias: float = -2.19, + num_inputs: int = 2, **kwargs): """ Args: @@ -23,30 +24,30 @@ def __init__(self, dictionary where the keys-value pairs denote the names of the output and the respective output tensor """ + super().__init__(**kwargs) self._task_outputs = task_outputs self._heatmap_bias = heatmap_bias + self._num_inputs = num_inputs - super().__init__(**kwargs) + self.outputs_layers = {} - def build(self, input_shape): - self.out_layers = {} + def build(self, inputs): for key in self._task_outputs: num_filters = self._task_outputs[key] bias = 0 if 'heatmaps' in key: bias = self._heatmap_bias - self.out_layers[key] = CenterNetDecoderConv(output_filters=num_filters, - name=key, bias_init=bias) - - super().build(input_shape) + self.outputs_layers[key] = [CenterNetDecoderConv(output_filters=num_filters, + name=key, bias_init=bias) for _ in range(self._num_inputs)] - def call(self, x): + def call(self, inputs): outputs = {} - for key in self.out_layers: - outputs[key] = self.out_layers[key](x) + for key in self._task_outputs: + outputs[key] = [self.outputs_layers[key][i](inputs[i]) for i in range(self._num_inputs)] + return outputs def get_config(self): @@ -58,12 +59,18 @@ def get_config(self): #layer_config.update(super().get_config()) return layer_config -def build_centernet_decoder(task_config: cfg.CenterNetTask) -> tf.keras.Model: +def build_centernet_decoder(input_specs, task_config, num_inputs): + # NOTE: For now just support the default config + + # model specific + heatmap_bias = task_config.model.base.decoder.heatmap_bias + + # task specific task_outputs = task_config._get_output_length_dict() - heatmap_bias = task_config.model.decoder.heatmap_bias - - print(task_outputs) - - return CenterNetDecoder( + model = CenterNetDecoder( task_outputs=task_outputs, - heatmap_bias=heatmap_bias) + heatmap_bias=heatmap_bias, + num_inputs=num_inputs) + + model.build(input_specs) + return model \ No newline at end of file diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index cf46022a3..6a8d65464 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -9,28 +9,36 @@ class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_create_decoder(self): - decoder = centernet_decoder.build_centernet_decoder(task_config=cfg.CenterNetTask()) + decoder = centernet_decoder.build_centernet_decoder( + task_config=cfg.CenterNetTask(), + input_specs=(None, 128, 128, 256), + num_inputs=2) + config = decoder.get_config() self.assertEqual(len(config), 2) self.assertEqual(config['heatmap_bias'], -2.19) def test_decoder_shape(self): - decoder = centernet_decoder.build_centernet_decoder(task_config=cfg.CenterNetTask()) + decoder = centernet_decoder.build_centernet_decoder( + task_config=cfg.CenterNetTask(), + input_specs=(2, 128, 128, 256), + num_inputs=2) # Output shape tests - outputs = decoder(np.zeros((2, 128, 128, 256), dtype=np.float32)) + outputs = decoder([np.zeros((2, 128, 128, 256), dtype=np.float32), + np.zeros((2, 128, 128, 256), dtype=np.float32)]) self.assertEqual(len(outputs), 3) - self.assertEqual(outputs['ct_heatmaps'].shape, (2, 128, 128, 80)) - self.assertEqual(outputs['ct_offset'].shape, (2, 128, 128, 2)) - self.assertEqual(outputs['ct_size'].shape, (2, 128, 128, 2)) + self.assertEqual(outputs['ct_heatmaps'][0].shape, (2, 128, 128, 90)) + self.assertEqual(outputs['ct_offset'][0].shape, (2, 128, 128, 2)) + self.assertEqual(outputs['ct_size'][0].shape, (2, 128, 128, 2)) # Weight initialization tests - hm_bias_vector = np.asarray(decoder.out_layers['ct_heatmaps'].weights[-1]) - off_bias_vector = np.asarray(decoder.out_layers['ct_offset'].weights[-1]) - size_bias_vector = np.asarray(decoder.out_layers['ct_size'].weights[-1]) + hm_bias_vector = np.asarray(decoder.layers[1].weights[-1]) + off_bias_vector = np.asarray(decoder.layers[2].weights[-1]) + size_bias_vector = np.asarray(decoder.layers[3].weights[-1]) self.assertArrayNear(hm_bias_vector, - np.repeat(-2.19, repeats=80), err=1.00e-6) + np.repeat(-2.19, repeats=90), err=1.00e-6) self.assertArrayNear(off_bias_vector, np.repeat(0, repeats=2), err=1.00e-6) self.assertArrayNear(size_bias_vector, diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 567bdeb52..db8bcf79f 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -209,37 +209,56 @@ def __init__(self, 'size and residual block repetition lists must have the same length' self._filters = channel_dims_per_stage[0] + if self._order > 0: + self._filters_downsampled = channel_dims_per_stage[1] self._reps = blocks_per_stage[0] - super().__init__() + super().__init__(**kwargs) + + def make_repeated_residual_blocks(self, reps, out_channels, + residual_channels=None, + initial_stride=1, initial_skip=False): + blocks = [] + + if residual_channels is None: + residual_channels = out_channels + + for i in range(reps - 1): + stride = initial_stride if i == 0 else 1 + skip_conv = stride > 1 + + blocks.append(official_nn_blocks.ResidualBlock( + filters=residual_channels, strides=stride, + use_projection=skip_conv)) + + if reps == 1: + stride = initial_stride + skip_conv = stride > 1 + else: + stride = 1 + skip_conv = residual_channels != out_channels + + blocks.append(official_nn_blocks.ResidualBlock( + filters=out_channels, strides=stride, + use_projection=skip_conv)) + + return tf.keras.Sequential(blocks) def build(self, input_shape): - if self._order == 1: + if self._order == 0: # base case, residual block repetitions in most inner part of hourglass - blocks = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.blocks = tf.keras.Sequential(blocks) + self.blocks = self.make_repeated_residual_blocks(reps=self._reps, + out_channels=self._filters) else: # outer hourglass structures - main_block = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.main_block = tf.keras.Sequential(main_block, name='Main_Block') - - side_block = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.side_block = tf.keras.Sequential(side_block, name='Side_Block') - - self.pool = tf.keras.layers.MaxPool2D(pool_size=2) + self.encoder_block1 = self.make_repeated_residual_blocks(reps=self._reps, + out_channels=self._filters) + + self.encoder_block2 = self.make_repeated_residual_blocks(reps=self._reps, + out_channels=self._filters_downsampled, initial_stride=2, initial_skip=self._filters != self._filters_downsampled) + + # self.pool = tf.keras.layers.MaxPool2D(pool_size=2) # recursively define inner hourglasses self.inner_hg = type(self)( @@ -248,12 +267,8 @@ def build(self, input_shape): strides=self._strides) # outer hourglass structures - end_block = [ - official_nn_blocks.ResidualBlock( - filters=self._filters, strides=self._strides, use_projection=True) - for _ in range(self._reps) - ] - self.end_block = tf.keras.Sequential(end_block, name='End_Block') + self.decoder_block = self.make_repeated_residual_blocks(reps=self._reps, + residual_channels=self._filters_downsampled, out_channels=self._filters) self.upsample_layer = tf.keras.layers.UpSampling2D( size=2, interpolation='nearest') @@ -261,15 +276,21 @@ def build(self, input_shape): super().build(input_shape) def call(self, x): - if self._order == 1: + if self._order == 0: return self.blocks(x) else: - x_pre_pooled = self.main_block(x) - x_side = self.side_block(x_pre_pooled) - x_pooled = self.pool(x_pre_pooled) - inner_output = self.inner_hg(x_pooled) - hg_output = self.end_block(inner_output) - return self.upsample_layer(hg_output) + x_side + encoded_outputs = self.encoder_block1(x) + encoded_downsampled_outputs = self.encoder_block2(x) + inner_outputs = self.inner_hg(encoded_downsampled_outputs) + hg_output = self.decoder_block(inner_outputs) + return self.upsample_layer(hg_output) + encoded_outputs + + # x_pre_pooled = self.encoder_block1(x) + # x_side = self.encoder_block2(x_pre_xpooled) + # x_pooled = self.pool(x_pre_pooled) + # inner_output = self.inner_hg(x_pooled) + # hg_output = self.decoder_block(inner_output) + # return self.upsample_layer(hg_output) + x_side def get_config(self): layer_config = { diff --git a/centernet/ops/loss_utils.py b/centernet/ops/loss_ops.py similarity index 100% rename from centernet/ops/loss_utils.py rename to centernet/ops/loss_ops.py diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py new file mode 100644 index 000000000..0dd42e503 --- /dev/null +++ b/centernet/ops/preprocessing_ops.py @@ -0,0 +1,2 @@ +# TODO: Move groundtruth.py to preprocessing_ops.py when possible +from centernet.utils.groundtruth import * diff --git a/centernet/ops/preprocessing_ops_test.py b/centernet/ops/preprocessing_ops_test.py new file mode 100644 index 000000000..44894df35 --- /dev/null +++ b/centernet/ops/preprocessing_ops_test.py @@ -0,0 +1,183 @@ +from absl.testing import parameterized +import tensorflow as tf +import numpy as np + +import centernet.utils.groundtruth as preprocessing_ops +from yolo.ops import box_ops + +def image_shape_to_grids(height, width): + """Computes xy-grids given the shape of the image. + Args: + height: The height of the image. + width: The width of the image. + Returns: + A tuple of two tensors: + y_grid: A float tensor with shape [height, width] representing the + y-coordinate of each pixel grid. + x_grid: A float tensor with shape [height, width] representing the + x-coordinate of each pixel grid. + """ + out_height = tf.cast(height, tf.float32) + out_width = tf.cast(width, tf.float32) + x_range = tf.range(out_width, dtype=tf.float32) + y_range = tf.range(out_height, dtype=tf.float32) + x_grid, y_grid = tf.meshgrid(x_range, y_range, indexing='xy') + return (y_grid, x_grid) + + +class CenterNetBoxTargetAssignerTest(parameterized.TestCase, tf.test.TestCase): + + def __init__(self, *args, **kwargs): + super(CenterNetBoxTargetAssignerTest, self).__init__(*args, **kwargs) + self._box_center = [0.0, 0.0, 1.0, 1.0] + self._box_center_small = [0.25, 0.25, 0.75, 0.75] + self._box_lower_left = [0.5, 0.0, 1.0, 0.5] + self._box_center_offset = [0.1, 0.05, 1.0, 1.0] + self._box_odd_coordinates = [0.1625, 0.2125, 0.5625, 0.9625] + + def test_max_distance_for_overlap(self): + """Test that the distance ensures the IoU with random boxes.""" + + # TODO(vighneshb) remove this after the `_smallest_positive_root` + # function if fixed. + self.skipTest(('Skipping test because we are using an incorrect version of' + 'the `max_distance_for_overlap` function to reproduce' + ' results.')) + + rng = np.random.RandomState(0) + n_samples = 100 + + width = rng.uniform(1, 100, size=n_samples) + height = rng.uniform(1, 100, size=n_samples) + min_iou = rng.uniform(0.1, 1.0, size=n_samples) + + max_dist = preprocessing_ops.gaussian_radius((height, width), min_iou) + + xmin1 = np.zeros(n_samples) + ymin1 = np.zeros(n_samples) + xmax1 = np.zeros(n_samples) + width + ymax1 = np.zeros(n_samples) + height + + xmin2 = max_dist * np.cos(rng.uniform(0, 2 * np.pi)) + ymin2 = max_dist * np.sin(rng.uniform(0, 2 * np.pi)) + xmax2 = width + max_dist * np.cos(rng.uniform(0, 2 * np.pi)) + ymax2 = height + max_dist * np.sin(rng.uniform(0, 2 * np.pi)) + + boxes1 = np.vstack([ymin1, xmin1, ymax1, xmax1]).T + boxes2 = np.vstack([ymin2, xmin2, ymax2, xmax2]).T + + iou = box_ops.compute_iou(boxes1, boxes2) + + self.assertTrue(np.all(iou >= min_iou)) + + def test_max_distance_for_overlap_centernet(self): + """Test the version of the function used in the CenterNet paper.""" + distance = preprocessing_ops.gaussian_radius((10, 5), 0.5) + self.assertAlmostEqual(2.807764064, distance.numpy()) + + @parameterized.parameters((False,), (True,)) + def test_coordinates_to_heatmap(self, sparse): + self.skipTest('Not yet functioning.') + + (y_grid, x_grid) = image_shape_to_grids(height=3, width=5) + y_coordinates = tf.constant([1.5, 0.5], dtype=tf.float32) + x_coordinates = tf.constant([2.5, 4.5], dtype=tf.float32) + sigma = tf.constant([0.1, 0.5], dtype=tf.float32) + channel_onehot = tf.constant([[1, 0, 0], [0, 1, 0]], dtype=tf.float32) + channel_weights = tf.constant([1, 1], dtype=tf.float32) + heatmap = ta_utils.coordinates_to_heatmap(y_grid, x_grid, y_coordinates, + x_coordinates, sigma, + channel_onehot, + channel_weights, sparse=sparse) + + # Peak at (1, 2) for the first class. + self.assertAlmostEqual(1.0, heatmap[1, 2, 0]) + # Peak at (0, 4) for the second class. + self.assertAlmostEqual(1.0, heatmap[0, 4, 1]) + + + + + def coordinates_to_heatmap(y_grid, + x_grid, + y_coordinates, + x_coordinates, + sigma, + channel_onehot, + channel_weights=None, + sparse=False): + """Returns the heatmap targets from a set of point coordinates. + This function maps a set of point coordinates to the output heatmap image + applied using a Gaussian kernel. Note that this function be can used by both + object detection and keypoint estimation tasks. For object detection, the + "channel" refers to the object class. For keypoint estimation, the "channel" + refers to the number of keypoint types. + Args: + y_grid: A 2D tensor with shape [height, width] which contains the grid + y-coordinates given in the (output) image dimensions. + x_grid: A 2D tensor with shape [height, width] which contains the grid + x-coordinates given in the (output) image dimensions. + y_coordinates: A 1D tensor with shape [num_instances] representing the + y-coordinates of the instances in the output space coordinates. + x_coordinates: A 1D tensor with shape [num_instances] representing the + x-coordinates of the instances in the output space coordinates. + sigma: A 1D tensor with shape [num_instances] representing the standard + deviation of the Gaussian kernel to be applied to the point. + channel_onehot: A 2D tensor with shape [num_instances, num_channels] + representing the one-hot encoded channel labels for each point. + channel_weights: A 1D tensor with shape [num_instances] corresponding to the + weight of each instance. + sparse: bool, indicating whether or not to use the sparse implementation + of the function. The sparse version scales better with number of channels, + but in some cases is known to cause OOM error. See (b/170989061). + Returns: + heatmap: A tensor of size [height, width, num_channels] representing the + heatmap. Output (height, width) match the dimensions of the input grids. + """ + + # if sparse: + # return _coordinates_to_heatmap_sparse( + # y_grid, x_grid, y_coordinates, x_coordinates, sigma, channel_onehot, + # channel_weights) + # else: + return _coordinates_to_heatmap_dense( + y_grid, x_grid, y_coordinates, x_coordinates, sigma, channel_onehot, + channel_weights) + + + def _coordinates_to_heatmap_dense(y_grid, x_grid, y_coordinates, x_coordinates, + sigma, channel_onehot, channel_weights=None): + """Dense version of coordinates to heatmap that uses an outer product.""" + num_instances, num_channels = ( + shape_utils.combined_static_and_dynamic_shape(channel_onehot)) + + x_grid = tf.expand_dims(x_grid, 2) + y_grid = tf.expand_dims(y_grid, 2) + # The raw center coordinates in the output space. + x_diff = x_grid - tf.math.floor(x_coordinates) + y_diff = y_grid - tf.math.floor(y_coordinates) + squared_distance = x_diff**2 + y_diff**2 + + gaussian_map = tf.exp(-squared_distance / (2 * sigma * sigma)) + + reshaped_gaussian_map = tf.expand_dims(gaussian_map, axis=-1) + reshaped_channel_onehot = tf.reshape(channel_onehot, + (1, 1, num_instances, num_channels)) + gaussian_per_box_per_class_map = ( + reshaped_gaussian_map * reshaped_channel_onehot) + + if channel_weights is not None: + reshaped_weights = tf.reshape(channel_weights, (1, 1, num_instances, 1)) + gaussian_per_box_per_class_map *= reshaped_weights + + # Take maximum along the "instance" dimension so that all per-instance + # heatmaps of the same class are merged together. + heatmap = tf.reduce_max(gaussian_per_box_per_class_map, axis=2) + + # Maximum of an empty tensor is -inf, the following is to avoid that. + heatmap = tf.maximum(heatmap, 0) + + return tf.stop_gradient(heatmap) + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/tasks/_centernet_input.py b/centernet/tasks/_centernet_input.py new file mode 100644 index 000000000..8321b1b23 --- /dev/null +++ b/centernet/tasks/_centernet_input.py @@ -0,0 +1,331 @@ +import tensorflow as tf +from tensorflow.keras.mixed_precision import experimental as mixed_precision + +from absl import logging +from official.core import base_task +from official.core import input_reader +from official.core import task_factory +from yolo.configs import yolo as exp_cfg + +from official.vision.beta.evaluation import coco_evaluator + +from yolo.dataloaders import yolo_input +from yolo.dataloaders.decoders import tfds_coco_decoder +from yolo.ops.kmeans_anchors import BoxGenInputReader +from yolo.ops.box_ops import xcycwh_to_yxyx + +from centernet.dataloaders import centernet_input + + +@task_factory.register_task_cls(exp_cfg.YoloTask) +class YoloTask(base_task.Task): + """A single-replica view of training procedure. + RetinaNet task provides artifacts for training/evalution procedures, including + loading/iterating over Datasets, initializing the model, calculating the loss, + post-processing, and customized metrics with reduction. + """ + + def __init__(self, params, logging_dir: str = None): + super().__init__(params, logging_dir) + self._loss_dict = None + self._num_boxes = None + self._anchors_built = False + + self._masks = None + self._path_scales = None + self._x_y_scales = None + self.coco_metric = None + return + + def build_model(self): + """get an instance of Yolo v3 or v4""" + from yolo.modeling.Yolo import build_yolo + params = self.task_config.train_data + model_base_cfg = self.task_config.model + l2_weight_decay = self.task_config.weight_decay / 2.0 + + masks, path_scales, xy_scales = self._get_masks() + self._get_boxes(gen_boxes=params.is_training) + + input_specs = tf.keras.layers.InputSpec(shape=[None] + + model_base_cfg.input_size) + l2_regularizer = ( + tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) + + model, losses = build_yolo(input_specs, model_base_cfg, l2_regularizer, + masks, xy_scales, path_scales) + self._loss_dict = losses + return model + + def build_losses(self, outputs, labels, aux_losses=None): + loss = 0.0 + loss_box = 0.0 + loss_conf = 0.0 + loss_class = 0.0 + metric_dict = dict() + + grid = labels['grid_form'] + for key in outputs.keys(): + # _loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](labels, outputs[key]) + _loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](grid[key], outputs[key]) + #_loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](labels[key], outputs[key]) + loss += _loss + loss_box += _loss_box + loss_conf += _loss_conf + loss_class += _loss_class + metric_dict[f"recall50_{key}"] = tf.stop_gradient(_recall50) + metric_dict[f"avg_iou_{key}"] = tf.stop_gradient(_avg_iou) + + metric_dict['box_loss'] = loss_box + metric_dict['conf_loss'] = loss_conf + metric_dict['class_loss'] = loss_class + + return loss, metric_dict + + def build_metrics(self, training=True): + #return super().build_metrics(training=training) + if not training: + self.coco_metric = coco_evaluator.COCOEvaluator( + annotation_file=self.task_config.annotation_file, + include_mask=False, + need_rescale_bboxes=False, + per_category_metrics=self._task_config.per_category_metrics) + return [] + + def train_step(self, inputs, model, optimizer, metrics=None): + #get the data point + image, label = inputs + num_replicas = tf.distribute.get_strategy().num_replicas_in_sync + with tf.GradientTape() as tape: + # compute a prediction + # cast to float32 + y_pred = model(image, training=True) + loss, metrics = self.build_losses(y_pred['raw_output'], label) + scaled_loss = loss / num_replicas + + # scale the loss for numerical stability + if isinstance(optimizer, mixed_precision.LossScaleOptimizer): + scaled_loss = optimizer.get_scaled_loss(scaled_loss) + # compute the gradient + train_vars = model.trainable_variables + gradients = tape.gradient(scaled_loss, train_vars) + # get unscaled loss if the scaled_loss was used + if isinstance(optimizer, mixed_precision.LossScaleOptimizer): + gradients = optimizer.get_unscaled_gradients(gradients) + if self.task_config.gradient_clip_norm > 0.0: + gradients, _ = tf.clip_by_global_norm(gradients, + self.task_config.gradient_clip_norm) + optimizer.apply_gradients(zip(gradients, train_vars)) + + #custom metrics + logs = {'loss': loss} + logs.update(metrics) + + #tf.print("loss: ", logs["loss"], end = "\n") + tf.print(logs, end='\n') + + ret = '\033[F' * (len(logs.keys()) + 1) + tf.print(ret, end='\n') + + return logs + + def validation_step(self, inputs, model, metrics=None): + #get the data point + image, label = inputs + + # computer detivative and apply gradients + y_pred = model(image, training=False) + loss, metrics = self.build_losses(y_pred['raw_output'], label) + + # #custom metrics + loss_metrics = {'loss': loss} + loss_metrics.update(metrics) + label['boxes'] = xcycwh_to_yxyx(label['bbox']) + del label['bbox'] + + coco_model_outputs = { + 'detection_boxes': y_pred['bbox'], + 'detection_scores': y_pred['confidence'], + 'detection_classes': y_pred['classes'], + 'num_detections': tf.shape(y_pred['bbox'])[:-1], + 'source_id': label['source_id'], + } + + loss_metrics.update({self.coco_metric.name: (label, coco_model_outputs)}) + return loss_metrics + + def aggregate_logs(self, state=None, step_outputs=None): + #return super().aggregate_logs(state=state, step_outputs=step_outputs) + if not state: + self.coco_metric.reset_states() + state = self.coco_metric + self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], + step_outputs[self.coco_metric.name][1]) + return state + + def reduce_aggregated_logs(self, aggregated_logs): + #return super().reduce_aggregated_logsI(aggregated_logs) + return self.coco_metric.result() + + @property + def anchors(self): + return self.task_config.model.boxes + + def _get_boxes(self, gen_boxes=True): + # gen_boxes = params.is_training + if gen_boxes and self.task_config.model.boxes == None and not self._anchors_built: + # must save the boxes! + model_base_cfg = self.task_config.model + self._num_boxes = (model_base_cfg.max_level - model_base_cfg.min_level + + 1) * model_base_cfg.boxes_per_scale + decoder = tfds_coco_decoder.MSCOCODecoder() + reader = BoxGenInputReader( + params, + dataset_fn=tf.data.TFRecordDataset, + decoder_fn=decoder.decode, + parser_fn=None) + anchors = reader.read( + k=9, image_width=params.parser.image_w, input_context=input_context) + self.task_config.model.set_boxes(anchors) + self._anchors_built = True + del reader + + return self.task_config.model.boxes + + def _get_masks(self, + xy_exponential=False, + exp_base=2, + xy_scale_base='default_value'): + start = 0 + boxes = {} + path_scales = {} + scale_x_y = {} + + if xy_scale_base == 'default_base': + xy_scale_base = 0.05 + xy_scale_base = xy_scale_base / ( + self._boxes_per_level * (self._max_level - self._min_level + 1) - 1) + elif xy_scale_base == 'default_value': + xy_scale_base = 0.00625 + + params = self.task_config.model + + if self._masks == None or self._path_scales == None or self._x_y_scales == None: + for i in range(params.min_level, params.max_level + 1): + boxes[str(i)] = list(range(start, params.boxes_per_scale + start)) + path_scales[str(i)] = 2**i + if xy_exponential: + scale_x_y[str(i)] = 1.0 + xy_scale_base * (exp_base**i) + else: + scale_x_y[str(i)] = 1.0 + start += params.boxes_per_scale + + self._masks = boxes + self._path_scales = path_scales + self._x_y_scales = scale_x_y + + return self._masks, self._path_scales, self._x_y_scales + + def initialize(self, model: tf.keras.Model): + if self.task_config.load_darknet_weights: + from yolo.utils import DarkNetConverter + from yolo.utils._darknet2tf.load_weights import split_converter + from yolo.utils._darknet2tf.load_weights2 import load_weights_backbone + from yolo.utils._darknet2tf.load_weights2 import load_weights_neck + from yolo.utils._darknet2tf.load_weights2 import load_head + from yolo.utils._darknet2tf.load_weights2 import load_weights_prediction_layers + from yolo.utils.downloads.file_manager import download + + weights_file = self.task_config.model.darknet_weights_file + config_file = self.task_config.model.darknet_weights_cfg + + if ('cache' not in weights_file and 'cache' not in config_file): + list_encdec = DarkNetConverter.read(config_file, weights_file) + else: + import os + path = os.path.abspath('cache') + if (not os.path.isdir(path)): + os.mkdir(path) + + cfg = f"{path}/cfg/{config_file.split('/')[-1]}" + if not os.path.isfile(cfg): + download(config_file.split('/')[-1]) + + wgt = f"{path}/weights/{weights_file.split('/')[-1]}" + if not os.path.isfile(wgt): + download(weights_file.split('/')[-1]) + + list_encdec = DarkNetConverter.read(cfg, wgt) + + splits = model.backbone._splits + if 'neck_split' in splits.keys(): + encoder, neck, decoder = split_converter(list_encdec, + splits['backbone_split'], + splits['neck_split']) + else: + encoder, decoder = split_converter(list_encdec, + splits['backbone_split']) + neck = None + + load_weights_backbone(model.backbone, encoder) + model.backbone.trainable = False + + if self.task_config.darknet_load_decoder: + if neck != None: + load_weights_neck(model.decoder.neck, neck) + model.decoder.neck.trainable = False + cfgheads = load_head(model.decoder.head, decoder) + model.decoder.head.trainable = False + load_weights_prediction_layers(cfgheads, model.head) + model.head.trainable = False + else: + """Loading pretrained checkpoint.""" + if not self.task_config.init_checkpoint: + return + + ckpt_dir_or_file = self.task_config.init_checkpoint + if tf.io.gfile.isdir(ckpt_dir_or_file): + ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) + + # Restoring checkpoint. + if self.task_config.init_checkpoint_modules == 'all': + ckpt = tf.train.Checkpoint(**model.checkpoint_items) + status = ckpt.restore(ckpt_dir_or_file) + status.assert_consumed() + elif self.task_config.init_checkpoint_modules == 'backbone': + ckpt = tf.train.Checkpoint(backbone=model.backbone) + status = ckpt.restore(ckpt_dir_or_file) + status.expect_partial().assert_existing_objects_matched() + else: + assert "Only 'all' or 'backbone' can be used to initialize the model." + + logging.info('Finished loading pretrained checkpoint from %s', + ckpt_dir_or_file) + + +if __name__ == '__main__': + import matplotlib.pyplot as plt + from yolo.utils.run_utils import prep_gpu + prep_gpu() + + config = exp_cfg.YoloTask(model=exp_cfg.Yolo(base='v3')) + task = YoloTask(config) + model = task.build_model() + model.summary() + task.initialize(model) + + train_data = task.build_inputs(config.train_data) + # test_data = task.build_inputs(config.task.validation_data) + + for l, (i, j) in enumerate(train_data): + preds = model(i, training=False) + boxes = xcycwh_to_yxyx(j['bbox']) + + i = tf.image.draw_bounding_boxes(i, boxes, [[1.0, 0.0, 0.0]]) + + i = tf.image.draw_bounding_boxes(i, preds['bbox'], [[0.0, 1.0, 0.0]]) + plt.imshow(i[0].numpy()) + plt.show() + + if l > 2: + break diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py new file mode 100644 index 000000000..b9c95d297 --- /dev/null +++ b/centernet/tasks/centernet.py @@ -0,0 +1,130 @@ +import tensorflow as tf +from tensorflow.keras.mixed_precision import experimental as mixed_precision + +from absl import logging +from official.core import base_task +from official.core import input_reader +from official.core import task_factory +from official.vision.beta.evaluation import coco_evaluator + +# TODO: Already added to official codebase, change to official version later +from yolo.dataloaders.decoders import tfds_coco_decoder + +from centernet.configs import centernet as cfg +from centernet.dataloaders import centernet_input +from centernet.modeling.CenterNet import build_centernet +from centernet.ops import loss_ops +from centernet.losses import penalty_reduced_logistic_focal_loss +from centernet.losses import l1_localization_loss + +@task_factory.register_task_cls(cfg.CenterNetTask) +class CenterNetTask(base_task.Task): + + def __init__(self, params, logging_dir: str = None): + super().__init__(params, logging_dir) + + def build_inputs(self, params, input_context=None): + """Build input dataset.""" + decoder = tfds_coco_decoder.MSCOCODecoder() + """ + decoder_cfg = params.decoder.get() + if params.decoder.type == 'simple_decoder': + decoder = tf_example_decoder.TfExampleDecoder( + regenerate_source_id=decoder_cfg.regenerate_source_id) + elif params.decoder.type == 'label_map_decoder': + decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap( + label_map=decoder_cfg.label_map, + regenerate_source_id=decoder_cfg.regenerate_source_id) + else: + raise ValueError('Unknown decoder type: {}!'.format(params.decoder.type)) + """ + + model = self.task_config.model + + masks, path_scales, xy_scales = self._get_masks() + anchors = self._get_boxes(gen_boxes=params.is_training) + + print(masks, path_scales, xy_scales) + parser = centernet_input.CenterNetParser( + num_classes=model.num_classes, + gaussian_iou=model.gaussian_iou + ) + + if params.is_training: + post_process_fn = parser.postprocess_fn() + else: + post_process_fn = None + + reader = input_reader.InputReader( + params, + dataset_fn=tf.data.TFRecordDataset, + decoder_fn=decoder.decode, + parser_fn=parser.parse_fn(params.is_training), + postprocess_fn=post_process_fn) + dataset = reader.read(input_context=input_context) + return dataset + + def build_model(self): + """get an instance of CenterNet""" + params = self.task_config.train_data + model_base_cfg = self.task_config.model + l2_weight_decay = self.task_config.weight_decay / 2.0 + + input_specs = tf.keras.layers.InputSpec(shape=[None] + + model_base_cfg.input_size) + l2_regularizer = ( + tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) + + model, losses = build_centernet(input_specs, self.task_config, l2_regularizer) + self._loss_dict = losses + return model + + def build_losses(self, outputs, labels, aux_losses=None): + total_loss = 0.0 + total_scale_loss = 0.0 + total_offset_loss = 0.0 + loss = 0.0 + scale_loss = 0.0 + offset_loss = 0.0 + + metric_dict = dict() + + # TODO: Calculate loss + flattened_ct_heatmaps = loss_ops._flatten_spatial_dimensions(labels['ct_heatmaps']) + num_boxes = loss_ops._to_float32(loss_ops.get_num_instances_from_weights(labels['tag_masks'])) #gt_weights_list here shouldn't be tag_masks here + + object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) + # Loop through each feature output head. + for pred in outputs['ct_heatmaps']: + pred = loss_ops._flatten_spatial_dimensions(pred) + total_loss += object_center_loss( + flattened_ct_heatmaps, pred) #removed weight parameter (weight = per_pixel_weight) + center_loss = tf.reduce_sum(total_loss) / ( + float(len(outputs['ct_heatmaps'])) * num_boxes) + loss += center_loss + metric_dict['ct_loss'] = center_loss + + #localization loss for offset and scale loss + localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) + for scale_pred, offset_pred in zip(outputs['ct_size'], outputs['ct_offset']): + # Compute the scale loss. + scale_pred = loss_ops.get_batch_predictions_from_indices( + scale_pred, labels['tag_locs']) + total_scale_loss += localization_loss_fn( + labels['ct_size'], scale_pred) #removed weights=batch_weights + # Compute the offset loss. + offset_pred = loss_ops.get_batch_predictions_from_indices( + offset_pred, labels['tag_locs']) + total_offset_loss += localization_loss_fn( + labels['ct_offset'], offset_pred) #removed weights=batch_weights + scale_loss += tf.reduce_sum(total_scale_loss) / ( + float(len(outputs['ct_size'])) * num_boxes) + offset_loss += tf.reduce_sum(total_offset_loss) / ( + float(len(outputs['ct_size'])) * num_boxes) + metric_dict['ct_scale_loss'] = scale_loss + metric_dict['ct_offset_loss'] = offset_loss + + return loss, metric_dict + + def build_metrics(self, training=True): + pass diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py index c2363a8f5..a50e1662a 100644 --- a/centernet/tasks/centernet_object_detection.py +++ b/centernet/tasks/centernet_object_detection.py @@ -39,7 +39,7 @@ def build_losses(self, outputs, labels, aux_losses=None): num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) - # Loop through each feature output head. + outputs['ct_heatmaps'] = utils._flatten_spatial_dimensions(outputs['ct_heatmaps']) total_loss += object_center_loss( flattened_ct_heatmaps, outputs['ct_heatmaps']) #removed weight parameter (weight = per_pixel_weight) diff --git a/centernet/tasks/centernet_object_detection_test.py b/centernet/tasks/centernet_object_detection_test.py index 302fe2b2c..43f445d03 100644 --- a/centernet/tasks/centernet_object_detection_test.py +++ b/centernet/tasks/centernet_object_detection_test.py @@ -1,10 +1,10 @@ import tensorflow as tf import numpy as np import centernet.tasks as tasks -import centernet.ops.loss_utils as utils +import centernet.ops.loss_ops as utils from centernet.losses import penalty_reduced_logistic_focal_loss from centernet.losses import l1_localization_loss -from centernet.tasks.centernet import CenterNetTask +from centernet.tasks.centernet_object_detection import CenterNetTask def gaussian2D(shape, sigma=1): m, n = [(ss - 1.) / 2. for ss in shape] @@ -118,7 +118,7 @@ def generate_heatmaps(batch_size, categories, output_size, detections, gaussian_ class ObjectDetectionTest(tf.test.TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.labels_heatmaps = tf.constant([[ + self.actual = tf.constant([[ (10, 25, 17, 18, 0) ]]) @@ -190,4 +190,49 @@ def generate_heatmaps(self, dectections): #plt.imshow(labels_heatmaps[0, 0, ...]) #plt.show() ''' + actual = tf.constant([[ + (10, 25, 17, 18, 0) + ]], dtype = tf.float32) + + predicted = tf.constant([[ + (10, 30, 15, 17, 0) + ]], dtype = tf.float32) + labels = dict() + outputs = dict() + + tl_heatmaps, br_heatmaps, ct_heatmaps, tl_regrs, br_regrs, ct_regrs, tl_tags, br_tags, ct_tags = generate_heatmaps(1, 2, (416, 416), predicted) + ct_heatmaps = tf.reshape(ct_heatmaps, [1, 416, 416, 2]) + + tl_labels_heatmaps, br_labels_heatmaps, ct_labels_heatmaps, tl_regrs_labels, br_regrs_labels, ct_regrs_labels, tl_tags_labels, br_tags_labels, ct_tags_labels = generate_heatmaps(1, 2, (416, 416), actual) + ct_labels_heatmaps = tf.reshape(ct_labels_heatmaps, [1, 416, 416, 2]) + + tag_masks = [[[True]]] + + labels = { + 'tl_size': tl_tags_labels, + 'br_size': br_tags_labels, + 'ct_size': ct_tags_labels, + 'tl_heatmaps': tl_labels_heatmaps, + 'br_heatmaps': br_labels_heatmaps, + 'ct_heatmaps': ct_labels_heatmaps, + 'tag_masks': tag_masks, + 'tl_offset': tl_regrs_labels, + 'br_offset': br_regrs_labels, + 'ct_offset': ct_regrs_labels, + } + + outputs = { + 'tl_size': tl_tags, + 'br_size': br_tags, + 'ct_size': ct_tags, + 'tl_heatmaps': tl_heatmaps, + 'br_heatmaps': br_heatmaps, + 'ct_heatmaps': ct_heatmaps, + 'tag_masks': tag_masks, + 'tl_offset': tl_regrs, + 'br_offset': br_regrs, + 'ct_offset': ct_regrs, + } + task = CenterNetTask(None) + loss,metric = task.build_losses(outputs, labels) tf.test.main() diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py new file mode 100644 index 000000000..1d03c9393 --- /dev/null +++ b/centernet/tasks/centernet_test.py @@ -0,0 +1,32 @@ +from absl.testing import parameterized +import tensorflow as tf +import numpy as np +import dataclasses + +from official.modeling import hyperparams +from official.vision.beta.configs import backbones + +from centernet.tasks.centernet import CenterNetTask +from centernet.configs import centernet as exp_cfg + + +class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): + + def testCenterNetTask(self): + model_config = exp_cfg.CenterNet(input_size=[512, 512, 3]) + config = exp_cfg.CenterNetTask(model=model_config) + task = CenterNetTask(config) + + model = task.build_model() + outputs = model(tf.zeros((3, 512, 512, 3))) + + self.assertEqual(len(outputs['raw_output']), 3) + self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (3, 128, 128, 90)) + self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (3, 128, 128, 2)) + self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (3, 128, 128, 2)) + + model.summary() + + +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/utils/__init__.py b/centernet/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/centernet/utils/groundtruth.py b/centernet/utils/groundtruth.py new file mode 100644 index 000000000..91f20d10f --- /dev/null +++ b/centernet/utils/groundtruth.py @@ -0,0 +1,99 @@ +import tensorflow as tf + +def _smallest_positive_root(a, b, c) -> tf.Tensor: + """ + Returns the smallest positive root of a quadratic equation. + This implements the fixed version in https://github.com/princeton-vl/CornerNet. + """ + + discriminant = tf.sqrt(b ** 2 - 4 * a * c) + + root1 = (-b - discriminant) / (2 * a) + root2 = (-b + discriminant) / (2 * a) + + return (-b + discriminant) / (2) # tf.where(tf.less(root1, 0), root2, root1) + +def gaussian_radius(det_size, min_overlap=0.7) -> int: + """ + Given a bounding box size, returns a lower bound on how far apart the + corners of another bounding box can lie while still maintaining the given + minimum overlap, or IoU. Modified from implementation found in + https://github.com/tensorflow/models/blob/master/research/object_detection/core/target_assigner.py. + + Params: + det_size (tuple): tuple of integers representing height and width + min_overlap (tf.float32): minimum IoU desired + Returns: + int representing desired gaussian radius + """ + height, width = det_size + + # Case where detected box is offset from ground truth and no box completely + # contains the other. + + a1 = 1 + b1 = -(height + width) + c1 = width * height * (1 - min_overlap) / (1 + min_overlap) + r1 = _smallest_positive_root(a1, b1, c1) + + # Case where detection is smaller than ground truth and completely contained + # in it. + + a2 = 4 + b2 = -2 * (height + width) + c2 = (1 - min_overlap) * width * height + r2 = _smallest_positive_root(a2, b2, c2) + + # Case where ground truth is smaller than detection and completely contained + # in it. + + a3 = 4 * min_overlap + b3 = 2 * min_overlap * (height + width) + c3 = (min_overlap - 1) * width * height + r3 = _smallest_positive_root(a3, b3, c3) + # TODO discuss whether to return scalar or tensor + # return tf.reduce_min([r1, r2, r3], axis=0) + + return tf.reduce_min([r1, r2, r3], axis=0) + +def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: + """ + This represents the penalty reduction around a point. + Params: + radius (int): integer for radius of penalty reduction + type (tf.dtypes.DType): datatype of returned tensor + Returns: + tf.Tensor of shape (2 * radius + 1, 2 * radius + 1). + """ + width = 2 * radius + 1 + sigma = radius / 3 + x = tf.reshape(tf.range(width, dtype=type) - radius, (width, 1)) + y = tf.reshape(tf.range(width, dtype=type) - radius, (1, width)) + exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) + return tf.math.exp(exponent) + +def draw_gaussian(heatmap, center, radius, k=1): + """ + Draws a gaussian heatmap around a center point given a radius. + Params: + heatmap (tf.Tensor): heatmap placeholder to fill + center (int): integer for center of gaussian + radius (int): integer for radius of gaussian + k (int): scaling factor for gaussian + """ + + diameter = 2 * radius + 1 + gaussian = _gaussian_penalty(radius) + + x, y = center + + height, width = heatmap.shape[0:2] + + left, right = min(x, radius), min(width - x, radius + 1) + top, bottom = min(y, radius), min(height - y, radius + 1) + + masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] + # TODO: make sure this replicates original functionality + # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) + masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * k) diff --git a/centernet/utils/groundtruth_test.py b/centernet/utils/groundtruth_test.py new file mode 100644 index 000000000..9d8af4660 --- /dev/null +++ b/centernet/utils/groundtruth_test.py @@ -0,0 +1,3 @@ +from centernet.ops.preprocessing_ops_test import * +if __name__ == '__main__': + tf.test.main() diff --git a/centernet/utils/weight_utils/MODEL_VARS.txt b/centernet/utils/weight_utils/MODEL_VARS.txt new file mode 100644 index 000000000..119fb548c --- /dev/null +++ b/centernet/utils/weight_utils/MODEL_VARS.txt @@ -0,0 +1,1386 @@ +model + _feature_extractor + _network + downsample_input + residual_block + conv + kernel + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + moving_mean + gamma + beta + moving_variance + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + hourglass_network + 0 + encoder_block2 + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + 0 + skip + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv_block + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + inner_block + 0 + inner_block + 0 + inner_block + 0 + encoder_block2 + 0 + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + skip + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + 1 + conv_block + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + inner_block + 0 + encoder_block2 + 0 + skip + norm + beta + gamma + moving_variance + moving_mean + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + norm + moving_mean + gamma + beta + moving_variance + inner_block + 3 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 2 + conv_block + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + 1 + norm + moving_mean + beta + gamma + moving_variance + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + conv + kernel + decoder_block + 1 + norm + gamma + beta + moving_mean + moving_variance + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + encoder_block1 + 0 + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + beta + gamma + moving_mean + moving_variance + 1 + conv_block + norm + gamma + moving_mean + beta + moving_variance + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + decoder_block + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block1 + 0 + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + norm + moving_mean + beta + gamma + moving_variance + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + encoder_block2 + 1 + norm + gamma + beta + moving_mean + moving_variance + conv_block + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + 0 + norm + moving_variance + beta + gamma + moving_mean + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block1 + 0 + norm + moving_variance + beta + gamma + moving_mean + conv_block + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + conv + kernel + 1 + norm + gamma + beta + moving_mean + moving_variance + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + conv + kernel + decoder_block + 0 + conv_block + norm + gamma + moving_variance + moving_mean + beta + conv + kernel + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + 1 + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block1 + 0 + norm + gamma + beta + moving_mean + moving_variance + conv_block + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + 1 + norm + moving_variance + beta + gamma + moving_mean + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + decoder_block + 0 + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block2 + 0 + skip + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + norm + beta + gamma + moving_mean + moving_variance + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + encoder_block1 + 0 + conv_block + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + 1 + conv_block + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + decoder_block + 0 + norm + moving_mean + moving_variance + beta + gamma + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + 1 + inner_block + 0 + inner_block + 0 + inner_block + 0 + encoder_block1 + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + 0 + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + inner_block + 0 + inner_block + 2 + norm + gamma + moving_mean + moving_variance + beta + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + conv + kernel + 0 + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + norm + beta + gamma + moving_variance + moving_mean + conv_block + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + conv + kernel + 3 + norm + beta + moving_variance + gamma + moving_mean + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv + kernel + encoder_block1 + 0 + conv_block + norm + moving_mean + moving_variance + beta + gamma + conv + kernel + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + encoder_block2 + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + 0 + norm + beta + moving_mean + gamma + moving_variance + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + skip + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + decoder_block + 1 + norm + moving_mean + moving_variance + beta + gamma + conv_block + norm + beta + moving_mean + moving_variance + gamma + conv + kernel + conv + kernel + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + norm + beta + gamma + moving_variance + moving_mean + conv_block + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + decoder_block + 1 + norm + moving_variance + beta + gamma + moving_mean + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + 0 + conv_block + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + encoder_block2 + 0 + skip + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + 1 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + decoder_block + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + encoder_block2 + 0 + conv_block + norm + moving_mean + moving_variance + beta + gamma + conv + kernel + norm + moving_variance + gamma + beta + moving_mean + skip + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + 1 + conv + kernel + conv_block + conv + kernel + norm + beta + moving_mean + gamma + moving_variance + norm + gamma + moving_mean + beta + moving_variance + encoder_block1 + 0 + conv_block + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + 1 + conv + kernel + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block1 + 0 + norm + beta + moving_mean + gamma + moving_variance + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + norm + moving_mean + moving_variance + gamma + beta + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + encoder_block2 + 0 + conv + kernel + conv_block + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + norm + beta + gamma + moving_mean + moving_variance + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + decoder_block + 0 + norm + gamma + beta + moving_variance + moving_mean + conv_block + conv + kernel + norm + gamma + moving_mean + beta + moving_variance + conv + kernel + 1 + skip + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + decoder_block + 1 + norm + moving_mean + moving_variance + beta + gamma + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv + kernel + 0 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + encoder_block2 + 1 + conv_block + norm + beta + moving_variance + moving_mean + gamma + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + 0 + conv_block + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + norm + moving_variance + moving_mean + beta + gamma + skip + norm + moving_mean + moving_variance + beta + gamma + conv + kernel + conv + kernel + encoder_block1 + 1 + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + 0 + conv + kernel + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + output_conv + 0 + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + 1 + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + intermediate_residual + 0 + conv_block + norm + gamma + moving_variance + moving_mean + beta + conv + kernel + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + intermediate_conv1 + 0 + norm + moving_mean + moving_variance + gamma + beta + conv + kernel + intermediate_conv2 + 0 + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + _prediction_head_dict + box.Soffset + 0 + layer_with_weights-0 + bias + kernel + layer_with_weights-1 + bias + kernel + 1 + layer_with_weights-0 + bias + kernel + layer_with_weights-1 + kernel + bias + box.Sscale + 1 + layer_with_weights-0 + kernel + bias + layer_with_weights-1 + bias + kernel + 0 + layer_with_weights-0 + kernel + bias + layer_with_weights-1 + bias + kernel + object_center + 1 + layer_with_weights-1 + bias + kernel + layer_with_weights-0 + bias + kernel + 0 + layer_with_weights-0 + kernel + bias + layer_with_weights-1 + bias + kernel +save_counter diff --git a/centernet/utils/weight_utils/__init__.py b/centernet/utils/weight_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/centernet/utils/weight_utils/config_classes.py b/centernet/utils/weight_utils/config_classes.py new file mode 100644 index 000000000..06a878d92 --- /dev/null +++ b/centernet/utils/weight_utils/config_classes.py @@ -0,0 +1,204 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +import numpy as np + +from typing import Tuple, List, Dict, ClassVar + +class Config(ABC): + def get_weights(self): + return None + + def load_weights(self, layer): + weights = self.get_weights() + layer.set_weights(weights) + n_weights = 0 + + for w in weights: + n_weights += w.size + return n_weights + + +@dataclass +class convBnCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + weights: np.array = field(repr=False, default=None) + beta: np.array = field(repr=False, default=None) + gamma: np.array = field(repr=False, default=None) + moving_mean: np.array = field(repr=False, default=None) + moving_variance: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_weights_dict = self.weights_dict['conv'] + norm_weights_dict = self.weights_dict['norm'] + + self.weights = conv_weights_dict['kernel'] + self.beta = norm_weights_dict['beta'] + self.gamma = norm_weights_dict['gamma'] + self.moving_mean = norm_weights_dict['moving_mean'] + self.moving_variance = norm_weights_dict['moving_variance'] + + def get_weights(self): + return [ + self.weights, + self.gamma, + self.beta, + self.moving_mean, + self.moving_variance + ] + +@dataclass +class residualBlockCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + + skip_weights: np.array = field(repr=False, default=None) + skip_beta: np.array = field(repr=False, default=None) + skip_gamma: np.array = field(repr=False, default=None) + skip_moving_mean: np.array = field(repr=False, default=None) + skip_moving_variance: np.array = field(repr=False, default=None) + + conv_weights: np.array = field(repr=False, default=None) + norm_beta: np.array = field(repr=False, default=None) + norm_gamma: np.array = field(repr=False, default=None) + norm_moving_mean: np.array = field(repr=False, default=None) + norm_moving_variance: np.array = field(repr=False, default=None) + + conv_block_weights: np.array = field(repr=False, default=None) + conv_block_beta: np.array = field(repr=False, default=None) + conv_block_gamma: np.array = field(repr=False, default=None) + conv_block_moving_mean: np.array = field(repr=False, default=None) + conv_block_moving_variance: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_weights_dict = self.weights_dict['conv'] + norm_weights_dict = self.weights_dict['norm'] + conv_block_weights_dict = self.weights_dict['conv_block'] + + if 'skip' in self.weights_dict: + skip_weights_dict = self.weights_dict['skip'] + self.skip_weights = skip_weights_dict['conv']['kernel'] + self.skip_beta = skip_weights_dict['norm']['beta'] + self.skip_gamma = skip_weights_dict['norm']['gamma'] + self.skip_moving_mean = skip_weights_dict['norm']['moving_mean'] + self.skip_moving_variance = skip_weights_dict['norm']['moving_variance'] + + self.conv_weights = conv_weights_dict['kernel'] + self.norm_beta = norm_weights_dict['beta'] + self.norm_gamma = norm_weights_dict['gamma'] + self.norm_moving_mean = norm_weights_dict['moving_mean'] + self.norm_moving_variance = norm_weights_dict['moving_variance'] + + self.conv_block_weights = conv_block_weights_dict['conv']['kernel'] + self.conv_block_beta = conv_block_weights_dict['norm']['beta'] + self.conv_block_gamma = conv_block_weights_dict['norm']['gamma'] + self.conv_block_moving_mean = conv_block_weights_dict['norm']['moving_mean'] + self.conv_block_moving_variance = conv_block_weights_dict['norm']['moving_variance'] + + def get_weights(self): + weights = [ + self.skip_weights, + self.skip_gamma, + self.skip_beta, + + self.conv_block_weights, + self.conv_block_gamma, + self.conv_block_beta, + + self.conv_weights, + self.norm_gamma, + self.norm_beta, + + self.skip_moving_mean, + self.skip_moving_variance, + self.conv_block_moving_mean, + self.conv_block_moving_variance, + self.norm_moving_mean, + self.norm_moving_variance, + ] + + weights = [x for x in weights if x is not None] + return weights + +@dataclass +class hourglassCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + is_last_stage: bool = field(repr=False, default=None) + + def __post_init__(self): + self.is_last_stage = False if 'inner_block' in self.weights_dict else True + + def generate_block_weights(self, weights_dict): + reps = len(weights_dict.keys()) + weights = [] + n_weights = 0 + + for i in range(reps): + res_config = residualBlockCFG(weights_dict=weights_dict[str(i)]) + res_weights = res_config.get_weights() + weights += res_weights + + for w in res_weights: + n_weights += w.size + + return weights, n_weights + + def load_block_weights(self, layer, weight_dict): + block_weights, n_weights = self.generate_block_weights(weight_dict) + layer.set_weights(block_weights) + return n_weights + + def load_weights(self, layer): + n_weights = 0 + + if not self.is_last_stage: + enc_dec_layers = [layer.submodules[0], + layer.submodules[1], + layer.submodules[3]] + enc_dec_weight_dicts = [self.weights_dict['encoder_block1'], + self.weights_dict['encoder_block2'], + self.weights_dict['decoder_block']] + + + for l, weights_dict in zip(enc_dec_layers, enc_dec_weight_dicts): + n_weights += self.load_block_weights(l, weights_dict) + + if len(self.weights_dict['inner_block']) == 1: # still in an outer hourglass + inner_weights_dict = self.weights_dict['inner_block']['0'] + else: + inner_weights_dict = self.weights_dict['inner_block'] # inner residual block chain + + inner_hg_layer = layer.submodules[2] + inner_hg_cfg = type(self)(weights_dict=inner_weights_dict) + n_weights += inner_hg_cfg.load_weights(inner_hg_layer) + + else: + inner_layer = layer.submodules[0] + n_weights += self.load_block_weights(inner_layer, self.weights_dict) + + return n_weights + +@dataclass +class decoderConvCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + + conv_1_weights: np.array = field(repr=False, default=None) + conv_2_bias: np.array = field(repr=False, default=None) + + conv_2_weights: np.array = field(repr=False, default=None) + conv_1_bias: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_1_weights_dict = self.weights_dict['layer_with_weights-0'] + conv_2_weights_dict = self.weights_dict['layer_with_weights-1'] + + self.conv_1_weights = conv_1_weights_dict['kernel'] + self.conv_1_bias = conv_1_weights_dict['bias'] + self.conv_2_weights = conv_2_weights_dict['kernel'] + self.conv_2_bias = conv_2_weights_dict['bias'] + + def get_weights(self): + return [ + self.conv_1_weights, + self.conv_1_bias, + self.conv_2_weights, + self.conv_2_bias + ] \ No newline at end of file diff --git a/centernet/utils/weight_utils/config_data.py b/centernet/utils/weight_utils/config_data.py new file mode 100644 index 000000000..764091b15 --- /dev/null +++ b/centernet/utils/weight_utils/config_data.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass, field +from typing import Dict + +from centernet.utils.weight_utils.config_classes import convBnCFG, residualBlockCFG, hourglassCFG, decoderConvCFG + +@dataclass +class BackboneConfigData(): + weights_dict: Dict = field(repr=False, default=None) + + def __post_init__(self): + self.hourglass104_512 = [ + # Downsampling Layers + convBnCFG(weights_dict=self.weights_dict['downsample_input']['conv_block']), + residualBlockCFG(weights_dict=self.weights_dict['downsample_input']['residual_block']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['0']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['0']), + # Intermediate + convBnCFG(weights_dict=self.weights_dict['intermediate_conv1']['0']), + convBnCFG(weights_dict=self.weights_dict['intermediate_conv2']['0']), + residualBlockCFG(weights_dict=self.weights_dict['intermediate_residual']['0']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['1']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), + ] + + def get_cfg_list(self, name): + if name == 'hourglass104_512': + return self.hourglass104_512 + +@dataclass +class DecoderConfigData(): + weights_dict: Dict = field(repr=False, default=None) + + def __post_init__(self): + self.detection_2d = [ + decoderConvCFG(weights_dict=self.weights_dict['object_center']['0']), + decoderConvCFG(weights_dict=self.weights_dict['object_center']['1']), + decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['0']), + decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['1']), + decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['0']), + decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['1']) + ] + + def get_cfg_list(self, name): + if name == 'detection_2d': + return self.detection_2d + diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py new file mode 100644 index 000000000..4918c3274 --- /dev/null +++ b/centernet/utils/weight_utils/load_weights.py @@ -0,0 +1,140 @@ +import tensorflow as tf +import numpy as np + +from centernet.modeling.CenterNet import build_centernet +from centernet.configs.centernet import CenterNetTask + +from centernet.utils.weight_utils.tf_to_dict import get_model_weights_as_dict, write_dict_as_tree +from centernet.utils.weight_utils.config_classes import convBnCFG, residualBlockCFG, hourglassCFG, decoderConvCFG +from centernet.utils.weight_utils.config_data import BackboneConfigData, DecoderConfigData + +from centernet.modeling.layers.nn_blocks import ConvBN, HourglassBlock, CenterNetDecoderConv + +from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock + +CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' +SAVED_MODEL_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\saved_model' + +def load_weights_model(model, weights_dict, backbone_name, decoder_name): + """ Loads weights into the model. + + Args: + model: keras.Model to load weights into + weights_dict: Dictionary that stores the weights of the model + backbone_name: String, indicating the desired backbone configuration + decoder_name: String, indicating the desired decoder configuration + """ + print("Loading model weights\n") + load_weights_backbone(model.backbone, + weights_dict['model']['_feature_extractor']['_network'], + backbone_name) + + load_weights_decoder(model.decoder, + weights_dict['model']['_prediction_head_dict'], + decoder_name) + print("Successfully loaded model weights\n") + +def get_backbone_layer_cfgs(weights_dict, backbone_name): + """ Fetches the config classes for the backbone. + + This function generates a list of config classes corresponding to + each building block in the backbone. + + Args: + weights_dict: Dictionary that stores the backbone model weights + backbone_name: String, indicating the desired backbone configuration + Returns: + A list containing the config classe of the backbone building block + """ + + print("Fetching backbone config classes for {}\n".format(backbone_name)) + cfgs = BackboneConfigData(weights_dict=weights_dict).get_cfg_list(backbone_name) + return cfgs + +def load_weights_backbone(backbone, weights_dict, backbone_name): + """ Loads the weights defined in the weights_dict into the backbone. + + This function loads the backbone weights by first fetching the necesary + config classes for the backbone, then loads them in one by one for + each layer that has weights associated with it. + + Args: + backbone: keras.Model backbone + weights_dict: Dictionary that stores the backbone model weights + backbone_name: String, indicating the desired backbone configuration + """ + print("Loading backbone weights\n") + backbone_layers = backbone.layers + cfgs = get_backbone_layer_cfgs(weights_dict, backbone_name) + n_weights_total = 0 + + cfg = cfgs.pop(0) + for i in range(len(backbone_layers)): + layer = backbone_layers[i] + if isinstance(layer, (ConvBN, HourglassBlock, ResidualBlock)): + n_weights = cfg.load_weights(layer) + print("Loading weights for: {}, weights loaded: {}".format(cfg, n_weights)) + n_weights_total += n_weights + if len(cfgs) == 0: + print("{} Weights have been loaded for {} / {} layers\n".format(n_weights_total, i+1, len(backbone_layers))) + return + cfg = cfgs.pop(0) + +def get_decoder_layer_cfgs(weights_dict, decoder_name): + """ Fetches the config classes for the decoder. + + This function generates a list of config classes corresponding to + each building block in the decoder. + + Args: + weights_dict: Dictionary that stores the decoder model weights + decoder_name: String, indicating the desired decoder configuration + Returns: + A list containing the config classe of the backbone building block + """ + print("Fetching decoder config classes\n") + + cfgs = DecoderConfigData(weights_dict=weights_dict).get_cfg_list(decoder_name) + return cfgs + +def load_weights_decoder(decoder, weights_dict, decoder_name): + """ Loads the weights defined in the weights_dict into the decoder. + + This function loads the decoder weights by first fetching the necesary + config classes for the decoder, then loads them in one by one for + each layer that has weights associated with it. + + Args: + decoder: keras.Model decoder + weights_dict: Dictionary that stores the decoder model weights + decoder_name: String, indicating the desired decoder configuration + """ + print("Loading decoder weights\n") + decoder_layers = decoder.layers + cfgs = get_decoder_layer_cfgs(weights_dict, decoder_name) + n_weights_total = 0 + + cfg = cfgs.pop(0) + for i in range(len(decoder_layers)): + layer = decoder_layers[i] + if isinstance(layer, CenterNetDecoderConv): + n_weights = cfg.load_weights(layer) + print("Loading weights for: {}, weights loaded: {}".format(cfg, n_weights)) + n_weights_total += n_weights + if len(cfgs) == 0: + print("{} Weights have been loaded for {} / {} layers\n".format(n_weights_total, i+1, len(decoder_layers))) + return + cfg = cfgs.pop(0) + +if __name__ == '__main__': + input_specs = tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]) + config = CenterNetTask() + + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + + weights_dict = get_model_weights_as_dict(CKPT_PATH) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + + # Uncomment line below to write weights dict key names to a file + # write_dict_as_tree(weights_dict, filename="centernet/utils/weight_utils/MODEL_VARS.txt") \ No newline at end of file diff --git a/centernet/utils/weight_utils/tf_to_dict.py b/centernet/utils/weight_utils/tf_to_dict.py new file mode 100644 index 000000000..2739e7383 --- /dev/null +++ b/centernet/utils/weight_utils/tf_to_dict.py @@ -0,0 +1,87 @@ +import tensorflow as tf + +def update_weights_dict(weights_dict, variable_key, value): + """ Inserts weight value into a weight dictionary. + + This function inserts a weight value into a weights_dict based on the + variable key. It is designed to organize TF checkpoint weights by organizing + them by submodules. + + Args: + weights_dict: Dictionary to store weights + variable_key: String, name of the variable assocaited with the value + value: An ndarray that stores the weights assocaited to the variable key + """ + current_dict = weights_dict + variable_key_list = variable_key.split('/') + + key = variable_key_list.pop(0) + while len(variable_key_list): + if variable_key_list[0] == '.ATTRIBUTES': + current_dict[key] = value + return + + if key not in current_dict.keys(): + current_dict[key] = {} + current_dict = current_dict[key] + key = variable_key_list.pop(0) + +def get_model_weights_as_dict(ckpt_path): + """ Converts a TF checkpoint into a nested dictionary of weights. + + Args: + ckpt_path: String, indicating filepath of the TF checkpoint + Returns: + Dictionary where the checkpoint weights are stored + """ + print("\nConverting model checkpoint from {} to weights dictionary\n".format(ckpt_path)) + reader = tf.train.load_checkpoint(ckpt_path) + shape_from_key = reader.get_variable_to_shape_map() + dtype_from_key = reader.get_variable_to_dtype_map() + + variable_keys = shape_from_key.keys() + weights_dict = {} + + for key in variable_keys: + shape = shape_from_key[key] + dtype = dtype_from_key[key] + value = reader.get_tensor(key) + update_weights_dict(weights_dict, key, value) + + print("Successfully read checkpoint weights\n") + return weights_dict + +def write_dict_as_tree(dictionary, filename, spaces=0): + """ Writes nested dictionary keys to a file. + + Given a dictionary that contains nested dictionaries, this function + writes the name of the keys recursively to a specified file as a tree + + Args: + dictionary: Desired dictionary to write to a file + filename: String, name of file to write dictionary to + spaces: Optional; Number of spaces to insert before writing + the dictionary key names + """ + if type(dictionary) is dict: + mode = "w" if spaces == 0 else "a" + for key in dictionary.keys(): + with open(filename, mode) as fp: + fp.write(" " * spaces + key + '\n') + mode = "a" + write_dict_as_tree(dictionary[key], filename, spaces + 2) + +def print_layer_weights_and_shape(layer): + """ Prints variables information corresponding to a Keras layer. + + This function prints the name and the shape of its associated weights + of all variables (trainable and untrainable) in a Keras layer. + + Args: + layer: A Keras.layer.Layer object + """ + weights = layer.get_weights() + variables = layer.variables + + for i in range(len(weights)): + tf.print(np.shape(weights[i]), variables[i].name) \ No newline at end of file From ec928408e76c3ebce33bfbf065d940664a6159b4 Mon Sep 17 00:00:00 2001 From: patel996 Date: Wed, 10 Mar 2021 13:07:01 -0500 Subject: [PATCH 062/132] fix the filename --- centernet/tasks/centernet_object_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centernet/tasks/centernet_object_detection.py b/centernet/tasks/centernet_object_detection.py index a50e1662a..70dcceaa9 100644 --- a/centernet/tasks/centernet_object_detection.py +++ b/centernet/tasks/centernet_object_detection.py @@ -8,7 +8,7 @@ from official.vision.beta.evaluation import coco_evaluator from centernet.configs import centernet as cfg -import centernet.ops.loss_utils as utils +import centernet.ops.loss_ops as utils from centernet.losses import penalty_reduced_logistic_focal_loss from centernet.losses import l1_localization_loss From 8653eb98c8074c7ecfda4a2a4a45c86a4b920934 Mon Sep 17 00:00:00 2001 From: patel996 Date: Wed, 10 Mar 2021 13:10:43 -0500 Subject: [PATCH 063/132] remove test from main --- .../tasks/centernet_object_detection_test.py | 49 +------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/centernet/tasks/centernet_object_detection_test.py b/centernet/tasks/centernet_object_detection_test.py index 43f445d03..0c5dc80ce 100644 --- a/centernet/tasks/centernet_object_detection_test.py +++ b/centernet/tasks/centernet_object_detection_test.py @@ -120,11 +120,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.actual = tf.constant([[ (10, 25, 17, 18, 0) - ]]) + ]], dtype = tf.float32) self.predicted = tf.constant([[ (10, 30, 15, 17, 0) - ]]) + ]], dtype = tf.float32) def generate_heatmaps(self, dectections): labels = dict() @@ -190,49 +190,4 @@ def generate_heatmaps(self, dectections): #plt.imshow(labels_heatmaps[0, 0, ...]) #plt.show() ''' - actual = tf.constant([[ - (10, 25, 17, 18, 0) - ]], dtype = tf.float32) - - predicted = tf.constant([[ - (10, 30, 15, 17, 0) - ]], dtype = tf.float32) - labels = dict() - outputs = dict() - - tl_heatmaps, br_heatmaps, ct_heatmaps, tl_regrs, br_regrs, ct_regrs, tl_tags, br_tags, ct_tags = generate_heatmaps(1, 2, (416, 416), predicted) - ct_heatmaps = tf.reshape(ct_heatmaps, [1, 416, 416, 2]) - - tl_labels_heatmaps, br_labels_heatmaps, ct_labels_heatmaps, tl_regrs_labels, br_regrs_labels, ct_regrs_labels, tl_tags_labels, br_tags_labels, ct_tags_labels = generate_heatmaps(1, 2, (416, 416), actual) - ct_labels_heatmaps = tf.reshape(ct_labels_heatmaps, [1, 416, 416, 2]) - - tag_masks = [[[True]]] - - labels = { - 'tl_size': tl_tags_labels, - 'br_size': br_tags_labels, - 'ct_size': ct_tags_labels, - 'tl_heatmaps': tl_labels_heatmaps, - 'br_heatmaps': br_labels_heatmaps, - 'ct_heatmaps': ct_labels_heatmaps, - 'tag_masks': tag_masks, - 'tl_offset': tl_regrs_labels, - 'br_offset': br_regrs_labels, - 'ct_offset': ct_regrs_labels, - } - - outputs = { - 'tl_size': tl_tags, - 'br_size': br_tags, - 'ct_size': ct_tags, - 'tl_heatmaps': tl_heatmaps, - 'br_heatmaps': br_heatmaps, - 'ct_heatmaps': ct_heatmaps, - 'tag_masks': tag_masks, - 'tl_offset': tl_regrs, - 'br_offset': br_regrs, - 'ct_offset': ct_regrs, - } - task = CenterNetTask(None) - loss,metric = task.build_losses(outputs, labels) tf.test.main() From e44befda5be60b4e260bb75484cc38c2fd74ad1e Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 10 Mar 2021 17:25:30 -0500 Subject: [PATCH 064/132] detection generator --- centernet/modeling/CenterNet.py | 15 +- .../modeling/layers/detection_generator.py | 206 ++++++++++++++++++ centernet/utils/weight_utils/load_weights.py | 23 +- .../utils/weight_utils/test_load_weights.py | 37 ++++ centernet/utils/weight_utils/tf_to_dict.py | 8 +- .../dataset_specs/coco-labels-paper.names | 91 ++++++++ 6 files changed, 361 insertions(+), 19 deletions(-) create mode 100644 centernet/modeling/layers/detection_generator.py create mode 100644 centernet/utils/weight_utils/test_load_weights.py create mode 100644 yolo/dataloaders/dataset_specs/coco-labels-paper.names diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 81dad60a5..bcbc6877c 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -6,6 +6,7 @@ from centernet.modeling.backbones.hourglass import Hourglass from centernet.modeling.backbones.hourglass import build_hourglass from centernet.modeling.decoders.centernet_decoder import CenterNetDecoder +from centernet.modeling.layers.detection_generator import CenterNetLayer # TODO: import prediction and filtering layers when made @@ -33,19 +34,14 @@ def build(self, input_shape): def call(self, inputs, training=False): features = self._backbone(inputs) - # final_backbone_output = features[-1] decoded_maps = self._decoder(features) - # TODO: head + filters - if training: return {"raw_output": decoded_maps} else: - # TODO: uncomment when filter is implemented - # predictions = self._filter(raw_predictions) - # predictions.update({"raw_output": raw_predictions}) - # return predictions - return {"raw_output": decoded_maps} + predictions = self._filter(decoded_maps) + predictions.update({"raw_output": decoded_maps}) + return predictions @property def backbone(self): @@ -79,7 +75,7 @@ def build_centernet_decoder(input_specs, task_config, num_inputs): return model def build_centernet_filter(model_config): - return None + return CenterNetLayer() def build_centernet_head(model_config): return None @@ -97,7 +93,6 @@ def build_centernet(input_specs, task_config, l2_regularization): filter = build_centernet_filter(model_config) model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) - model.build(input_specs.shape) # TODO: uncommend when filter is implemented diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py new file mode 100644 index 000000000..ed92362d5 --- /dev/null +++ b/centernet/modeling/layers/detection_generator.py @@ -0,0 +1,206 @@ +from tensorflow import keras as ks +import tensorflow as tf + +class CenterNetLayer(ks.Model): + + def __init__(self, + max_detections=100, + detect_per_channel=False, + center_thresh=0.1, + **kwargs): + + super().__init__(**kwargs) + self._max_detections = max_detections + self._detect_per_channel = detect_per_channel + self._center_thresh = center_thresh + + def process_heatmap(self, + feature_map, + kernel_size=3, + peak_error=1e-6): + feature_map = tf.math.sigmoid(feature_map) + if not kernel_size or kernel_size == 1: + feature_map_peaks = feature_map + else: + feature_map_max_pool = tf.nn.max_pool( + feature_map, ksize=kernel_size, strides=1, padding='SAME') + + feature_map_peak_mask = tf.math.abs( + feature_map - feature_map_max_pool) < peak_error + + # Zero out everything that is not a peak. + feature_map_peaks = ( + feature_map * tf.cast(feature_map_peak_mask, tf.float32)) + + return feature_map_peaks + + def get_row_col_channel_indices_from_flattened_indices(self, + indices, + num_cols, + num_channels): + """Computes row, column and channel indices from flattened indices. + Args: + indices: An integer tensor of any shape holding the indices in the flattened + space. + num_cols: Number of columns in the image (width). + num_channels: Number of channels in the image. + Returns: + row_indices: The row indices corresponding to each of the input indices. + Same shape as indices. + col_indices: The column indices corresponding to each of the input indices. + Same shape as indices. + channel_indices. The channel indices corresponding to each of the input + indices. + """ + # Avoid using mod operator to make the ops more easy to be compatible with + # different environments, e.g. WASM. + row_indices = (indices // num_channels) // num_cols + col_indices = (indices // num_channels) - row_indices * num_cols + channel_indices_temp = indices // num_channels + channel_indices = indices - channel_indices_temp * num_channels + + return row_indices, col_indices, channel_indices + + def multi_range(self, + limit, + value_repetitions=1, + range_repetitions=1, + dtype=tf.int32): + """Creates a sequence with optional value duplication and range repetition. + As an example (see the Args section for more details), + _multi_range(limit=2, value_repetitions=3, range_repetitions=4) returns: + [0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1] + Args: + limit: A 0-D Tensor (scalar). Upper limit of sequence, exclusive. + value_repetitions: Integer. The number of times a value in the sequence is + repeated. With value_repetitions=3, the result is [0, 0, 0, 1, 1, 1, ..]. + range_repetitions: Integer. The number of times the range is repeated. With + range_repetitions=3, the result is [0, 1, 2, .., 0, 1, 2, ..]. + dtype: The type of the elements of the resulting tensor. + Returns: + A 1-D tensor of type `dtype` and size + [`limit` * `value_repetitions` * `range_repetitions`] that contains the + specified range with given repetitions. + """ + return tf.reshape( + tf.tile( + tf.expand_dims(tf.range(limit, dtype=dtype), axis=-1), + multiples=[range_repetitions, value_repetitions]), [-1]) + + def get_top_k_peaks(self, + feature_map_peaks, + k=100, + per_channel=False): + (batch_size, _, width, num_channels) = feature_map_peaks.get_shape() + + if per_channel: + if k == 1: + feature_map_flattened = tf.reshape( + feature_map_peaks, [batch_size, -1, num_channels]) + scores = tf.math.reduce_max(feature_map_flattened, axis=1) + peak_flat_indices = tf.math.argmax( + feature_map_flattened, axis=1, output_type=tf.dtypes.int32) + peak_flat_indices = tf.expand_dims(peak_flat_indices, axis=-1) + else: + # Perform top k over batch and channels. + feature_map_peaks_transposed = tf.transpose(feature_map_peaks, + perm=[0, 3, 1, 2]) + feature_map_peaks_transposed = tf.reshape( + feature_map_peaks_transposed, [batch_size, num_channels, -1]) + scores, peak_flat_indices = tf.math.top_k( + feature_map_peaks_transposed, k=k) + # Convert the indices such that they represent the location in the full + # (flattened) feature map of size [batch, height * width * channels]. + channel_idx = tf.range(num_channels)[tf.newaxis, :, tf.newaxis] + peak_flat_indices = num_channels * peak_flat_indices + channel_idx + scores = tf.reshape(scores, [batch_size, -1]) + peak_flat_indices = tf.reshape(peak_flat_indices, [batch_size, -1]) + else: + if k == 1: + feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) + scores = tf.math.reduce_max(feature_map_peaks_flat, axis=1, keepdims=True) + peak_flat_indices = tf.expand_dims(tf.math.argmax( + feature_map_peaks_flat, axis=1, output_type=tf.dtypes.int32), axis=-1) + else: + feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) + scores, peak_flat_indices = tf.math.top_k(feature_map_peaks_flat, k=k) + + # Get x, y and channel indices corresponding to the top indices in the flat + # array. + y_indices, x_indices, channel_indices = ( + self.get_row_col_channel_indices_from_flattened_indices( + peak_flat_indices, width, num_channels)) + return scores, y_indices, x_indices, channel_indices + + def get_boxes(self, + detection_scores, + y_indices, + x_indices, + channel_indices, + height_width_predictions, + offset_predictions): + batch_size, num_boxes = y_indices.get_shape() + + # TF Lite does not support tf.gather with batch_dims > 0, so we need to use + # tf_gather_nd instead and here we prepare the indices for that. + combined_indices = tf.stack([ + self.multi_range(batch_size, value_repetitions=num_boxes), + tf.reshape(y_indices, [-1]), + tf.reshape(x_indices, [-1]) + ], axis=1) + new_height_width = tf.gather_nd(height_width_predictions, combined_indices) + new_height_width = tf.reshape(new_height_width, [batch_size, num_boxes, -1]) + + new_offsets = tf.gather_nd(offset_predictions, combined_indices) + offsets = tf.reshape(new_offsets, [batch_size, num_boxes, -1]) + + y_indices = tf.cast(y_indices, dtype=tf.float32) + x_indices = tf.cast(x_indices, dtype=tf.float32) + + height_width = tf.maximum(new_height_width, 0) + heights, widths = tf.unstack(height_width, axis=2) + y_offsets, x_offsets = tf.unstack(offsets, axis=2) + + detection_classes = channel_indices + + num_detections = tf.reduce_sum(tf.cast(detection_scores > 0, dtype=tf.int32), axis=1) + + boxes = tf.stack([y_indices + y_offsets - heights / 2.0, + x_indices + x_offsets - widths / 2.0, + y_indices + y_offsets + heights / 2.0, + x_indices + x_offsets + widths / 2.0], axis=2) + + return boxes, detection_classes, detection_scores, num_detections + + def call(self, inputs): + # Get heatmaps from decoded ourputs via final hourglass stack output + ct_heatmaps = inputs['ct_heatmaps'][-1] + ct_sizes = inputs['ct_size'][-1] + ct_offsets = inputs['ct_offset'][-1] + + batch_size = ct_heatmaps.get_shape()[0] + + # Process heatmaps using 3x3 convolution and applying sigmoid + peaks = self.process_heatmap(ct_heatmaps) + + scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, k=self._max_detections, per_channel=self._detect_per_channel) + + boxes, classes, scores, num_det = self.get_boxes(scores, + y_indices, x_indices, channel_indices, ct_sizes, ct_offsets) + + boxes = boxes / ct_heatmaps.get_shape()[1] + + boxes = tf.expand_dims(boxes, axis=-2) + multi_class_scores = tf.gather_nd( + peaks, tf.stack([y_indices, x_indices], -1), batch_dims=1) + + boxes, scores, _, num_det = tf.image.combined_non_max_suppression(boxes=boxes, + scores=multi_class_scores, max_output_size_per_class=self._max_detections, + max_total_size=self._max_detections, score_threshold=0.1) + + return { + 'bbox': boxes, + 'classes': classes, + 'confidence': scores, + 'num_dets': num_det + } \ No newline at end of file diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index 4918c3274..d431f75f6 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -25,14 +25,15 @@ def load_weights_model(model, weights_dict, backbone_name, decoder_name): decoder_name: String, indicating the desired decoder configuration """ print("Loading model weights\n") - load_weights_backbone(model.backbone, + n_weights = 0 + n_weights += load_weights_backbone(model.backbone, weights_dict['model']['_feature_extractor']['_network'], backbone_name) - load_weights_decoder(model.decoder, + n_weights += load_weights_decoder(model.decoder, weights_dict['model']['_prediction_head_dict'], decoder_name) - print("Successfully loaded model weights\n") + print("Successfully loaded {} model weights.\n".format(n_weights)) def get_backbone_layer_cfgs(weights_dict, backbone_name): """ Fetches the config classes for the backbone. @@ -62,6 +63,8 @@ def load_weights_backbone(backbone, weights_dict, backbone_name): backbone: keras.Model backbone weights_dict: Dictionary that stores the backbone model weights backbone_name: String, indicating the desired backbone configuration + Returns: + Number of weights loaded in """ print("Loading backbone weights\n") backbone_layers = backbone.layers @@ -77,7 +80,7 @@ def load_weights_backbone(backbone, weights_dict, backbone_name): n_weights_total += n_weights if len(cfgs) == 0: print("{} Weights have been loaded for {} / {} layers\n".format(n_weights_total, i+1, len(backbone_layers))) - return + return n_weights_total cfg = cfgs.pop(0) def get_decoder_layer_cfgs(weights_dict, decoder_name): @@ -108,6 +111,8 @@ def load_weights_decoder(decoder, weights_dict, decoder_name): decoder: keras.Model decoder weights_dict: Dictionary that stores the decoder model weights decoder_name: String, indicating the desired decoder configuration + Returns: + Number of weights loaded in """ print("Loading decoder weights\n") decoder_layers = decoder.layers @@ -123,18 +128,22 @@ def load_weights_decoder(decoder, weights_dict, decoder_name): n_weights_total += n_weights if len(cfgs) == 0: print("{} Weights have been loaded for {} / {} layers\n".format(n_weights_total, i+1, len(decoder_layers))) - return + return n_weights_total cfg = cfgs.pop(0) if __name__ == '__main__': - input_specs = tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]) + input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) config = CenterNetTask() model, loss = build_centernet(input_specs=input_specs, task_config=config, l2_regularization=0) - weights_dict = get_model_weights_as_dict(CKPT_PATH) + weights_dict, n_weights = get_model_weights_as_dict(CKPT_PATH) load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + # Note number of weights read and loaded differ by two because + # we also read in the checkpoint save_counter and object_graph + # that are not weights to the model + # Uncomment line below to write weights dict key names to a file # write_dict_as_tree(weights_dict, filename="centernet/utils/weight_utils/MODEL_VARS.txt") \ No newline at end of file diff --git a/centernet/utils/weight_utils/test_load_weights.py b/centernet/utils/weight_utils/test_load_weights.py new file mode 100644 index 000000000..fbf3873c5 --- /dev/null +++ b/centernet/utils/weight_utils/test_load_weights.py @@ -0,0 +1,37 @@ +from yolo.demos.video_detect_gpu import FastVideo +from yolo.demos.video_detect_cpu import runner + +from centernet.modeling.CenterNet import build_centernet +from centernet.configs.centernet import CenterNetTask + +from centernet.utils.weight_utils.load_weights import get_model_weights_as_dict, load_weights_model + +import tensorflow as tf + +CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' + +if __name__ == '__main__': + input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) + config = CenterNetTask() + + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + + weights_dict, n_weights = get_model_weights_as_dict(CKPT_PATH) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + + cap = FastVideo( + 0, + model=model, + process_width=512, + process_height=512, + preprocess_with_gpu=False, + classes=91, + print_conf=True, + max_batch=1, + disp_h=512, + scale_que=1, + wait_time='dynamic') + cap.run() + # runner(model, 0, 512, 512) + diff --git a/centernet/utils/weight_utils/tf_to_dict.py b/centernet/utils/weight_utils/tf_to_dict.py index 2739e7383..4b6465181 100644 --- a/centernet/utils/weight_utils/tf_to_dict.py +++ b/centernet/utils/weight_utils/tf_to_dict.py @@ -33,6 +33,7 @@ def get_model_weights_as_dict(ckpt_path): ckpt_path: String, indicating filepath of the TF checkpoint Returns: Dictionary where the checkpoint weights are stored + Number of weights read """ print("\nConverting model checkpoint from {} to weights dictionary\n".format(ckpt_path)) reader = tf.train.load_checkpoint(ckpt_path) @@ -41,15 +42,18 @@ def get_model_weights_as_dict(ckpt_path): variable_keys = shape_from_key.keys() weights_dict = {} + n_read = 0 for key in variable_keys: shape = shape_from_key[key] dtype = dtype_from_key[key] value = reader.get_tensor(key) + n_read += tf.size(value) update_weights_dict(weights_dict, key, value) - print("Successfully read checkpoint weights\n") - return weights_dict + print("Successfully read {} checkpoint weights\n".format(n_read)) + return weights_dict, n_read + def write_dict_as_tree(dictionary, filename, spaces=0): """ Writes nested dictionary keys to a file. diff --git a/yolo/dataloaders/dataset_specs/coco-labels-paper.names b/yolo/dataloaders/dataset_specs/coco-labels-paper.names new file mode 100644 index 000000000..5378c6cda --- /dev/null +++ b/yolo/dataloaders/dataset_specs/coco-labels-paper.names @@ -0,0 +1,91 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +traffic light +fire hydrant +street sign +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +hat +backpack +umbrella +shoe +eye glasses +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +plate +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +couch +potted plant +bed +mirror +dining table +window +desk +toilet +door +tv +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +blender +book +clock +vase +scissors +teddy bear +hair drier +toothbrush +hair brush \ No newline at end of file From 96bcd9eaf0251689ba07322588c165803ea70371 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 10 Mar 2021 17:27:29 -0500 Subject: [PATCH 065/132] need to merge from yolo_debug --- .../modeling/layers/detection_generator.py | 29 ++++-- .../dataset_specs/coco-labels-paper.names | 91 ------------------- 2 files changed, 19 insertions(+), 101 deletions(-) delete mode 100644 yolo/dataloaders/dataset_specs/coco-labels-paper.names diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index ed92362d5..1a2fcc979 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -89,9 +89,11 @@ def multi_range(self, def get_top_k_peaks(self, feature_map_peaks, + batch_size, + width, + num_channels, k=100, per_channel=False): - (batch_size, _, width, num_channels) = feature_map_peaks.get_shape() if per_channel: if k == 1: @@ -138,8 +140,9 @@ def get_boxes(self, x_indices, channel_indices, height_width_predictions, - offset_predictions): - batch_size, num_boxes = y_indices.get_shape() + offset_predictions, + batch_size, + num_boxes): # TF Lite does not support tf.gather with batch_dims > 0, so we need to use # tf_gather_nd instead and here we prepare the indices for that. @@ -158,8 +161,8 @@ def get_boxes(self, x_indices = tf.cast(x_indices, dtype=tf.float32) height_width = tf.maximum(new_height_width, 0) - heights, widths = tf.unstack(height_width, axis=2) - y_offsets, x_offsets = tf.unstack(offsets, axis=2) + heights, widths = tf.split(height_width, 2, axis=-1) + y_offsets, x_offsets = tf.split(offsets, 2, axis=-1) detection_classes = channel_indices @@ -177,18 +180,24 @@ def call(self, inputs): ct_heatmaps = inputs['ct_heatmaps'][-1] ct_sizes = inputs['ct_size'][-1] ct_offsets = inputs['ct_offset'][-1] - - batch_size = ct_heatmaps.get_shape()[0] + + shape = tf.shape(ct_heatmaps) + batch_size, height, width, num_channels = shape[0], shape[1], shape[2], shape[3] # Process heatmaps using 3x3 convolution and applying sigmoid peaks = self.process_heatmap(ct_heatmaps) - scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, k=self._max_detections, per_channel=self._detect_per_channel) + scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, + batch_size, + width, + num_channels, + k=self._max_detections, + per_channel=self._detect_per_channel) boxes, classes, scores, num_det = self.get_boxes(scores, - y_indices, x_indices, channel_indices, ct_sizes, ct_offsets) + y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) - boxes = boxes / ct_heatmaps.get_shape()[1] + boxes = boxes / tf.cast(height, tf.float32) boxes = tf.expand_dims(boxes, axis=-2) multi_class_scores = tf.gather_nd( diff --git a/yolo/dataloaders/dataset_specs/coco-labels-paper.names b/yolo/dataloaders/dataset_specs/coco-labels-paper.names deleted file mode 100644 index 5378c6cda..000000000 --- a/yolo/dataloaders/dataset_specs/coco-labels-paper.names +++ /dev/null @@ -1,91 +0,0 @@ -person -bicycle -car -motorcycle -airplane -bus -train -truck -boat -traffic light -fire hydrant -street sign -stop sign -parking meter -bench -bird -cat -dog -horse -sheep -cow -elephant -bear -zebra -giraffe -hat -backpack -umbrella -shoe -eye glasses -handbag -tie -suitcase -frisbee -skis -snowboard -sports ball -kite -baseball bat -baseball glove -skateboard -surfboard -tennis racket -bottle -plate -wine glass -cup -fork -knife -spoon -bowl -banana -apple -sandwich -orange -broccoli -carrot -hot dog -pizza -donut -cake -chair -couch -potted plant -bed -mirror -dining table -window -desk -toilet -door -tv -laptop -mouse -remote -keyboard -cell phone -microwave -oven -toaster -sink -refrigerator -blender -book -clock -vase -scissors -teddy bear -hair drier -toothbrush -hair brush \ No newline at end of file From ce61a7c4654091afe6f35b0cb8d99f22ce8d1b4e Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 11 Mar 2021 15:21:11 -0500 Subject: [PATCH 066/132] demo working, adding docs later --- .../modeling/layers/detection_generator.py | 100 ++++++++---------- 1 file changed, 43 insertions(+), 57 deletions(-) diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 1a2fcc979..21eab1830 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -1,23 +1,28 @@ from tensorflow import keras as ks import tensorflow as tf +from yolo.ops.nms_ops import nms + class CenterNetLayer(ks.Model): def __init__(self, max_detections=100, - detect_per_channel=False, center_thresh=0.1, + peak_error=1e-6, + class_offset=1, + use_nms=True, **kwargs): super().__init__(**kwargs) self._max_detections = max_detections - self._detect_per_channel = detect_per_channel self._center_thresh = center_thresh + self._peak_error = peak_error + self._class_offset = class_offset + self._use_nms = use_nms def process_heatmap(self, feature_map, - kernel_size=3, - peak_error=1e-6): + kernel_size=3): feature_map = tf.math.sigmoid(feature_map) if not kernel_size or kernel_size == 1: feature_map_peaks = feature_map @@ -26,7 +31,7 @@ def process_heatmap(self, feature_map, ksize=kernel_size, strides=1, padding='SAME') feature_map_peak_mask = tf.math.abs( - feature_map - feature_map_max_pool) < peak_error + feature_map - feature_map_max_pool) < self._peak_error # Zero out everything that is not a peak. feature_map_peaks = ( @@ -91,48 +96,22 @@ def get_top_k_peaks(self, feature_map_peaks, batch_size, width, - num_channels, - k=100, - per_channel=False): - - if per_channel: - if k == 1: - feature_map_flattened = tf.reshape( - feature_map_peaks, [batch_size, -1, num_channels]) - scores = tf.math.reduce_max(feature_map_flattened, axis=1) - peak_flat_indices = tf.math.argmax( - feature_map_flattened, axis=1, output_type=tf.dtypes.int32) - peak_flat_indices = tf.expand_dims(peak_flat_indices, axis=-1) - else: - # Perform top k over batch and channels. - feature_map_peaks_transposed = tf.transpose(feature_map_peaks, - perm=[0, 3, 1, 2]) - feature_map_peaks_transposed = tf.reshape( - feature_map_peaks_transposed, [batch_size, num_channels, -1]) - scores, peak_flat_indices = tf.math.top_k( - feature_map_peaks_transposed, k=k) - # Convert the indices such that they represent the location in the full - # (flattened) feature map of size [batch, height * width * channels]. - channel_idx = tf.range(num_channels)[tf.newaxis, :, tf.newaxis] - peak_flat_indices = num_channels * peak_flat_indices + channel_idx - scores = tf.reshape(scores, [batch_size, -1]) - peak_flat_indices = tf.reshape(peak_flat_indices, [batch_size, -1]) - else: - if k == 1: - feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) - scores = tf.math.reduce_max(feature_map_peaks_flat, axis=1, keepdims=True) - peak_flat_indices = tf.expand_dims(tf.math.argmax( - feature_map_peaks_flat, axis=1, output_type=tf.dtypes.int32), axis=-1) - else: - feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) - scores, peak_flat_indices = tf.math.top_k(feature_map_peaks_flat, k=k) + num_classes, + k=100): + + # Flatten the entire prediction per batch + # Now the feature_map_peaks are in shape [batch_size, num_classes * width * height] + feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) + + # top_scores and top_indeces have shape [batch_size, k] + top_scores, top_indeces = tf.math.top_k(feature_map_peaks_flat, k=k) # Get x, y and channel indices corresponding to the top indices in the flat # array. y_indices, x_indices, channel_indices = ( - self.get_row_col_channel_indices_from_flattened_indices( - peak_flat_indices, width, num_channels)) - return scores, y_indices, x_indices, channel_indices + self.get_row_col_channel_indices_from_flattened_indices( + top_indeces, width, num_classes)) + return top_scores, y_indices, x_indices, channel_indices def get_boxes(self, detection_scores, @@ -151,20 +130,26 @@ def get_boxes(self, tf.reshape(y_indices, [-1]), tf.reshape(x_indices, [-1]) ], axis=1) + + # Get the heights and widths of center points new_height_width = tf.gather_nd(height_width_predictions, combined_indices) new_height_width = tf.reshape(new_height_width, [batch_size, num_boxes, -1]) + height_width = tf.maximum(new_height_width, 0) + + heights = height_width[..., 0] + widths = height_width[..., 1] + # Get the heights and widths of center points new_offsets = tf.gather_nd(offset_predictions, combined_indices) offsets = tf.reshape(new_offsets, [batch_size, num_boxes, -1]) + y_offsets = offsets[..., 0] + x_offsets = offsets[..., 1] + y_indices = tf.cast(y_indices, dtype=tf.float32) x_indices = tf.cast(x_indices, dtype=tf.float32) - height_width = tf.maximum(new_height_width, 0) - heights, widths = tf.split(height_width, 2, axis=-1) - y_offsets, x_offsets = tf.split(offsets, 2, axis=-1) - - detection_classes = channel_indices + detection_classes = channel_indices + self._class_offset num_detections = tf.reduce_sum(tf.cast(detection_scores > 0, dtype=tf.int32), axis=1) @@ -187,25 +172,26 @@ def call(self, inputs): # Process heatmaps using 3x3 convolution and applying sigmoid peaks = self.process_heatmap(ct_heatmaps) + # Get top scores along with their x, y, and class scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, - batch_size, - width, - num_channels, - k=self._max_detections, - per_channel=self._detect_per_channel) + batch_size, width, num_channels, k=self._max_detections) + # Parse the score and indeces into bounding boxes boxes, classes, scores, num_det = self.get_boxes(scores, y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) + # Normalize bounding boxes boxes = boxes / tf.cast(height, tf.float32) - + boxes = tf.expand_dims(boxes, axis=-2) multi_class_scores = tf.gather_nd( peaks, tf.stack([y_indices, x_indices], -1), batch_dims=1) - boxes, scores, _, num_det = tf.image.combined_non_max_suppression(boxes=boxes, - scores=multi_class_scores, max_output_size_per_class=self._max_detections, - max_total_size=self._max_detections, score_threshold=0.1) + boxes, _, scores = nms(boxes=boxes, classes=multi_class_scores, + confidence=scores, k=self._max_detections, limit_pre_thresh=True, + pre_nms_thresh=0.1, nms_thresh=0.4) + + num_dets = tf.reduce_sum(tf.cast(scores > 0, dtype=tf.int32), axis=1) return { 'bbox': boxes, From 40e5cf8a35265083a996d070c8d19c9927bcc2e6 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 11 Mar 2021 21:45:07 -0500 Subject: [PATCH 067/132] detection generator finished --- .../modeling/layers/detection_generator.py | 181 +++++++++++++++--- 1 file changed, 153 insertions(+), 28 deletions(-) diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 21eab1830..6b5a78952 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -4,31 +4,88 @@ from yolo.ops.nms_ops import nms class CenterNetLayer(ks.Model): - + """ CenterNet Detection Generator + + Parses predictions from the CenterNet decoder into the final bounding boxes, + confidences, and classes. This class contains repurposed methods from the + TensorFlow Object Detection API + in: https://github.com/tensorflow/models/blob/master/research/object_detection/meta_architectures/center_net_meta_arch.py + """ def __init__(self, max_detections=100, - center_thresh=0.1, peak_error=1e-6, - class_offset=1, + peak_extract_kernel_size=3, use_nms=True, + center_thresh=0.1, + iou_thresh=0.4, + class_offset=1, **kwargs): + """ + Args: + max_detections: An integer specifying the maximum number of bounding + boxes generated. This is an upper bound, so the number of generated + boxes may be less than this due to thresholding/non-maximum suppression. + peak_error: A float for determining non-valid heatmap locations to mask. + peak_extract_kernel_size: An integer indicating the kernel size used when + performing max-pool over the heatmaps to detect valid center locations + from its neighbors. From the paper, set this to 3 to detect valid. + locations that have responses greater than its 8-connected neighbors + use_nms: Boolean for whether or not to use non-maximum suppression to + filter the bounding boxes. + center_thresh: A float that sets the threshold for a heatmap peak to be + considered a valid detection. + iou_thresh: A float for the threshold IOU when filtering bounding boxes. + class_offset: An integer indicating to add an offset to the class + prediction if the dataset labels have been shifted. + call Returns: + Dictionary with keys 'bbox', 'classes', 'confidence', and 'num_dets' + storing each bounding box in [y_min, x_min, y_max, x_max] format, + its respective class and confidence score, and the number of detections + made. + """ super().__init__(**kwargs) + + # Object center selection parameters self._max_detections = max_detections - self._center_thresh = center_thresh self._peak_error = peak_error - self._class_offset = class_offset + self._peak_extract_kernel_size = peak_extract_kernel_size + + # Non-maximum suppression parameters self._use_nms = use_nms + self._center_thresh = center_thresh + self._iou_thresh = iou_thresh + + self._class_offset = class_offset def process_heatmap(self, feature_map, - kernel_size=3): + kernel_size, + center_thresh): + """ Processes the heatmap into peaks for box selection. + + Given a heatmap, this function first masks out nearby heatmap locations of + the same class using max-pooling such that, ideally, only one center for the + object remains. Then, center locations are masked according to their scores + in comparison to a threshold. NOTE: Repurposed from Google OD API. + + Args: + feature_map: A Tensor with shape [batch_size, height, width, num_classes] + which is the center heatmap predictions. + kernel_size: An integer value for max-pool kernel size. + center_thresh: The threshold value for valid center location scores. + + Returns: + A Tensor with the same shape as the input but with non-valid center + prediction locations masked out. + """ feature_map = tf.math.sigmoid(feature_map) if not kernel_size or kernel_size == 1: feature_map_peaks = feature_map else: feature_map_max_pool = tf.nn.max_pool( - feature_map, ksize=kernel_size, strides=1, padding='SAME') + feature_map, ksize=kernel_size, + strides=1, padding='SAME') feature_map_peak_mask = tf.math.abs( feature_map - feature_map_max_pool) < self._peak_error @@ -37,18 +94,26 @@ def process_heatmap(self, feature_map_peaks = ( feature_map * tf.cast(feature_map_peak_mask, tf.float32)) + # Zero out peaks whose scores do not exceed threshold + valid_peaks_mask = feature_map_peaks > center_thresh + feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.float32) + return feature_map_peaks def get_row_col_channel_indices_from_flattened_indices(self, indices, num_cols, num_channels): - """Computes row, column and channel indices from flattened indices. + """ Computes row, column and channel indices from flattened indices. + + NOTE: Repurposed from Google OD API. + Args: indices: An integer tensor of any shape holding the indices in the flattened space. num_cols: Number of columns in the image (width). num_channels: Number of channels in the image. + Returns: row_indices: The row indices corresponding to each of the input indices. Same shape as indices. @@ -67,14 +132,17 @@ def get_row_col_channel_indices_from_flattened_indices(self, return row_indices, col_indices, channel_indices def multi_range(self, - limit, - value_repetitions=1, - range_repetitions=1, - dtype=tf.int32): - """Creates a sequence with optional value duplication and range repetition. + limit, + value_repetitions=1, + range_repetitions=1, + dtype=tf.int32): + """ Creates a sequence with optional value duplication and range repetition. + As an example (see the Args section for more details), _multi_range(limit=2, value_repetitions=3, range_repetitions=4) returns: [0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1] + NOTE: Repurposed from Google OD API. + Args: limit: A 0-D Tensor (scalar). Upper limit of sequence, exclusive. value_repetitions: Integer. The number of times a value in the sequence is @@ -82,6 +150,7 @@ def multi_range(self, range_repetitions: Integer. The number of times the range is repeated. With range_repetitions=3, the result is [0, 1, 2, .., 0, 1, 2, ..]. dtype: The type of the elements of the resulting tensor. + Returns: A 1-D tensor of type `dtype` and size [`limit` * `value_repetitions` * `range_repetitions`] that contains the @@ -98,19 +167,43 @@ def get_top_k_peaks(self, width, num_classes, k=100): + """ Gets the scores and indices of the top-k peaks from the feature map + + This function flattens the feature map in order to retrieve the top-k + peaks, then computes the x, y, and class indices for those scores. + NOTE: Repurposed from Google OD API. + Args: + feature_map_peaks: A Tensor with shape [batch_size, height, width, num_classes] + which is the processed center heatmap peaks. + batch_size: An integer that indicates the batch size of the input. + width: An integer that indicates the width (and also height) of the input. + num_classes: An integer for the number of possible classes. This is also + the channel depth of the input. + k: Integer that controls how many peaks to select. + + Returns: + top_scores: A Tensor with shape [batch_size, k] containing the top-k scores. + y_indices: A Tensor with shape [batch_size, k] containing the top-k + y-indices corresponding to top_scores. + x_indices: A Tensor with shape [batch_size, k] containing the top-k + x-indices corresponding to top_scores. + channel_indices: A Tensor with shape [batch_size, k] containing the top-k + channel indices corresponding to top_scores. + """ # Flatten the entire prediction per batch # Now the feature_map_peaks are in shape [batch_size, num_classes * width * height] feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) - # top_scores and top_indeces have shape [batch_size, k] - top_scores, top_indeces = tf.math.top_k(feature_map_peaks_flat, k=k) + # top_scores and top_indices have shape [batch_size, k] + top_scores, top_indices = tf.math.top_k(feature_map_peaks_flat, k=k) # Get x, y and channel indices corresponding to the top indices in the flat # array. y_indices, x_indices, channel_indices = ( self.get_row_col_channel_indices_from_flattened_indices( - top_indeces, width, num_classes)) + top_indices, width, num_classes)) + return top_scores, y_indices, x_indices, channel_indices def get_boxes(self, @@ -122,14 +215,42 @@ def get_boxes(self, offset_predictions, batch_size, num_boxes): + """ Organizes prediction information into the final bounding boxes. + + NOTE: Repurposed from Google OD API. + + Args: + detection_scores: A Tensor with shape [batch_size, k] containing the + top-k scores. + y_indices: A Tensor with shape [batch_size, k] containing the top-k + y-indices corresponding to top_scores. + x_indices: A Tensor with shape [batch_size, k] containing the top-k + x-indices corresponding to top_scores. + channel_indices: A Tensor with shape [batch_size, k] containing the top-k + channel indices corresponding to top_scores. + height_width_predictions: A Tensor with shape [batch_size, height, width, 2] + containing the object size predictions. + offset_predictions: A Tensor with shape [batch_size, height, width, 2] + containing the object local offset predictions. + batch_size: An integer for the batch size of the input. + num_boxes: The number of boxes. + Returns: + boxes: A Tensor with shape [batch_size, num_boxes, 4] that contains the + bounding box coordinates in [y_min, x_min, y_max, x_max] format. + detection_classes: A Tensor with shape [batch_size, 100] that gives the + class prediction for each box. + detection_scores: A Tensor with shape [batch_size, 100] that gives the + confidence score for each box. + num_detections: Number of non-zero confidence detections made. + """ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use # tf_gather_nd instead and here we prepare the indices for that. combined_indices = tf.stack([ self.multi_range(batch_size, value_repetitions=num_boxes), tf.reshape(y_indices, [-1]), tf.reshape(x_indices, [-1]) - ], axis=1) + ], axis=1) # Get the heights and widths of center points new_height_width = tf.gather_nd(height_width_predictions, combined_indices) @@ -139,7 +260,7 @@ def get_boxes(self, heights = height_width[..., 0] widths = height_width[..., 1] - # Get the heights and widths of center points + # Get the offsets of center points new_offsets = tf.gather_nd(offset_predictions, combined_indices) offsets = tf.reshape(new_offsets, [batch_size, num_boxes, -1]) @@ -161,7 +282,7 @@ def get_boxes(self, return boxes, detection_classes, detection_scores, num_detections def call(self, inputs): - # Get heatmaps from decoded ourputs via final hourglass stack output + # Get heatmaps from decoded outputs via final hourglass stack output ct_heatmaps = inputs['ct_heatmaps'][-1] ct_sizes = inputs['ct_size'][-1] ct_offsets = inputs['ct_offset'][-1] @@ -170,26 +291,30 @@ def call(self, inputs): batch_size, height, width, num_channels = shape[0], shape[1], shape[2], shape[3] # Process heatmaps using 3x3 convolution and applying sigmoid - peaks = self.process_heatmap(ct_heatmaps) + peaks = self.process_heatmap(ct_heatmaps, + kernel_size=self._peak_extract_kernel_size, + center_thresh=self._center_thresh) # Get top scores along with their x, y, and class scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, batch_size, width, num_channels, k=self._max_detections) - # Parse the score and indeces into bounding boxes + # Parse the score and indices into bounding boxes boxes, classes, scores, num_det = self.get_boxes(scores, y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) # Normalize bounding boxes boxes = boxes / tf.cast(height, tf.float32) - - boxes = tf.expand_dims(boxes, axis=-2) - multi_class_scores = tf.gather_nd( - peaks, tf.stack([y_indices, x_indices], -1), batch_dims=1) - boxes, _, scores = nms(boxes=boxes, classes=multi_class_scores, - confidence=scores, k=self._max_detections, limit_pre_thresh=True, - pre_nms_thresh=0.1, nms_thresh=0.4) + # Apply nms + if self._use_nms: + boxes = tf.expand_dims(boxes, axis=-2) + multi_class_scores = tf.gather_nd(peaks, + tf.stack([y_indices, x_indices], -1), batch_dims=1) + + boxes, _, scores = nms(boxes=boxes, classes=multi_class_scores, + confidence=scores, k=self._max_detections, limit_pre_thresh=True, + pre_nms_thresh=self._center_thresh, nms_thresh=self._iou_thresh) num_dets = tf.reduce_sum(tf.cast(scores > 0, dtype=tf.int32), axis=1) From 6dedb04f440f44ba8aa541c64d436a5291d7723e Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 12 Mar 2021 11:30:30 -0500 Subject: [PATCH 068/132] comment fix --- centernet/modeling/layers/detection_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 6b5a78952..cb284e44f 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -290,7 +290,7 @@ def call(self, inputs): shape = tf.shape(ct_heatmaps) batch_size, height, width, num_channels = shape[0], shape[1], shape[2], shape[3] - # Process heatmaps using 3x3 convolution and applying sigmoid + # Process heatmaps using 3x3 max pool and applying sigmoid peaks = self.process_heatmap(ct_heatmaps, kernel_size=self._peak_extract_kernel_size, center_thresh=self._center_thresh) From b5e5b3aad470f1382d4c6b58338886181a5515d9 Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Mon, 15 Mar 2021 10:21:36 -0700 Subject: [PATCH 069/132] moved groundtruth to preprocessing ops --- centernet/ops/preprocessing_ops.py | 104 +++++++++++++++++++++++++++- centernet/utils/groundtruth.py | 99 -------------------------- centernet/utils/groundtruth_test.py | 3 - 3 files changed, 102 insertions(+), 104 deletions(-) delete mode 100644 centernet/utils/groundtruth.py delete mode 100644 centernet/utils/groundtruth_test.py diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 0dd42e503..cd19e37ba 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -1,2 +1,102 @@ -# TODO: Move groundtruth.py to preprocessing_ops.py when possible -from centernet.utils.groundtruth import * +# Moved groundtruth.py here +import tensorflow as tf +from yolo.ops import preprocessing_ops + + +def _smallest_positive_root(a, b, c) -> tf.Tensor: + """ + Returns the smallest positive root of a quadratic equation. + This implements the fixed version in https://github.com/princeton-vl/CornerNet. + """ + + discriminant = tf.sqrt(b ** 2 - 4 * a * c) + + root1 = (-b - discriminant) / (2 * a) + root2 = (-b + discriminant) / (2 * a) + + return (-b + discriminant) / (2) # tf.where(tf.less(root1, 0), root2, root1) + +def gaussian_radius(det_size, min_overlap=0.7) -> int: + """ + Given a bounding box size, returns a lower bound on how far apart the + corners of another bounding box can lie while still maintaining the given + minimum overlap, or IoU. Modified from implementation found in + https://github.com/tensorflow/models/blob/master/research/object_detection/core/target_assigner.py. + + Params: + det_size (tuple): tuple of integers representing height and width + min_overlap (tf.float32): minimum IoU desired + Returns: + int representing desired gaussian radius + """ + height, width = det_size + + # Case where detected box is offset from ground truth and no box completely + # contains the other. + + a1 = 1 + b1 = -(height + width) + c1 = width * height * (1 - min_overlap) / (1 + min_overlap) + r1 = _smallest_positive_root(a1, b1, c1) + + # Case where detection is smaller than ground truth and completely contained + # in it. + + a2 = 4 + b2 = -2 * (height + width) + c2 = (1 - min_overlap) * width * height + r2 = _smallest_positive_root(a2, b2, c2) + + # Case where ground truth is smaller than detection and completely contained + # in it. + + a3 = 4 * min_overlap + b3 = 2 * min_overlap * (height + width) + c3 = (min_overlap - 1) * width * height + r3 = _smallest_positive_root(a3, b3, c3) + # TODO discuss whether to return scalar or tensor + # return tf.reduce_min([r1, r2, r3], axis=0) + + return tf.reduce_min([r1, r2, r3], axis=0) + +def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: + """ + This represents the penalty reduction around a point. + Params: + radius (int): integer for radius of penalty reduction + type (tf.dtypes.DType): datatype of returned tensor + Returns: + tf.Tensor of shape (2 * radius + 1, 2 * radius + 1). + """ + width = 2 * radius + 1 + sigma = radius / 3 + x = tf.reshape(tf.range(width, dtype=type) - radius, (width, 1)) + y = tf.reshape(tf.range(width, dtype=type) - radius, (1, width)) + exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) + return tf.math.exp(exponent) + +def draw_gaussian(heatmap, center, radius, k=1): + """ + Draws a gaussian heatmap around a center point given a radius. + Params: + heatmap (tf.Tensor): heatmap placeholder to fill + center (int): integer for center of gaussian + radius (int): integer for radius of gaussian + k (int): scaling factor for gaussian + """ + + diameter = 2 * radius + 1 + gaussian = _gaussian_penalty(radius) + + x, y = center + + height, width = heatmap.shape[0:2] + + left, right = min(x, radius), min(width - x, radius + 1) + top, bottom = min(y, radius), min(height - y, radius + 1) + + masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] + # TODO: make sure this replicates original functionality + # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) + masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * k) diff --git a/centernet/utils/groundtruth.py b/centernet/utils/groundtruth.py deleted file mode 100644 index 91f20d10f..000000000 --- a/centernet/utils/groundtruth.py +++ /dev/null @@ -1,99 +0,0 @@ -import tensorflow as tf - -def _smallest_positive_root(a, b, c) -> tf.Tensor: - """ - Returns the smallest positive root of a quadratic equation. - This implements the fixed version in https://github.com/princeton-vl/CornerNet. - """ - - discriminant = tf.sqrt(b ** 2 - 4 * a * c) - - root1 = (-b - discriminant) / (2 * a) - root2 = (-b + discriminant) / (2 * a) - - return (-b + discriminant) / (2) # tf.where(tf.less(root1, 0), root2, root1) - -def gaussian_radius(det_size, min_overlap=0.7) -> int: - """ - Given a bounding box size, returns a lower bound on how far apart the - corners of another bounding box can lie while still maintaining the given - minimum overlap, or IoU. Modified from implementation found in - https://github.com/tensorflow/models/blob/master/research/object_detection/core/target_assigner.py. - - Params: - det_size (tuple): tuple of integers representing height and width - min_overlap (tf.float32): minimum IoU desired - Returns: - int representing desired gaussian radius - """ - height, width = det_size - - # Case where detected box is offset from ground truth and no box completely - # contains the other. - - a1 = 1 - b1 = -(height + width) - c1 = width * height * (1 - min_overlap) / (1 + min_overlap) - r1 = _smallest_positive_root(a1, b1, c1) - - # Case where detection is smaller than ground truth and completely contained - # in it. - - a2 = 4 - b2 = -2 * (height + width) - c2 = (1 - min_overlap) * width * height - r2 = _smallest_positive_root(a2, b2, c2) - - # Case where ground truth is smaller than detection and completely contained - # in it. - - a3 = 4 * min_overlap - b3 = 2 * min_overlap * (height + width) - c3 = (min_overlap - 1) * width * height - r3 = _smallest_positive_root(a3, b3, c3) - # TODO discuss whether to return scalar or tensor - # return tf.reduce_min([r1, r2, r3], axis=0) - - return tf.reduce_min([r1, r2, r3], axis=0) - -def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: - """ - This represents the penalty reduction around a point. - Params: - radius (int): integer for radius of penalty reduction - type (tf.dtypes.DType): datatype of returned tensor - Returns: - tf.Tensor of shape (2 * radius + 1, 2 * radius + 1). - """ - width = 2 * radius + 1 - sigma = radius / 3 - x = tf.reshape(tf.range(width, dtype=type) - radius, (width, 1)) - y = tf.reshape(tf.range(width, dtype=type) - radius, (1, width)) - exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) - return tf.math.exp(exponent) - -def draw_gaussian(heatmap, center, radius, k=1): - """ - Draws a gaussian heatmap around a center point given a radius. - Params: - heatmap (tf.Tensor): heatmap placeholder to fill - center (int): integer for center of gaussian - radius (int): integer for radius of gaussian - k (int): scaling factor for gaussian - """ - - diameter = 2 * radius + 1 - gaussian = _gaussian_penalty(radius) - - x, y = center - - height, width = heatmap.shape[0:2] - - left, right = min(x, radius), min(width - x, radius + 1) - top, bottom = min(y, radius), min(height - y, radius + 1) - - masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] - masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] - # TODO: make sure this replicates original functionality - # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * k) diff --git a/centernet/utils/groundtruth_test.py b/centernet/utils/groundtruth_test.py deleted file mode 100644 index 9d8af4660..000000000 --- a/centernet/utils/groundtruth_test.py +++ /dev/null @@ -1,3 +0,0 @@ -from centernet.ops.preprocessing_ops_test import * -if __name__ == '__main__': - tf.test.main() From efefbe78f484ee43cd24a41fc598aaa213f8bb4a Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 17 Mar 2021 00:16:22 -0400 Subject: [PATCH 070/132] Factorize out heatmap function --- centernet/dataloaders/centernet_input.py | 193 ++++++++++++----------- 1 file changed, 102 insertions(+), 91 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index d22033947..ebb4036f9 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -13,6 +13,98 @@ def __init__( self._max_num_instances = max_num_instances self._gaussian_iou = gaussian_iou + def _generate_heatmap(self, boxes, output_size, input_size): + tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) + + width_ratio = output_size[1] / input_size[1] + height_ratio = output_size[0] / input_size[0] + + for ind, detection in enumerate(boxes): + category = int(detection[-1]) - 1 + # category = 0 + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + + xct, yct = ( + (detection[2] + detection[0]) / 2, + (detection[3] + detection[1]) / 2 + ) + + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + xtl = int(fxtl) + ytl = int(fytl) + xbr = int(fxbr) + ybr = int(fybr) + xct = int(fxct) + yct = int(fyct) + + if gaussian_bump: + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + width = math.ceil(width * width_ratio) + height = math.ceil(height * height_ratio) + + if gaussian_rad == -1: + radius = gaussian_radius((height, width), self._gaussian_iou) + radius = max(0, int(radius)) + else: + radius = gaussian_rad + tl_heatmaps = draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) + br_heatmaps = draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) + ct_heatmaps = draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) + + else: + # tl_heatmaps[category, ytl, xtl] = 1 + # br_heatmaps[category, ybr, xbr] = 1 + # ct_heatmaps[category, yct, xct] = 1 + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ytl, xtl]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, ytl, xtl]], [1]) + + # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] + # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] + # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] + # tl_size[tag_ind] = ytl * output_size[1] + xtl + # br_size[tag_ind] = ybr * output_size[1] + xbr + # ct_size[tag_ind] = yct * output_size[1] + xct + tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) + tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) + br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) + ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) + + labels = { + 'tl_size': tl_size, + 'br_size': br_size, + 'ct_size': ct_size, + 'tl_heatmaps': tl_heatmaps, + 'br_heatmaps': br_heatmaps, + 'ct_heatmaps': ct_heatmaps, + 'tag_masks': tag_masks, + 'tl_offset': tl_offset, + 'br_offset', br_offset, + 'ct_offset': ct_offset, + } + return labels + def _parse_train_data(self, decoded_tensors): """Generates images and labels that are usable for model training. @@ -23,99 +115,18 @@ def _parse_train_data(self, decoded_tensors): images: the image tensor. labels: a dict of Tensors that contains labels. """ - tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) - # TODO: input size, output size image = decoded_tensors["image"] - - width_ratio = output_size[1] / input_size[1] - height_ratio = output_size[0] / input_size[0] - - for ind, detection in enumerate(decoded_tensors["groundtruth_boxes"]): - category = int(detection[-1]) - 1 - # category = 0 - - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] - - xct, yct = ( - (detection[2] + detection[0]) / 2, - (detection[3] + detection[1]) / 2 - ) - - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - xtl = int(fxtl) - ytl = int(fytl) - xbr = int(fxbr) - ybr = int(fybr) - xct = int(fxct) - yct = int(fyct) - - if gaussian_bump: - width = detection[2] - detection[0] - height = detection[3] - detection[1] - - width = math.ceil(width * width_ratio) - height = math.ceil(height * height_ratio) - - if gaussian_rad == -1: - radius = gaussian_radius((height, width), self._gaussian_iou) - radius = max(0, int(radius)) - else: - radius = gaussian_rad - tl_heatmaps = draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) - br_heatmaps = draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) - ct_heatmaps = draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) - - else: - # tl_heatmaps[category, ytl, xtl] = 1 - # br_heatmaps[category, ybr, xbr] = 1 - # ct_heatmaps[category, yct, xct] = 1 - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ytl, xtl]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, ytl, xtl]], [1]) - - # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] - # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] - # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] - # tl_size[tag_ind] = ytl * output_size[1] + xtl - # br_size[tag_ind] = ybr * output_size[1] + xbr - # ct_size[tag_ind] = yct * output_size[1] + xct - tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) - br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) - ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) - - labels = { - 'tl_size': tl_size, - 'br_size': br_size, - 'ct_size': ct_size, - 'tl_heatmaps': tl_heatmaps, - 'br_heatmaps': br_heatmaps, - 'ct_heatmaps': ct_heatmaps, - 'tag_masks': tag_masks, - 'tl_offset': tl_offset, - 'br_offset', br_offset, - 'ct_offset': ct_offset, - } + labels = self._generate_heatmap( + decoded_tensors["groundtruth_boxes"], + output_size, input_size + ) return image, labels def _parse_eval_data(self, data): - pass + image = decoded_tensors["image"] + labels = self._generate_heatmap( + decoded_tensors["groundtruth_boxes"], + output_size, input_size + ) + return image, labels From b7b4c086839596d970b5c30d8835c33b75a09a98 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Wed, 17 Mar 2021 00:28:59 -0400 Subject: [PATCH 071/132] Code to show heatmaps again --- centernet/dataloaders/centernet_input.py | 2 +- centernet/tasks/show_heatmaps.py | 107 +++++++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 centernet/tasks/show_heatmaps.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 1f2192dc0..a9646189e 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -101,7 +101,7 @@ def _generate_heatmap(self, boxes, output_size, input_size): 'ct_heatmaps': ct_heatmaps, 'tag_masks': tag_masks, 'tl_offset': tl_offset, - 'br_offset', br_offset, + 'br_offset': br_offset, 'ct_offset': ct_offset, } return labels diff --git a/centernet/tasks/show_heatmaps.py b/centernet/tasks/show_heatmaps.py new file mode 100644 index 000000000..69a5f23e3 --- /dev/null +++ b/centernet/tasks/show_heatmaps.py @@ -0,0 +1,107 @@ +import tensorflow as tf +import numpy as np +import centernet.tasks as tasks +import centernet.utils as utils + +def gaussian2D(shape, sigma=1): + m, n = [(ss - 1.) / 2. for ss in shape] + y, x = np.ogrid[-m:m+1,-n:n+1] + + h = np.exp(-(x * x + y * y) / (2 * sigma * sigma)) + h[h < np.finfo(h.dtype).eps * h.max()] = 0 + return h + +def draw_gaussian(heatmap, center, radius, k=1, delte=6): + diameter = 2 * radius + 1 + gaussian = gaussian2D((diameter, diameter), sigma=diameter / delte) + + x, y = center + + height, width = heatmap.shape[0:2] + + left, right = min(x, radius), min(width - x, radius + 1) + top, bottom = min(y, radius), min(height - y, radius + 1) + + masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] + np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) + +def gaussian_radius(det_size, min_overlap): + height, width = det_size + + a1 = 1 + b1 = (height + width) + c1 = width * height * (1 - min_overlap) / (1 + min_overlap) + sq1 = np.sqrt(b1 ** 2 - 4 * a1 * c1) + r1 = (b1 + sq1) / 2 + + a2 = 4 + b2 = 2 * (height + width) + c2 = (1 - min_overlap) * width * height + sq2 = np.sqrt(b2 ** 2 - 4 * a2 * c2) + r2 = (b2 + sq2) / 2 + + a3 = 4 * min_overlap + b3 = -2 * min_overlap * (height + width) + c3 = (min_overlap - 1) * width * height + sq3 = np.sqrt(b3 ** 2 - 4 * a3 * c3) + r3 = (b3 + sq3) / 2 + return min(r1, r2, r3) + +def generate_heatmaps(batch_size, categories, output_size, detections, gaussian_iou=0.7): + tl_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) + br_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) + ct_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) + + for b_ind, detection_batches in enumerate(detections): + for ind, detection in enumerate(detection_batches): + category = int(detection[-1]) + #category = 0 + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. + + xtl = int(xtl) + ytl = int(ytl) + xbr = int(xbr) + ybr = int(ybr) + xct = int(xct) + yct = int(yct) + + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + radius = gaussian_radius((height, width), gaussian_iou) + radius = max(0, int(radius)) + + draw_gaussian(tl_heatmaps[b_ind, category], [xtl, ytl], radius) + draw_gaussian(br_heatmaps[b_ind, category], [xbr, ybr], radius) + draw_gaussian(ct_heatmaps[b_ind, category], [xct, yct], radius, delte = 5) + + return tl_heatmaps, br_heatmaps, ct_heatmaps + +class ObjectDetectionTest(tf.test.TestCase): + def generate_heatmaps(self, dectections): + detections = [[ + (10, 30, 15, 17, 0) + ]] + tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) + pass + +if __name__ == '__main__': + # This code is for visualization + import matplotlib.pyplot as plt + detections = [[ + (10, 300, 15, 370, 0), + (100, 300, 150, 370, 0), + (200, 100, 15, 170, 0), + ], + # more images can go here if you like + ] + tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) + # ct_heatmaps[batch_id, class_id, ...] + plt.imshow(ct_heatmaps[0, 0, ...]) + plt.show() + # This is to run the test + # tf.test.main() From 148d2f47653e8cfa59d603a050e90de5b9fcafff Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Tue, 16 Mar 2021 21:32:28 -0700 Subject: [PATCH 072/132] add input test --- centernet/dataloaders/centernet_input.py | 2 +- centernet/dataloaders/centernet_input_test.py | 122 ++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100755 centernet/dataloaders/centernet_input_test.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index f7611819a..345e64d60 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -1,6 +1,6 @@ import tensorflow as tf from official.vision.beta.dataloaders import parser - +from yolo.ops import preprocessing_ops class CenterNetParser(parser.Parser): def __init__( diff --git a/centernet/dataloaders/centernet_input_test.py b/centernet/dataloaders/centernet_input_test.py new file mode 100755 index 000000000..753cf80a6 --- /dev/null +++ b/centernet/dataloaders/centernet_input_test.py @@ -0,0 +1,122 @@ +from yolo.tasks import yolo +from yolo.configs import yolo as yolocfg +from official.core import input_reader +from yolo.dataloaders import yolo_input as YOLO_Detection_Input +from yolo.dataloaders.decoders import tfds_coco_decoder +from yolo.ops import box_ops +import matplotlib.pyplot as plt +import dataclasses +from official.modeling import hyperparams +from official.core import config_definitions as cfg +import tensorflow as tf + + +@dataclasses.dataclass +class Parser(hyperparams.Config): + image_w: int = 416 + image_h: int = 416 + fixed_size: bool = False + jitter_im: float = 0.1 + jitter_boxes: float = 0.005 + min_level: int = 3 + max_level: int = 5 + min_process_size: int = 320 + max_process_size: int = 608 + max_num_instances: int = 200 + random_flip: bool = True + pct_rand: float = 0.5 + aug_rand_saturation: bool = True + aug_rand_brightness: bool = True + aug_rand_zoom: bool = True + aug_rand_hue: bool = True + seed: int = 10 + shuffle_buffer_size: int = 10000 + + +@dataclasses.dataclass +class DataConfig(cfg.DataConfig): + """Input config for training.""" + input_path: str = '' + tfds_name: str = 'coco' + tfds_split: str = 'train' + global_batch_size: int = 10 + is_training: bool = True + dtype: str = 'float16' + decoder = None + parser: Parser = Parser() + shuffle_buffer_size: int = 10000 + tfds_download: bool = True + + +def test_yolo_input_task(): + with tf.device('/CPU:0'): + config = yolocfg.YoloTask( + model=yolocfg.Yolo( + base='v4', + min_level=3, + norm_activation=yolocfg.common.NormActivation(activation='mish'), + #norm_activation = yolocfg.common.NormActivation(activation="leaky"), + #_boxes = ['(10, 14)', '(23, 27)', '(37, 58)', '(81, 82)', '(135, 169)', '(344, 319)'], + #_boxes = ["(10, 13)", "(16, 30)", "(33, 23)","(30, 61)", "(62, 45)", "(59, 119)","(116, 90)", "(156, 198)", "(373, 326)"], + _boxes=[ + '(12, 16)', '(19, 36)', '(40, 28)', '(36, 75)', '(76, 55)', + '(72, 146)', '(142, 110)', '(192, 243)', '(459, 401)' + ], + filter=yolocfg.YoloLossLayer(use_nms=False))) + task = yolo.YoloTask(config) + + # loading both causes issues, but oen at a time is not issue, why? + train_data = task.build_inputs(config.train_data) + test_data = task.build_inputs(config.validation_data) + return train_data, test_data + + +def test_yolo_input(): + with tf.device('/CPU:0'): + params = DataConfig(is_training=True) + num_boxes = 9 + + decoder = tfds_coco_decoder.MSCOCODecoder() + + #anchors = box_rd.read(k = num_boxes, image_width = params.parser.image_w, input_context=None) + anchors = [[12.0, 19.0], [31.0, 46.0], [96.0, 54.0], [46.0, 114.0], + [133.0, 127.0], [79.0, 225.0], [301.0, 150.0], [172.0, 286.0], + [348.0, 340.0]] + # write the boxes to a file + + parser = YOLO_Detection_Input.Parser( + image_w=params.parser.image_w, + fixed_size=params.parser.fixed_size, + jitter_im=params.parser.jitter_im, + jitter_boxes=params.parser.jitter_boxes, + min_level=params.parser.min_level, + max_level=params.parser.max_level, + min_process_size=params.parser.min_process_size, + max_process_size=params.parser.max_process_size, + max_num_instances=params.parser.max_num_instances, + random_flip=params.parser.random_flip, + pct_rand=params.parser.pct_rand, + seed=params.parser.seed, + anchors=anchors) + + reader = input_reader.InputReader( + params, + dataset_fn=tf.data.TFRecordDataset, + decoder_fn=decoder.decode, + parser_fn=parser.parse_fn(params.is_training)) + dataset = reader.read(input_context=None) + return dataset + + +if __name__ == '__main__': + dataset, dsp = test_yolo_input_task() + + for l, (i, j) in enumerate(dataset): + + boxes = box_ops.xcycwh_to_yxyx(j['bbox']) + i = tf.image.draw_bounding_boxes(i, boxes, [[1.0, 0.0, 1.0]]) + plt.imshow(i[0].numpy()) + plt.show() + + if l > 30: + break From ec9242ab11068951b18d1dc29d3d03ee1a698ad5 Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Fri, 19 Mar 2021 17:47:41 -0700 Subject: [PATCH 073/132] added batch sizes to centernet --- centernet/dataloaders/centernet_input.py | 70 +++++++++++++++++++----- centernet/ops/preprocessing_ops.py | 54 ++++++++++++++++-- 2 files changed, 105 insertions(+), 19 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 4de9021be..17ee86781 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -3,7 +3,6 @@ from centernet.ops import preprocessing_ops - class CenterNetParser(parser.Parser): def __init__( self, @@ -16,23 +15,24 @@ def __init__( self._gaussian_iou = gaussian_iou def _generate_heatmap(self, boxes, output_size, input_size): - tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) + batch_size = tf.shape(boxes)[0] + tl_heatmaps = tf.zeros((batch_size, self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + br_heatmaps = tf.zeros((batch_size, self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + ct_heatmaps = tf.zeros((batch_size, self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + tl_offset = tf.zeros((batch_size, self._max_num_instances, 2), dtype=tf.float32) + br_offset = tf.zeros((batch_size, self._max_num_instances, 2), dtype=tf.float32) + ct_offset = tf.zeros((batch_size, self._max_num_instances, 2), dtype=tf.float32) + tl_size = tf.zeros((batch_size, self._max_num_instances), dtype=tf.int64) + br_size = tf.zeros((batch_size, self._max_num_instances), dtype=tf.int64) + ct_size = tf.zeros((batch_size, self._max_num_instances), dtype=tf.int64) + tag_masks = tf.zeros((batch_size, self._max_num_instances), dtype=tf.uint8) width_ratio = output_size[1] / input_size[1] height_ratio = output_size[0] / input_size[0] - for ind, detection in enumerate(boxes): + for tag_ind, detection in enumerate(boxes): category = int(detection[-1]) - 1 - # category = 0 + category = 0 xtl, ytl = detection[0], detection[1] xbr, ybr = detection[2], detection[3] @@ -55,19 +55,26 @@ def _generate_heatmap(self, boxes, output_size, input_size): ybr = int(fybr) xct = int(fxct) yct = int(fyct) + # temporarily set to true + gaussian_bump=True + gaussian_rad=1 if gaussian_bump: width = detection[2] - detection[0] height = detection[3] - detection[1] - width = math.ceil(width * width_ratio) - height = math.ceil(height * height_ratio) + width = tf.math.ceil(width * width_ratio) + height = tf.math.ceil(height * height_ratio) if gaussian_rad == -1: radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) radius = max(0, int(radius)) else: radius = gaussian_rad + + # test + # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[b_ind, category], category, [xtl, ytl], radius) + # inputs heatmap, center, radius, k=1 tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) @@ -132,3 +139,36 @@ def _parse_eval_data(self, data): output_size, input_size ) return image, labels + +class ObjectDetectionTest(tf.test.TestCase): + def generate_heatmaps(self, dectections): + detections = [[ + (10, 30, 15, 17, 0) + ]] + + tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) + pass + +if __name__ == '__main__': + # This code is for visualization + import matplotlib.pyplot as plt + detections = [[ + (10, 300, 15, 370, 0), + (100, 300, 150, 370, 0), + (200, 100, 15, 170, 0), + ], + # more images can go here if you like + ] + + tl_heatmaps, br_heatmaps, ct_heatmaps = CenterNetParser(2, 200, 0.7)._generate_heatmap(tf.constant([ + [10, 300, 15, 370, 0], + [100, 300, 150, 370, 0], + [200, 100, 15, 170, 0], + ], dtype=tf.float32), [416, 416], [416, 416]) + +# tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) + # ct_heatmaps[batch_id, class_id, ...] + plt.imshow(ct_heatmaps[0, 0, ...]) + plt.show() + # This is to run the test + # tf.test.main() diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index cd19e37ba..a79c7b977 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -75,14 +75,15 @@ def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) return tf.math.exp(exponent) -def draw_gaussian(heatmap, center, radius, k=1): +# scaling_factor doesn't do anything right now +def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): """ Draws a gaussian heatmap around a center point given a radius. Params: heatmap (tf.Tensor): heatmap placeholder to fill center (int): integer for center of gaussian radius (int): integer for radius of gaussian - k (int): scaling factor for gaussian + scaling_factor (int): scaling factor for gaussian """ diameter = 2 * radius + 1 @@ -95,8 +96,53 @@ def draw_gaussian(heatmap, center, radius, k=1): left, right = min(x, radius), min(width - x, radius + 1) top, bottom = min(y, radius), min(height - y, radius + 1) - masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] + print('heatmap ',heatmap) + print(len(heatmap)) + print('category ',category) + + # why is this 0? where does 0 come from and is it correct to put it here + heatmap_category = heatmap[0,category, ...] + print('heatmap_category ',heatmap_category) + + # masked_heatmap = heatmap_category[y - top:y + bottom, x - left:x + right] + masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] # TODO: make sure this replicates original functionality # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * k) + masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * scaling_factor) + return masked_heatmap +# def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): +# """ +# Draws a gaussian heatmap around a center point given a radius. +# Params: +# heatmap (tf.Tensor): heatmap placeholder to fill +# center (int): integer for center of gaussian +# radius (int): integer for radius of gaussian +# scaling_factor (int): scaling factor for gaussian +# """ + +# diameter = 2 * radius + 1 +# gaussian = _gaussian_penalty(radius) + +# x, y = center + +# height, width = heatmap.shape[0:2] + +# left, right = min(x, radius), min(width - x, radius + 1) +# top, bottom = min(y, radius), min(height - y, radius + 1) + +# print('heatmap ',heatmap) +# print(len(heatmap)) +# print('category ',category) + +# heatmap_category = heatmap[0][category, ...] + +# print('heatmap_category ',heatmap_category) + +# masked_heatmap = heatmap_category[y - top:y + bottom, x - left:x + right] +# masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] +# # TODO: make sure this replicates original functionality +# # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) +# masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * scaling_factor) +# return masked_heatmap + From c10e55dc9d1a587bfde1e8c6481f74869607ab9c Mon Sep 17 00:00:00 2001 From: anivegesana Date: Sat, 20 Mar 2021 13:52:35 -0400 Subject: [PATCH 074/132] Get it to run again --- centernet/dataloaders/centernet_input.py | 153 ++++++++++++----------- centernet/ops/preprocessing_ops.py | 72 ++++++++--- 2 files changed, 132 insertions(+), 93 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 17ee86781..c55f39df9 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -8,11 +8,14 @@ def __init__( self, num_classes: int, max_num_instances: int, - gaussian_iou: float + gaussian_iou: float, + ): self._num_classes = num_classes self._max_num_instances = max_num_instances self._gaussian_iou = gaussian_iou + self._gaussian_bump = True + self._gaussian_rad = -1 def _generate_heatmap(self, boxes, output_size, input_size): batch_size = tf.shape(boxes)[0] @@ -30,75 +33,74 @@ def _generate_heatmap(self, boxes, output_size, input_size): width_ratio = output_size[1] / input_size[1] height_ratio = output_size[0] / input_size[0] - for tag_ind, detection in enumerate(boxes): - category = int(detection[-1]) - 1 - category = 0 - - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] + for b_ind, boxes_batches in enumerate(boxes): + for tag_ind, detection in enumerate(boxes_batches): + category = int(detection[-1]) - 1 + category = 0 + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + + xct, yct = ( + (detection[2] + detection[0]) / 2, + (detection[3] + detection[1]) / 2 + ) + + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + xtl = int(fxtl) + ytl = int(fytl) + xbr = int(fxbr) + ybr = int(fybr) + xct = int(fxct) + yct = int(fyct) + + if self._gaussian_bump: + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + width = tf.math.ceil(width * width_ratio) + height = tf.math.ceil(height * height_ratio) + + if self._gaussian_rad == -1: + radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) + print(radius) + radius = max(0, int(radius)) + else: + radius = self._gaussian_rad + + # test + # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[b_ind, category], category, [xtl, ytl], radius) + # inputs heatmap, center, radius, k=1 + tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) + br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) + ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) - xct, yct = ( - (detection[2] + detection[0]) / 2, - (detection[3] + detection[1]) / 2 - ) - - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - xtl = int(fxtl) - ytl = int(fytl) - xbr = int(fxbr) - ybr = int(fybr) - xct = int(fxct) - yct = int(fyct) - # temporarily set to true - gaussian_bump=True - gaussian_rad=1 - - if gaussian_bump: - width = detection[2] - detection[0] - height = detection[3] - detection[1] - - width = tf.math.ceil(width * width_ratio) - height = tf.math.ceil(height * height_ratio) - - if gaussian_rad == -1: - radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - radius = max(0, int(radius)) else: - radius = gaussian_rad - - # test - # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[b_ind, category], category, [xtl, ytl], radius) - # inputs heatmap, center, radius, k=1 - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) - - else: - # tl_heatmaps[category, ytl, xtl] = 1 - # br_heatmaps[category, ybr, xbr] = 1 - # ct_heatmaps[category, yct, xct] = 1 - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ytl, xtl]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, ytl, xtl]], [1]) - - # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] - # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] - # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] - # tl_size[tag_ind] = ytl * output_size[1] + xtl - # br_size[tag_ind] = ybr * output_size[1] + xbr - # ct_size[tag_ind] = yct * output_size[1] + xct - tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) - br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) - ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) + # tl_heatmaps[category, ytl, xtl] = 1 + # br_heatmaps[category, ybr, xbr] = 1 + # ct_heatmaps[category, yct, xct] = 1 + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[b_ind, category, ytl, xtl]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[b_ind, category, ytl, xtl]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[b_ind, category, ytl, xtl]], [1]) + + # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] + # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] + # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] + # tl_size[tag_ind] = ytl * output_size[1] + xtl + # br_size[tag_ind] = ybr * output_size[1] + xbr + # ct_size[tag_ind] = yct * output_size[1] + xct + tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[b_ind, tag_ind, 0], [b_ind, tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + br_offset = tf.tensor_scatter_nd_update(br_offset, [[b_ind, tag_ind, 0], [b_ind, tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[b_ind, tag_ind, 0], [b_ind, tag_ind, 1]], [fxct - xct, fyct - yct]) + tl_size = tf.tensor_scatter_nd_update(tl_size, [[b_ind, tag_ind]], [ytl * output_size[1] + xtl]) + br_size = tf.tensor_scatter_nd_update(br_size, [[b_ind, tag_ind]], [ybr * output_size[1] + xbr]) + ct_size = tf.tensor_scatter_nd_update(ct_size, [[b_ind, tag_ind]], [yct * output_size[1] + xct]) labels = { 'tl_size': tl_size, @@ -145,8 +147,8 @@ def generate_heatmaps(self, dectections): detections = [[ (10, 30, 15, 17, 0) ]] - - tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) + + labels = generate_heatmaps(1, 2, (416, 416), detections) pass if __name__ == '__main__': @@ -160,11 +162,12 @@ def generate_heatmaps(self, dectections): # more images can go here if you like ] - tl_heatmaps, br_heatmaps, ct_heatmaps = CenterNetParser(2, 200, 0.7)._generate_heatmap(tf.constant([ - [10, 300, 15, 370, 0], - [100, 300, 150, 370, 0], - [200, 100, 15, 170, 0], - ], dtype=tf.float32), [416, 416], [416, 416]) + labels = CenterNetParser(2, 200, 0.7)._generate_heatmap( + tf.constant(detections, dtype=tf.float32), [416, 416], [416, 416] + ) + tl_heatmaps = labels['tl_heatmaps'] + br_heatmaps = labels['br_heatmaps'] + ct_heatmaps = labels['ct_heatmaps'] # tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) # ct_heatmaps[batch_id, class_id, ...] diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index a79c7b977..3e17a1e8d 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -3,18 +3,22 @@ from yolo.ops import preprocessing_ops +LARGE_NUM = 1. / tf.keras.backend.epsilon() + def _smallest_positive_root(a, b, c) -> tf.Tensor: """ Returns the smallest positive root of a quadratic equation. This implements the fixed version in https://github.com/princeton-vl/CornerNet. """ - discriminant = tf.sqrt(b ** 2 - 4 * a * c) + discriminant = b ** 2 - 4 * a * c + discriminant_sqrt = tf.sqrt(discriminant) - root1 = (-b - discriminant) / (2 * a) - root2 = (-b + discriminant) / (2 * a) + root1 = (-b - discriminant_sqrt) / (2 * a) + root2 = (-b + discriminant_sqrt) / (2 * a) - return (-b + discriminant) / (2) # tf.where(tf.less(root1, 0), root2, root1) + return tf.where(tf.less(discriminant, 0), LARGE_NUM, (-b + discriminant_sqrt) / (2)) + # return tf.where(tf.less(discriminant, 0), LARGE_NUM, tf.where(tf.less(root1, 0), root2, root1)) def gaussian_radius(det_size, min_overlap=0.7) -> int: """ @@ -57,6 +61,7 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: # TODO discuss whether to return scalar or tensor # return tf.reduce_min([r1, r2, r3], axis=0) + print(r1, r2, r3) return tf.reduce_min([r1, r2, r3], axis=0) def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: @@ -75,6 +80,39 @@ def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) return tf.math.exp(exponent) +def cartesian_product(*tensors, repeat=1): + """ + Equivalent of itertools.product except for TensorFlow tensors. + + Example: + cartesian_product(tf.range(3), tf.range(4)) + + array([[0, 0], + [0, 1], + [0, 2], + [0, 3], + [1, 0], + [1, 1], + [1, 2], + [1, 3], + [2, 0], + [2, 1], + [2, 2], + [2, 3]], dtype=int32)> + + Params: + tensors (list[tf.Tensor]): a list of 1D tensors to compute the product of + repeat (int): number of times to repeat the tensors + (https://docs.python.org/3/library/itertools.html#itertools.product) + + Returns: + An nD tensor where n is the number of tensors + """ + tensors = tensors * repeat + print('tensors ',tensors) + return tf.reshape(tf.transpose(tf.stack(tf.meshgrid(*tensors, indexing='ij')), + [*[i + 1 for i in range(len(tensors))], 0]), (-1, len(tensors))) + # scaling_factor doesn't do anything right now def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): """ @@ -85,32 +123,31 @@ def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): radius (int): integer for radius of gaussian scaling_factor (int): scaling factor for gaussian """ - diameter = 2 * radius + 1 gaussian = _gaussian_penalty(radius) x, y = center - height, width = heatmap.shape[0:2] + height, width = tf.shape(heatmap)[-2:] - left, right = min(x, radius), min(width - x, radius + 1) - top, bottom = min(y, radius), min(height - y, radius + 1) + left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) + top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) print('heatmap ',heatmap) print(len(heatmap)) print('category ',category) - # why is this 0? where does 0 come from and is it correct to put it here - heatmap_category = heatmap[0,category, ...] - print('heatmap_category ',heatmap_category) - - # masked_heatmap = heatmap_category[y - top:y + bottom, x - left:x + right] - masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] - masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] # TODO: make sure this replicates original functionality + # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] + # masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * scaling_factor) - return masked_heatmap + + # heatmap_mask = cartesian_product([0], [category], tf.range(y - top, y + bottom), tf.range(x - left, x + right)) + heatmap_mask = cartesian_product([0], [category], tf.range(y - top, y + bottom), tf.range(x - left, x + right)) + masked_gaussian = tf.reshape(gaussian[radius - top:radius + bottom, radius - left:radius + right], (-1,)) + heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, masked_gaussian * scaling_factor) + return heatmap + # def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): # """ # Draws a gaussian heatmap around a center point given a radius. @@ -145,4 +182,3 @@ def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): # # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) # masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * scaling_factor) # return masked_heatmap - From d138373f1bfc7603e70d2b37487b45428fb60c7b Mon Sep 17 00:00:00 2001 From: anivegesana Date: Sat, 20 Mar 2021 15:04:24 -0400 Subject: [PATCH 075/132] Parallel Gaussian penalty --- centernet/dataloaders/centernet_input.py | 11 +++++---- centernet/ops/preprocessing_ops.py | 30 +++++++++++++++++------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index c55f39df9..fcb659b16 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -77,17 +77,18 @@ def _generate_heatmap(self, boxes, output_size, input_size): # test # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[b_ind, category], category, [xtl, ytl], radius) # inputs heatmap, center, radius, k=1 - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, category, [xtl, ytl], radius) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, category, [xbr, ybr], radius) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, category, [xct, yct], radius, scaling_factor=5) + tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [b_ind, category, xtl, ytl, radius]) + br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [b_ind, category, xbr, ybr, radius]) + ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [b_ind, category, xct, yct, radius], scaling_factor=5) else: + # TODO: See if this is a typo # tl_heatmaps[category, ytl, xtl] = 1 # br_heatmaps[category, ybr, xbr] = 1 # ct_heatmaps[category, yct, xct] = 1 tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[b_ind, category, ytl, xtl]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[b_ind, category, ytl, xtl]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[b_ind, category, ytl, xtl]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[b_ind, category, ybr, xbr]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[b_ind, category, yct, xct]], [1]) # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 3e17a1e8d..2357be2c4 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -73,10 +73,13 @@ def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: Returns: tf.Tensor of shape (2 * radius + 1, 2 * radius + 1). """ + radius_shape = tf.shape(radius) width = 2 * radius + 1 - sigma = radius / 3 - x = tf.reshape(tf.range(width, dtype=type) - radius, (width, 1)) - y = tf.reshape(tf.range(width, dtype=type) - radius, (1, width)) + sigma = tf.cast(radius / 3, dtype=type) + range_width = tf.map_fn(lambda limit: tf.range(limit), width) + range_width = tf.cast(range_width - tf.expand_dims(radius, axis=-1), dtype=type) + x = tf.expand_dims(range_width, axis=-1) + y = tf.expand_dims(range_width, axis=-2) exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) return tf.math.exp(exponent) @@ -114,19 +117,30 @@ def cartesian_product(*tensors, repeat=1): [*[i + 1 for i in range(len(tensors))], 0]), (-1, len(tensors))) # scaling_factor doesn't do anything right now -def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): +def draw_gaussian(heatmap, blobs, scaling_factor=1): """ Draws a gaussian heatmap around a center point given a radius. Params: heatmap (tf.Tensor): heatmap placeholder to fill - center (int): integer for center of gaussian - radius (int): integer for radius of gaussian + blobs (tf.Tensor): a tensor whose last dimension is 5 integers for + the batch index, the category of the object, center (x, y), and + for radius of the gaussian scaling_factor (int): scaling factor for gaussian """ + bn_ind, category, x, y, radius = tf.split(blobs, [1, 1, 1, 1, 1], -1) # blobs + bn_ind = bn_ind[0] + category = category[0] + x = x[0] + y = y[0] + # radius not indexed on purpose + diameter = 2 * radius + 1 gaussian = _gaussian_penalty(radius) - x, y = center + # TODO: fix + radius = radius[0] + diameter = diameter[0] + gaussian = gaussian[0] height, width = tf.shape(heatmap)[-2:] @@ -143,7 +157,7 @@ def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) # heatmap_mask = cartesian_product([0], [category], tf.range(y - top, y + bottom), tf.range(x - left, x + right)) - heatmap_mask = cartesian_product([0], [category], tf.range(y - top, y + bottom), tf.range(x - left, x + right)) + heatmap_mask = cartesian_product([bn_ind], [category], tf.range(y - top, y + bottom), tf.range(x - left, x + right)) masked_gaussian = tf.reshape(gaussian[radius - top:radius + bottom, radius - left:radius + right], (-1,)) heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, masked_gaussian * scaling_factor) return heatmap From 4bf47ca63d64fed5d16fcd5706709be42686b61e Mon Sep 17 00:00:00 2001 From: anivegesana Date: Sat, 20 Mar 2021 20:19:38 -0400 Subject: [PATCH 076/132] Make code ugly to make it work better in TF --- centernet/dataloaders/centernet_input.py | 26 ++++---- centernet/ops/preprocessing_ops.py | 82 ++++++++++++++++-------- 2 files changed, 70 insertions(+), 38 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index fcb659b16..0b5460f86 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -35,8 +35,8 @@ def _generate_heatmap(self, boxes, output_size, input_size): for b_ind, boxes_batches in enumerate(boxes): for tag_ind, detection in enumerate(boxes_batches): - category = int(detection[-1]) - 1 - category = 0 + category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary + category = 0 # FIXME: For testing only xtl, ytl = detection[0], detection[1] xbr, ybr = detection[2], detection[3] @@ -53,12 +53,12 @@ def _generate_heatmap(self, boxes, output_size, input_size): fxct = (xct * width_ratio) fyct = (yct * height_ratio) - xtl = int(fxtl) - ytl = int(fytl) - xbr = int(fxbr) - ybr = int(fybr) - xct = int(fxct) - yct = int(fyct) + xtl = tf.math.floor(fxtl) + ytl = tf.math.floor(fytl) + xbr = tf.math.floor(fxbr) + ybr = tf.math.floor(fybr) + xct = tf.math.floor(fxct) + yct = tf.math.floor(fyct) if self._gaussian_bump: width = detection[2] - detection[0] @@ -69,17 +69,16 @@ def _generate_heatmap(self, boxes, output_size, input_size): if self._gaussian_rad == -1: radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - print(radius) - radius = max(0, int(radius)) + radius = tf.math.maximum(0, tf.math.floor(radius)) else: radius = self._gaussian_rad # test # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[b_ind, category], category, [xtl, ytl], radius) # inputs heatmap, center, radius, k=1 - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [b_ind, category, xtl, ytl, radius]) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [b_ind, category, xbr, ybr, radius]) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [b_ind, category, xct, yct, radius], scaling_factor=5) + tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[b_ind, category, xtl, ytl, radius]]) + br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[b_ind, category, xbr, ybr, radius]]) + ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[b_ind, category, xct, yct, radius]], scaling_factor=5) else: # TODO: See if this is a typo @@ -163,6 +162,7 @@ def generate_heatmaps(self, dectections): # more images can go here if you like ] + #labels = tf.function(CenterNetParser(2, 200, 0.7)._generate_heatmap)( labels = CenterNetParser(2, 200, 0.7)._generate_heatmap( tf.constant(detections, dtype=tf.float32), [416, 416], [416, 416] ) diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 2357be2c4..b81c640ce 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -5,6 +5,7 @@ LARGE_NUM = 1. / tf.keras.backend.epsilon() +@tf.function def _smallest_positive_root(a, b, c) -> tf.Tensor: """ Returns the smallest positive root of a quadratic equation. @@ -20,6 +21,7 @@ def _smallest_positive_root(a, b, c) -> tf.Tensor: return tf.where(tf.less(discriminant, 0), LARGE_NUM, (-b + discriminant_sqrt) / (2)) # return tf.where(tf.less(discriminant, 0), LARGE_NUM, tf.where(tf.less(root1, 0), root2, root1)) +@tf.function def gaussian_radius(det_size, min_overlap=0.7) -> int: """ Given a bounding box size, returns a lower bound on how far apart the @@ -64,7 +66,8 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: print(r1, r2, r3) return tf.reduce_min([r1, r2, r3], axis=0) -def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: +@tf.function +def _gaussian_penalty(radius: int, dtype=tf.float32) -> tf.Tensor: """ This represents the penalty reduction around a point. Params: @@ -73,16 +76,19 @@ def _gaussian_penalty(radius: int, type=tf.float32) -> tf.Tensor: Returns: tf.Tensor of shape (2 * radius + 1, 2 * radius + 1). """ - radius_shape = tf.shape(radius) width = 2 * radius + 1 - sigma = tf.cast(radius / 3, dtype=type) - range_width = tf.map_fn(lambda limit: tf.range(limit), width) - range_width = tf.cast(range_width - tf.expand_dims(radius, axis=-1), dtype=type) + sigma = tf.cast(radius / 3, dtype=dtype) + + range_width = tf.range(width) + range_width = tf.cast(range_width - tf.expand_dims(radius, axis=-1), dtype=dtype) + x = tf.expand_dims(range_width, axis=-1) y = tf.expand_dims(range_width, axis=-2) - exponent = (-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2) + + exponent = ((-1 * (x ** 2) - (y ** 2)) / (2 * sigma ** 2)) return tf.math.exp(exponent) +@tf.function def cartesian_product(*tensors, repeat=1): """ Equivalent of itertools.product except for TensorFlow tensors. @@ -112,12 +118,18 @@ def cartesian_product(*tensors, repeat=1): An nD tensor where n is the number of tensors """ tensors = tensors * repeat - print('tensors ',tensors) return tf.reshape(tf.transpose(tf.stack(tf.meshgrid(*tensors, indexing='ij')), [*[i + 1 for i in range(len(tensors))], 0]), (-1, len(tensors))) +@tf.function +def write_all(ta, index, values): + for i in range(tf.shape(values)[0]): + ta = ta.write(index + i, values[i, ...]) + return ta, index + i + # scaling_factor doesn't do anything right now -def draw_gaussian(heatmap, blobs, scaling_factor=1): +@tf.function +def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): """ Draws a gaussian heatmap around a center point given a radius. Params: @@ -127,22 +139,18 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1): for radius of the gaussian scaling_factor (int): scaling factor for gaussian """ - bn_ind, category, x, y, radius = tf.split(blobs, [1, 1, 1, 1, 1], -1) # blobs - bn_ind = bn_ind[0] - category = category[0] - x = x[0] - y = y[0] - # radius not indexed on purpose + blobs = tf.cast(blobs, tf.int32) + bn_ind = blobs[..., 0] + category = blobs[..., 1] + x = blobs[..., 2] + y = blobs[..., 3] + radius = blobs[..., 4] + num_boxes = tf.shape(radius)[0] diameter = 2 * radius + 1 - gaussian = _gaussian_penalty(radius) - - # TODO: fix - radius = radius[0] - diameter = diameter[0] - gaussian = gaussian[0] - height, width = tf.shape(heatmap)[-2:] + heatmap_shape = tf.shape(heatmap) + height, width = heatmap_shape[-2], heatmap_shape[-1] left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) @@ -155,10 +163,34 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1): # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] # masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - - # heatmap_mask = cartesian_product([0], [category], tf.range(y - top, y + bottom), tf.range(x - left, x + right)) - heatmap_mask = cartesian_product([bn_ind], [category], tf.range(y - top, y + bottom), tf.range(x - left, x + right)) - masked_gaussian = tf.reshape(gaussian[radius - top:radius + bottom, radius - left:radius + right], (-1,)) + update_count = tf.reduce_sum((bottom + top) * (right + left)) + masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) + heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((4,)), size=update_count) + i = 0 + for j in range(num_boxes): + bn_i = bn_ind[j] + cat = category[j] + X = x[j] + Y = y[j] + R = radius[j] + l = left[j] + r = right[j] + t = top[j] + b = bottom[j] + + gaussian = _gaussian_penalty(R, dtype=dtype) + masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) + heatmap_mask_instance = cartesian_product([bn_i], [cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) + masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) + heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) + masked_gaussian = masked_gaussian_ta.concat() + heatmap_mask = heatmap_mask_ta.concat() + print(masked_gaussian, heatmap_mask) + heatmap_mask = tf.reshape(heatmap_mask, (-1, 4)) + masked_gaussian_ta.close() + heatmap_mask_ta.close() + # masked_gaussian = tf.concat(masked_gaussian, axis = 0) + # heatmap_mask = tf.concat(heatmap_mask, axis = 0) heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, masked_gaussian * scaling_factor) return heatmap From 8b9627ce6bf432eb3c840e97d607c3289c746508 Mon Sep 17 00:00:00 2001 From: anivegesana Date: Tue, 23 Mar 2021 10:45:20 -0400 Subject: [PATCH 077/132] Remove batches --- centernet/dataloaders/centernet_input.py | 160 +++++++++++------------ centernet/ops/preprocessing_ops.py | 37 ++---- 2 files changed, 92 insertions(+), 105 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 0b5460f86..1758e0496 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -18,89 +18,89 @@ def __init__( self._gaussian_rad = -1 def _generate_heatmap(self, boxes, output_size, input_size): - batch_size = tf.shape(boxes)[0] - tl_heatmaps = tf.zeros((batch_size, self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - br_heatmaps = tf.zeros((batch_size, self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - ct_heatmaps = tf.zeros((batch_size, self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - tl_offset = tf.zeros((batch_size, self._max_num_instances, 2), dtype=tf.float32) - br_offset = tf.zeros((batch_size, self._max_num_instances, 2), dtype=tf.float32) - ct_offset = tf.zeros((batch_size, self._max_num_instances, 2), dtype=tf.float32) - tl_size = tf.zeros((batch_size, self._max_num_instances), dtype=tf.int64) - br_size = tf.zeros((batch_size, self._max_num_instances), dtype=tf.int64) - ct_size = tf.zeros((batch_size, self._max_num_instances), dtype=tf.int64) - tag_masks = tf.zeros((batch_size, self._max_num_instances), dtype=tf.uint8) + boxes = tf.cast(boxes, dtype=tf.float32) + + tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) width_ratio = output_size[1] / input_size[1] height_ratio = output_size[0] / input_size[0] - for b_ind, boxes_batches in enumerate(boxes): - for tag_ind, detection in enumerate(boxes_batches): - category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary - category = 0 # FIXME: For testing only - - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] - - xct, yct = ( - (detection[2] + detection[0]) / 2, - (detection[3] + detection[1]) / 2 - ) - - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - xtl = tf.math.floor(fxtl) - ytl = tf.math.floor(fytl) - xbr = tf.math.floor(fxbr) - ybr = tf.math.floor(fybr) - xct = tf.math.floor(fxct) - yct = tf.math.floor(fyct) - - if self._gaussian_bump: - width = detection[2] - detection[0] - height = detection[3] - detection[1] - - width = tf.math.ceil(width * width_ratio) - height = tf.math.ceil(height * height_ratio) - - if self._gaussian_rad == -1: - radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - radius = tf.math.maximum(0, tf.math.floor(radius)) - else: - radius = self._gaussian_rad - - # test - # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[b_ind, category], category, [xtl, ytl], radius) - # inputs heatmap, center, radius, k=1 - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[b_ind, category, xtl, ytl, radius]]) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[b_ind, category, xbr, ybr, radius]]) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[b_ind, category, xct, yct, radius]], scaling_factor=5) + for tag_ind, detection in enumerate(boxes): + category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary + category = 0 # FIXME: For testing only + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + + xct, yct = ( + (detection[2] + detection[0]) / 2, + (detection[3] + detection[1]) / 2 + ) + + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + xtl = tf.math.floor(fxtl) + ytl = tf.math.floor(fytl) + xbr = tf.math.floor(fxbr) + ybr = tf.math.floor(fybr) + xct = tf.math.floor(fxct) + yct = tf.math.floor(fyct) + + if self._gaussian_bump: + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + width = tf.math.ceil(width * width_ratio) + height = tf.math.ceil(height * height_ratio) + + if self._gaussian_rad == -1: + radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) + radius = tf.math.maximum(0, tf.math.floor(radius)) else: - # TODO: See if this is a typo - # tl_heatmaps[category, ytl, xtl] = 1 - # br_heatmaps[category, ybr, xbr] = 1 - # ct_heatmaps[category, yct, xct] = 1 - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[b_ind, category, ytl, xtl]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[b_ind, category, ybr, xbr]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[b_ind, category, yct, xct]], [1]) - - # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] - # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] - # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] - # tl_size[tag_ind] = ytl * output_size[1] + xtl - # br_size[tag_ind] = ybr * output_size[1] + xbr - # ct_size[tag_ind] = yct * output_size[1] + xct - tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[b_ind, tag_ind, 0], [b_ind, tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - br_offset = tf.tensor_scatter_nd_update(br_offset, [[b_ind, tag_ind, 0], [b_ind, tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[b_ind, tag_ind, 0], [b_ind, tag_ind, 1]], [fxct - xct, fyct - yct]) - tl_size = tf.tensor_scatter_nd_update(tl_size, [[b_ind, tag_ind]], [ytl * output_size[1] + xtl]) - br_size = tf.tensor_scatter_nd_update(br_size, [[b_ind, tag_ind]], [ybr * output_size[1] + xbr]) - ct_size = tf.tensor_scatter_nd_update(ct_size, [[b_ind, tag_ind]], [yct * output_size[1] + xct]) + radius = self._gaussian_rad + + # test + # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[category], category, [xtl, ytl], radius) + # inputs heatmap, center, radius, k=1 + tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[category, xtl, ytl, radius]]) + br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[category, xbr, ybr, radius]]) + ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[category, xct, yct, radius]], scaling_factor=5) + + else: + # TODO: See if this is a typo + # tl_heatmaps[category, ytl, xtl] = 1 + # br_heatmaps[category, ybr, xbr] = 1 + # ct_heatmaps[category, yct, xct] = 1 + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ybr, xbr]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, yct, xct]], [1]) + + # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] + # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] + # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] + # tl_size[tag_ind] = ytl * output_size[1] + xtl + # br_size[tag_ind] = ybr * output_size[1] + xbr + # ct_size[tag_ind] = yct * output_size[1] + xct + tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) + tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) + br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) + ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) labels = { 'tl_size': tl_size, @@ -154,12 +154,10 @@ def generate_heatmaps(self, dectections): if __name__ == '__main__': # This code is for visualization import matplotlib.pyplot as plt - detections = [[ + detections = [ (10, 300, 15, 370, 0), (100, 300, 150, 370, 0), (200, 100, 15, 170, 0), - ], - # more images can go here if you like ] #labels = tf.function(CenterNetParser(2, 200, 0.7)._generate_heatmap)( @@ -172,7 +170,7 @@ def generate_heatmaps(self, dectections): # tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) # ct_heatmaps[batch_id, class_id, ...] - plt.imshow(ct_heatmaps[0, 0, ...]) + plt.imshow(ct_heatmaps[0, ...]) plt.show() # This is to run the test # tf.test.main() diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index b81c640ce..784ab3e53 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -5,7 +5,6 @@ LARGE_NUM = 1. / tf.keras.backend.epsilon() -@tf.function def _smallest_positive_root(a, b, c) -> tf.Tensor: """ Returns the smallest positive root of a quadratic equation. @@ -21,7 +20,6 @@ def _smallest_positive_root(a, b, c) -> tf.Tensor: return tf.where(tf.less(discriminant, 0), LARGE_NUM, (-b + discriminant_sqrt) / (2)) # return tf.where(tf.less(discriminant, 0), LARGE_NUM, tf.where(tf.less(root1, 0), root2, root1)) -@tf.function def gaussian_radius(det_size, min_overlap=0.7) -> int: """ Given a bounding box size, returns a lower bound on how far apart the @@ -66,7 +64,6 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: print(r1, r2, r3) return tf.reduce_min([r1, r2, r3], axis=0) -@tf.function def _gaussian_penalty(radius: int, dtype=tf.float32) -> tf.Tensor: """ This represents the penalty reduction around a point. @@ -121,30 +118,27 @@ def cartesian_product(*tensors, repeat=1): return tf.reshape(tf.transpose(tf.stack(tf.meshgrid(*tensors, indexing='ij')), [*[i + 1 for i in range(len(tensors))], 0]), (-1, len(tensors))) -@tf.function def write_all(ta, index, values): for i in range(tf.shape(values)[0]): ta = ta.write(index + i, values[i, ...]) return ta, index + i # scaling_factor doesn't do anything right now -@tf.function def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): """ Draws a gaussian heatmap around a center point given a radius. Params: heatmap (tf.Tensor): heatmap placeholder to fill - blobs (tf.Tensor): a tensor whose last dimension is 5 integers for - the batch index, the category of the object, center (x, y), and - for radius of the gaussian + blobs (tf.Tensor): a tensor whose last dimension is 4 integers for + the category of the object, center (x, y), and for radius of the + gaussian scaling_factor (int): scaling factor for gaussian """ blobs = tf.cast(blobs, tf.int32) - bn_ind = blobs[..., 0] - category = blobs[..., 1] - x = blobs[..., 2] - y = blobs[..., 3] - radius = blobs[..., 4] + category = blobs[..., 0] + x = blobs[..., 1] + y = blobs[..., 2] + radius = blobs[..., 3] num_boxes = tf.shape(radius)[0] diameter = 2 * radius + 1 @@ -165,10 +159,9 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) update_count = tf.reduce_sum((bottom + top) * (right + left)) masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) - heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((4,)), size=update_count) + heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((3,)), size=update_count) i = 0 for j in range(num_boxes): - bn_i = bn_ind[j] cat = category[j] X = x[j] Y = y[j] @@ -180,18 +173,14 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): gaussian = _gaussian_penalty(R, dtype=dtype) masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) - heatmap_mask_instance = cartesian_product([bn_i], [cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) + heatmap_mask_instance = cartesian_product([cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) - masked_gaussian = masked_gaussian_ta.concat() - heatmap_mask = heatmap_mask_ta.concat() - print(masked_gaussian, heatmap_mask) - heatmap_mask = tf.reshape(heatmap_mask, (-1, 4)) - masked_gaussian_ta.close() - heatmap_mask_ta.close() - # masked_gaussian = tf.concat(masked_gaussian, axis = 0) - # heatmap_mask = tf.concat(heatmap_mask, axis = 0) + masked_gaussian = masked_gaussian_ta.stack() + heatmap_mask = heatmap_mask_ta.stack() + heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, masked_gaussian * scaling_factor) + print('after ',heatmap) return heatmap # def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): From 72248ea95915aaf52151449f3b0cac452af1bb41 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 29 Mar 2021 12:38:09 -0400 Subject: [PATCH 078/132] working on testing validation --- centernet/configs/centernet.py | 3 + .../modeling/layers/detection_generator.py | 2 +- centernet/modeling/plot_models.py | 42 + centernet/tasks/centernet.py | 93 +- centernet/tasks/centernet_test.py | 55 +- .../utils/weight_utils/EXTREME_NET_VARS.txt | 1338 +++++++++++++++++ centernet/utils/weight_utils/config_data.py | 18 + centernet/utils/weight_utils/load_weights.py | 2 +- .../utils/weight_utils/test_load_weights.py | 45 +- 9 files changed, 1568 insertions(+), 30 deletions(-) create mode 100644 centernet/modeling/plot_models.py create mode 100644 centernet/utils/weight_utils/EXTREME_NET_VARS.txt diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index b94bcffcd..d6725e3c3 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -87,6 +87,9 @@ class CenterNetTask(cfg.TaskConfig): subtasks: CenterNetSubTasks = CenterNetSubTasks() losses: Losses = Losses() + annotation_file: Optional[str] = None + per_category_metrics: bool = False + weight_decay: float = 5e-4 def _get_output_length_dict(self): diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index cb284e44f..bd90682ae 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -15,7 +15,7 @@ def __init__(self, max_detections=100, peak_error=1e-6, peak_extract_kernel_size=3, - use_nms=True, + use_nms=False, center_thresh=0.1, iou_thresh=0.4, class_offset=1, diff --git a/centernet/modeling/plot_models.py b/centernet/modeling/plot_models.py new file mode 100644 index 000000000..a981cbc48 --- /dev/null +++ b/centernet/modeling/plot_models.py @@ -0,0 +1,42 @@ +import tensorflow as tf + +from centernet.configs import centernet as cfg +from centernet.modeling.backbones import hourglass +from centernet.modeling.layers import nn_blocks +from centernet.modeling.decoders import centernet_decoder +from centernet.modeling.CenterNet import build_centernet + + +BACKBONE_OUT = "./centernet/modeling/backbone.png" +DECODER_OUT = "./centernet/modeling/decoder.png" +MODEL_OUT = "./centernet/modeling/model.png" + + + +if __name__ == '__main__': + dims = [256, 256, 384, 384, 384, 512] + modules = [2, 2, 2, 2, 2, 4] + + backbone = hourglass.build_hourglass( + tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]), cfg.CenterNetBase()) + + hg_block = nn_blocks.HourglassBlock(dims, modules) + hg_block.build((1, 512, 512, 256)) + + decoder = centernet_decoder.build_centernet_decoder( + task_config=cfg.CenterNetTask(), + input_specs=(None, 128, 128, 256)) + + input_specs = tf.keras.layers.InputSpec(shape=[None, 512, 512, 3]) + + config = cfg.CenterNetTask() + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + + tf.keras.utils.plot_model(backbone, to_file=BACKBONE_OUT, show_shapes=True, dpi=300) + tf.keras.utils.plot_model(decoder, to_file=DECODER_OUT, show_shapes=True, dpi=300) + + hg_block.summary() + model.summary() + + \ No newline at end of file diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index dfa49ef9b..a3f82610e 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -43,18 +43,105 @@ def build_model(self): self._loss_dict = losses return model - # Everything below was from YOLO def build_inputs(self, params, input_context=None): pass def build_losses(self, outputs, labels, aux_losses=None): - pass + total_loss = 0.0 + total_scale_loss = 0.0 + total_offset_loss = 0.0 + loss = 0.0 + scale_loss = 0.0 + offset_loss = 0.0 + + metric_dict = dict() + + # TODO: Calculate loss + flattened_ct_heatmaps = utils._flatten_spatial_dimensions(labels['ct_heatmaps']) + num_boxes = utils._to_float32(utils.get_num_instances_from_weights(labels['tag_masks'])) #gt_weights_list here shouldn't be tag_masks here + + object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss() + # Loop through each feature output head. + for pred in outputs['ct_heatmaps']: + pred = utils._flatten_spatial_dimensions(pred) + total_loss += object_center_loss( + flattened_ct_heatmaps, pred) #removed weight parameter (weight = per_pixel_weight) + center_loss = tf.reduce_sum(total_loss) / ( + float(len(outputs['ct_heatmaps'])) * num_boxes) + loss += center_loss + metric_dict['ct_loss'] = center_loss + + #localization loss for offset and scale loss + localization_loss_fn = l1_localization_loss.L1LocalizationLoss() + for scale_pred, offset_pred in zip(outputs['ct_size'], outputs['ct_offset']): + # Compute the scale loss. + scale_pred = utils.get_batch_predictions_from_indices( + scale_pred, labels['tag_locs']) + total_scale_loss += localization_loss_fn( + labels['ct_size'], scale_pred) #removed weights=batch_weights + # Compute the offset loss. + offset_pred = utils.get_batch_predictions_from_indices( + offset_pred, labels['tag_locs']) + total_offset_loss += localization_loss_fn( + labels['ct_offset'], offset_pred) #removed weights=batch_weights + scale_loss += tf.reduce_sum(total_scale_loss) / ( + float(len(outputs['ct_size'])) * num_boxes) + offset_loss += tf.reduce_sum(total_offset_loss) / ( + float(len(outputs['ct_size'])) * num_boxes) + metric_dict['ct_scale_loss'] = scale_loss + metric_dict['ct_offset_loss'] = offset_loss + + return loss, metric_dict def build_metrics(self, training=True): - pass + metrics = [] + metric_names = self._metric_names + + for name in metric_names: + metrics.append(tf.keras.metrics.Mean(name, dtype=tf.float32)) + + self._metrics = metrics + if not training: + self.coco_metric = coco_evaluator.COCOEvaluator( + annotation_file=self.task_config.annotation_file, + include_mask=False, + need_rescale_bboxes=False, + per_category_metrics=self._task_config.per_category_metrics) + return metrics def train_step(self, inputs, model, optimizer, metrics=None): pass + + def validation_step(self, inputs, model, metrics=None): + # get the data point + image, label = inputs + + # computer detivative and apply gradients + y_pred = model(image, training=False) + y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred['raw_output']) + loss_metrics = self.build_losses(y_pred, label) + logs = {self.loss: loss_metrics['total_loss']} + + coco_model_outputs = { + 'detection_boxes': + box_ops.denormalize_boxes( + tf.cast(y_pred['bbox'], tf.float32), image_shape), + 'detection_scores': + y_pred['confidence'], + 'detection_classes': + y_pred['classes'], + 'num_detections': + tf.shape(y_pred['bbox'])[:-1], + } + + logs.update({self.coco_metric.name: (label, coco_model_outputs)}) + + if metrics: + for m in metrics: + m.update_state(loss_metrics[m.name]) + logs.update({m.name: m.result()}) + + return logs def aggregate_logs(self, state=None, step_outputs=None): pass diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index b0d4e7fad..464a2beda 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -3,30 +3,69 @@ import numpy as np import dataclasses +import orbit + from official.modeling import hyperparams from official.vision.beta.configs import backbones from centernet.tasks.centernet import CenterNetTask from centernet.configs import centernet as exp_cfg +from centernet.utils.weight_utils.load_weights import get_model_weights_as_dict, load_weights_model + +import tensorflow_datasets as tfds +CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): - def testCenterNetTask(self): + # def testCenterNetTask(self): + # model_config = exp_cfg.CenterNet(input_size=[512, 512, 3]) + # config = exp_cfg.CenterNetTask(model=model_config) + # task = CenterNetTask(config) + + # model = task.build_model() + # outputs = model(tf.zeros((3, 512, 512, 3))) + + # self.assertEqual(len(outputs['raw_output']), 3) + # self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (3, 128, 128, 90)) + # self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (3, 128, 128, 2)) + # self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (3, 128, 128, 2)) + + # model.summary() + + def testCenterNetValidation(self): model_config = exp_cfg.CenterNet(input_size=[512, 512, 3]) config = exp_cfg.CenterNetTask(model=model_config) + task = CenterNetTask(config) - model = task.build_model() - outputs = model(tf.zeros((3, 512, 512, 3))) + metrics = task.build_metrics(training=False) - self.assertEqual(len(outputs['raw_output']), 3) - self.assertEqual(outputs['raw_output']['ct_heatmaps'][0].shape, (3, 128, 128, 90)) - self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (3, 128, 128, 2)) - self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (3, 128, 128, 2)) + weights_dict, _ = get_model_weights_as_dict(CENTERNET_CKPT_PATH) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') - model.summary() + data_dir = 'D:\Datasets' + ds, ds_info = tfds.load(name='coco/2017', + split='validation', + shuffle_files=True, + data_dir=data_dir, + with_info=True, + download=True) + strategy = tf.distribute.get_strategy() + dataset = orbit.utils.make_distributed_dataset(strategy, ds) + + iterator = iter(dataset) + print(next(iterator)) + # logs = task.validation_step(next(iterator), model, metrics=metrics) + # print(logs) if __name__ == '__main__': tf.test.main() + + + + + + + diff --git a/centernet/utils/weight_utils/EXTREME_NET_VARS.txt b/centernet/utils/weight_utils/EXTREME_NET_VARS.txt new file mode 100644 index 000000000..144f2c156 --- /dev/null +++ b/centernet/utils/weight_utils/EXTREME_NET_VARS.txt @@ -0,0 +1,1338 @@ +feature_extractor + hourglass_network + 0 + encoder_block1 + 0 + norm + moving_mean + moving_variance + beta + gamma + conv_block + norm + gamma + moving_mean + beta + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + norm + beta + gamma + moving_variance + moving_mean + conv + kernel + inner_block + 0 + decoder_block + 1 + skip + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + beta + gamma + moving_mean + moving_variance + inner_block + 0 + inner_block + 0 + encoder_block2 + 1 + conv_block + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + norm + beta + moving_mean + gamma + moving_variance + skip + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + inner_block + 0 + decoder_block + 0 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + skip + conv + kernel + norm + gamma + beta + moving_variance + moving_mean + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + encoder_block1 + 0 + conv_block + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + norm + gamma + moving_mean + beta + moving_variance + conv + kernel + encoder_block2 + 0 + conv_block + conv + kernel + norm + moving_mean + moving_variance + beta + gamma + norm + gamma + moving_mean + moving_variance + beta + conv + kernel + skip + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + 1 + conv_block + norm + moving_variance + gamma + moving_mean + beta + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + inner_block + 3 + conv_block + norm + gamma + beta + moving_variance + moving_mean + conv + kernel + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + 2 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + 1 + norm + moving_variance + beta + gamma + moving_mean + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + 0 + conv_block + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + decoder_block + 0 + conv_block + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + 1 + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv_block + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + encoder_block1 + 0 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + 1 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block1 + 0 + norm + gamma + beta + moving_mean + moving_variance + conv_block + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + conv + kernel + 1 + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + gamma + beta + moving_mean + moving_variance + encoder_block2 + 0 + conv_block + norm + gamma + beta + moving_variance + moving_mean + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + skip + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + decoder_block + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + norm + gamma + moving_mean + beta + moving_variance + conv + kernel + 0 + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block2 + 0 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + beta + gamma + moving_mean + moving_variance + 1 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block1 + 1 + norm + moving_variance + moving_mean + beta + gamma + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + 0 + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block2 + 0 + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + norm + moving_mean + beta + moving_variance + gamma + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + decoder_block + 1 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + inner_block + 0 + inner_block + 0 + inner_block + 0 + inner_block + 0 + encoder_block1 + 0 + conv_block + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + 1 + conv + kernel + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + encoder_block2 + 1 + conv + kernel + conv_block + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + skip + norm + moving_mean + moving_variance + beta + gamma + conv + kernel + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + decoder_block + 1 + skip + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + norm + moving_variance + moving_mean + beta + gamma + conv_block + conv + kernel + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + 0 + conv_block + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + inner_block + 3 + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + norm + moving_mean + moving_variance + beta + gamma + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + 2 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + encoder_block2 + 0 + norm + moving_variance + gamma + beta + moving_mean + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + skip + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + norm + moving_mean + moving_variance + beta + gamma + conv + kernel + encoder_block1 + 0 + norm + beta + gamma + moving_mean + moving_variance + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + 1 + conv_block + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + decoder_block + 0 + conv_block + norm + moving_variance + beta + moving_mean + gamma + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + encoder_block1 + 0 + norm + moving_mean + beta + gamma + moving_variance + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + 1 + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block2 + 0 + conv_block + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + conv + kernel + norm + moving_variance + gamma + moving_mean + beta + skip + norm + gamma + moving_mean + moving_variance + beta + conv + kernel + 1 + conv_block + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + decoder_block + 0 + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + 1 + conv_block + norm + moving_mean + moving_variance + beta + gamma + conv + kernel + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + encoder_block1 + 1 + conv + kernel + conv_block + norm + gamma + moving_mean + beta + moving_variance + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + conv + kernel + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + encoder_block2 + 1 + conv_block + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + 0 + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + conv + kernel + skip + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + decoder_block + 0 + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + gamma + beta + moving_variance + moving_mean + conv + kernel + 1 + conv + kernel + norm + gamma + beta + moving_variance + moving_mean + skip + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv_block + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + encoder_block1 + 1 + conv + kernel + conv_block + norm + moving_variance + moving_mean + beta + gamma + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + 0 + conv_block + norm + beta + moving_variance + moving_mean + gamma + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + encoder_block2 + 0 + skip + norm + moving_variance + gamma + beta + moving_mean + conv + kernel + conv_block + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + norm + beta + moving_variance + gamma + moving_mean + conv + kernel + 1 + conv_block + norm + beta + moving_mean + gamma + moving_variance + conv + kernel + conv + kernel + norm + gamma + beta + moving_mean + moving_variance + decoder_block + 1 + conv_block + norm + moving_variance + beta + gamma + moving_mean + conv + kernel + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + 0 + conv_block + norm + gamma + beta + moving_mean + moving_variance + conv + kernel + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + downsample_input + residual_block + conv_block + conv + kernel + norm + beta + gamma + moving_mean + moving_variance + norm + moving_variance + moving_mean + beta + gamma + skip + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + conv + kernel + conv_block + conv + kernel + norm + moving_variance + beta + gamma + moving_mean + intermediate_conv2 + 0 + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + output_conv + 1 + conv + kernel + norm + moving_variance + moving_mean + gamma + beta + 0 + norm + gamma + moving_variance + beta + moving_mean + conv + kernel + intermediate_residual + 0 + conv_block + norm + moving_mean + gamma + beta + moving_variance + conv + kernel + norm + moving_mean + beta + gamma + moving_variance + conv + kernel + intermediate_conv1 + 0 + conv + kernel + norm + beta + gamma + moving_mean + moving_variance +save_counter diff --git a/centernet/utils/weight_utils/config_data.py b/centernet/utils/weight_utils/config_data.py index 764091b15..081a5dfd0 100644 --- a/centernet/utils/weight_utils/config_data.py +++ b/centernet/utils/weight_utils/config_data.py @@ -24,9 +24,27 @@ def __post_init__(self): convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), ] + self.extremenet = [ + # Downsampling Layers + convBnCFG(weights_dict=self.weights_dict['downsample_input']['conv_block']), + residualBlockCFG(weights_dict=self.weights_dict['downsample_input']['residual_block']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['0']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['0']), + # Intermediate + convBnCFG(weights_dict=self.weights_dict['intermediate_conv1']['0']), + convBnCFG(weights_dict=self.weights_dict['intermediate_conv2']['0']), + residualBlockCFG(weights_dict=self.weights_dict['intermediate_residual']['0']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['1']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), + ] + def get_cfg_list(self, name): if name == 'hourglass104_512': return self.hourglass104_512 + elif name == 'extremenet': + return self.extremenet @dataclass class DecoderConfigData(): diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index d431f75f6..e9947bf34 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -93,7 +93,7 @@ def get_decoder_layer_cfgs(weights_dict, decoder_name): weights_dict: Dictionary that stores the decoder model weights decoder_name: String, indicating the desired decoder configuration Returns: - A list containing the config classe of the backbone building block + A list containing the config classes of the backbone building block """ print("Fetching decoder config classes\n") diff --git a/centernet/utils/weight_utils/test_load_weights.py b/centernet/utils/weight_utils/test_load_weights.py index fbf3873c5..4f7b79776 100644 --- a/centernet/utils/weight_utils/test_load_weights.py +++ b/centernet/utils/weight_utils/test_load_weights.py @@ -5,10 +5,12 @@ from centernet.configs.centernet import CenterNetTask from centernet.utils.weight_utils.load_weights import get_model_weights_as_dict, load_weights_model +from centernet.utils.weight_utils.load_weights import load_weights_backbone import tensorflow as tf -CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' +CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' +EXTREMENET_CKPT_PATH = 'D:\\weights\extremenet' if __name__ == '__main__': input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) @@ -17,21 +19,30 @@ model, loss = build_centernet(input_specs=input_specs, task_config=config, l2_regularization=0) - weights_dict, n_weights = get_model_weights_as_dict(CKPT_PATH) - load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') - - cap = FastVideo( - 0, - model=model, - process_width=512, - process_height=512, - preprocess_with_gpu=False, - classes=91, - print_conf=True, - max_batch=1, - disp_h=512, - scale_que=1, - wait_time='dynamic') - cap.run() + + # Test for running validation step on pretrained model + + + # # Test for loading extremenet backbone weights + # extreme_net_weights_dict, _ = get_model_weights_as_dict(EXTREMENET_CKPT_PATH) + # load_weights_backbone(model.backbone, extreme_net_weights_dict['feature_extractor'], backbone_name='extremenet') + + # # Test for loading ODAPI weights and running inference using webcam + # weights_dict, _ = get_model_weights_as_dict(CENTERNET_CKPT_PATH) + # load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + + # cap = FastVideo( + # 0, + # model=model, + # process_width=512, + # process_height=512, + # preprocess_with_gpu=False, + # classes=91, + # print_conf=True, + # max_batch=1, + # disp_h=512, + # scale_que=1, + # wait_time='dynamic') + # cap.run() # runner(model, 0, 512, 512) From b24f93240d5cd220f0d89d881ae1c2010419cc01 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 31 Mar 2021 23:57:08 -0400 Subject: [PATCH 079/132] build inputs --- centernet/configs/centernet.py | 57 ++++ centernet/dataloaders/centernet_input.py | 315 +++++++++++++---------- centernet/ops/preprocessing_ops.py | 10 +- centernet/tasks/centernet.py | 61 +++-- centernet/tasks/centernet_test.py | 32 +-- yolo/ops/preprocessing_ops.py | 34 ++- 6 files changed, 303 insertions(+), 206 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index d6725e3c3..36e3dafaa 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -24,6 +24,59 @@ from official.vision.beta.configs import common from centernet.configs import backbones + +@dataclasses.dataclass +class TfExampleDecoder(hyperparams.Config): + regenerate_source_id: bool = False + +@dataclasses.dataclass +class TfExampleDecoderLabelMap(hyperparams.Config): + regenerate_source_id: bool = False + label_map: str = '' + +@dataclasses.dataclass +class DataDecoder(hyperparams.OneOfConfig): + type: Optional[str] = 'simple_decoder' + simple_decoder: TfExampleDecoder = TfExampleDecoder() + label_map_decoder: TfExampleDecoderLabelMap = TfExampleDecoderLabelMap() + +# dataset parsers +@dataclasses.dataclass +class Parser(hyperparams.Config): + image_w: int = 512 + image_h: int = 512 + fixed_size: bool = True + max_num_instances: int = 200 + min_process_size: int = 320 + max_process_size: int = 608 + letter_box: bool = True + random_flip: bool = True + pct_rand: float = 0.0 + jitter_im: float = 0.0 + jitter_boxes: float = 0.000 + aug_rand_transalate: float = 0.0 + aug_rand_saturation: float = 0.0 + aug_rand_brightness: float = 0.0 + aug_rand_zoom: float = 0.0 + aug_rand_hue: float = 0.0 + keep_thresh: float = 0.0 + mosaic_frequency: float = 1.0 + use_tie_breaker: bool = True + +@dataclasses.dataclass +class DataConfig(cfg.DataConfig): + """Input config for training.""" + input_path: str = 'D:/Datasets/coco/2017/1.1.0/val' #'gs://tensorflow2/coco_records/train/2017*' + tfds_name: str = None #'coco' + tfds_split: str = 'validation' #'train' + global_batch_size: int = 32 + is_training: bool = True + dtype: str = 'float16' + decoder: DataDecoder = DataDecoder() + parser: Parser = Parser() + shuffle_buffer_size: int = 10000 + tfds_download: bool = True + @dataclasses.dataclass class Loss(hyperparams.Config): pass @@ -77,6 +130,8 @@ class CenterNetBase(hyperparams.OneOfConfig): @dataclasses.dataclass class CenterNet(hyperparams.Config): num_classes: int = 90 + gaussian_iou: float = 0.7 + max_num_instances: int = 200 input_size: Optional[List[int]] = dataclasses.field( default_factory=lambda: [None, None, 3]) base: Union[str, CenterNetBase] = CenterNetBase() @@ -84,6 +139,8 @@ class CenterNet(hyperparams.Config): @dataclasses.dataclass class CenterNetTask(cfg.TaskConfig): model: CenterNet = CenterNet() + train_data: DataConfig = DataConfig(is_training=True) + validation_data: DataConfig = DataConfig(is_training=False) subtasks: CenterNetSubTasks = CenterNetSubTasks() losses: Losses = Losses() diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 1758e0496..ce9f27230 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -2,145 +2,186 @@ from official.vision.beta.dataloaders import parser from centernet.ops import preprocessing_ops +from yolo.ops import preprocessing_ops as yolo_preprocessing_ops class CenterNetParser(parser.Parser): - def __init__( - self, - num_classes: int, - max_num_instances: int, - gaussian_iou: float, - - ): - self._num_classes = num_classes - self._max_num_instances = max_num_instances - self._gaussian_iou = gaussian_iou - self._gaussian_bump = True - self._gaussian_rad = -1 - - def _generate_heatmap(self, boxes, output_size, input_size): - boxes = tf.cast(boxes, dtype=tf.float32) - - tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) - - width_ratio = output_size[1] / input_size[1] - height_ratio = output_size[0] / input_size[0] - - for tag_ind, detection in enumerate(boxes): - category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary - category = 0 # FIXME: For testing only - - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] - - xct, yct = ( - (detection[2] + detection[0]) / 2, - (detection[3] + detection[1]) / 2 - ) - - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - xtl = tf.math.floor(fxtl) - ytl = tf.math.floor(fytl) - xbr = tf.math.floor(fxbr) - ybr = tf.math.floor(fybr) - xct = tf.math.floor(fxct) - yct = tf.math.floor(fyct) - - if self._gaussian_bump: - width = detection[2] - detection[0] - height = detection[3] - detection[1] - - width = tf.math.ceil(width * width_ratio) - height = tf.math.ceil(height * height_ratio) - - if self._gaussian_rad == -1: - radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - radius = tf.math.maximum(0, tf.math.floor(radius)) - else: - radius = self._gaussian_rad - - # test - # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[category], category, [xtl, ytl], radius) - # inputs heatmap, center, radius, k=1 - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[category, xtl, ytl, radius]]) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[category, xbr, ybr, radius]]) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[category, xct, yct, radius]], scaling_factor=5) - + def __init__(self, + image_w: int = 512, + image_h: int = 512, + num_classes: int = 90, + max_num_instances: int = 200, + gaussian_iou: float = 0.7, + dtype: str = 'float32'): + + self._image_w = image_w + self._image_h = image_h + self._num_classes = num_classes + self._max_num_instances = max_num_instances + self._gaussian_iou = gaussian_iou + self._gaussian_bump = True + self._gaussian_rad = -1 + + if dtype == 'float16': + self._dtype = tf.float16 + elif dtype == 'bfloat16': + self._dtype = tf.bfloat16 + elif dtype == 'float32': + self._dtype = tf.float32 + else: + raise Exception( + 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' + ) + + def _generate_heatmap(self, boxes, output_size, input_size): + boxes = tf.cast(boxes, dtype=tf.float32) + + tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) + tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) + tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) + + width_ratio = output_size[1] / input_size[1] + height_ratio = output_size[0] / input_size[0] + + width_ratio = tf.cast(width_ratio, tf.float32) + height_ratio = tf.cast(height_ratio, tf.float32) + + num_boxes = tf.shape(boxes)[0] + + for tag_ind in tf.range(num_boxes): + detection = boxes[tag_ind] + + category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary + category = 0 # FIXME: For testing only + + xtl, ytl = detection[0], detection[1] + xbr, ybr = detection[2], detection[3] + + xct, yct = ( + (detection[2] + detection[0]) / 2, + (detection[3] + detection[1]) / 2 + ) + + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + xtl = tf.math.floor(fxtl) + ytl = tf.math.floor(fytl) + xbr = tf.math.floor(fxbr) + ybr = tf.math.floor(fybr) + xct = tf.math.floor(fxct) + yct = tf.math.floor(fyct) + + if self._gaussian_bump: + width = detection[2] - detection[0] + height = detection[3] - detection[1] + + width = tf.math.ceil(width * width_ratio) + height = tf.math.ceil(height * height_ratio) + + if self._gaussian_rad == -1: + radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) + radius = tf.math.maximum(0.0, tf.math.floor(radius)) else: - # TODO: See if this is a typo - # tl_heatmaps[category, ytl, xtl] = 1 - # br_heatmaps[category, ybr, xbr] = 1 - # ct_heatmaps[category, yct, xct] = 1 - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ybr, xbr]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, yct, xct]], [1]) - - # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] - # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] - # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] - # tl_size[tag_ind] = ytl * output_size[1] + xtl - # br_size[tag_ind] = ybr * output_size[1] + xbr - # ct_size[tag_ind] = yct * output_size[1] + xct - tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) - br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) - ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) - - labels = { - 'tl_size': tl_size, - 'br_size': br_size, - 'ct_size': ct_size, - 'tl_heatmaps': tl_heatmaps, - 'br_heatmaps': br_heatmaps, - 'ct_heatmaps': ct_heatmaps, - 'tag_masks': tag_masks, - 'tl_offset': tl_offset, - 'br_offset': br_offset, - 'ct_offset': ct_offset, - } - return labels - - def _parse_train_data(self, decoded_tensors): - """Generates images and labels that are usable for model training. - - Args: - decoded_tensors: a dict of Tensors produced by the decoder. - - Returns: - images: the image tensor. - labels: a dict of Tensors that contains labels. - """ - # TODO: input size, output size - image = decoded_tensors["image"] - labels = self._generate_heatmap( - decoded_tensors["groundtruth_boxes"], - output_size, input_size - ) - return image, labels - - def _parse_eval_data(self, data): - image = decoded_tensors["image"] - labels = self._generate_heatmap( - decoded_tensors["groundtruth_boxes"], - output_size, input_size - ) - return image, labels + radius = self._gaussian_rad + + # test + # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[category], category, [xtl, ytl], radius) + # inputs heatmap, center, radius, k=1 + tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[category, xtl, ytl, radius]]) + br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[category, xbr, ybr, radius]]) + ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[category, xct, yct, radius]], scaling_factor=5) + + else: + # TODO: See if this is a typo + # tl_heatmaps[category, ytl, xtl] = 1 + # br_heatmaps[category, ybr, xbr] = 1 + # ct_heatmaps[category, yct, xct] = 1 + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ybr, xbr]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, yct, xct]], [1]) + + # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] + # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] + # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] + # tl_size[tag_ind] = ytl * output_size[1] + xtl + # br_size[tag_ind] = ybr * output_size[1] + xbr + # ct_size[tag_ind] = yct * output_size[1] + xct + tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) + tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) + br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) + ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) + + labels = { + 'tl_size': tl_size, + 'br_size': br_size, + 'ct_size': ct_size, + 'tl_heatmaps': tl_heatmaps, + 'br_heatmaps': br_heatmaps, + 'ct_heatmaps': ct_heatmaps, + 'tag_masks': tag_masks, + 'tl_offset': tl_offset, + 'br_offset': br_offset, + 'ct_offset': ct_offset, + } + return labels + + def _parse_train_data(self, decoded_tensors): + """Generates images and labels that are usable for model training. + + Args: + decoded_tensors: a dict of Tensors produced by the decoder. + + Returns: + images: the image tensor. + labels: a dict of Tensors that contains labels. + """ + # TODO: input size, output size + image = decoded_tensors["image"] + labels = self._generate_heatmap( + decoded_tensors["groundtruth_boxes"], + output_size, input_size + ) + return image, labels + + def _parse_eval_data(self, data): + image = data["image"] / 255 + boxes = data['groundtruth_boxes'] + classes = data['groundtruth_classes'] + + image, boxes, info = yolo_preprocessing_ops.letter_box( + image, boxes, xs = 0.5, ys = 0.5, target_dim=self._image_w) + + image = tf.cast(image, self._dtype) + shape = tf.shape(image) + height = shape[0] + width = shape[1] + + labels = self._generate_heatmap( + boxes=boxes, output_size=[self._image_h, self._image_w], input_size=[height, width] + ) + + labels.update({'bbox': boxes}) + + return image, labels + + def postprocess_fn(self, is_training): + if is_training: #or self._cutmix + return None # if not self._fixed_size or self._mosaic else None + else: + return None class ObjectDetectionTest(tf.test.TestCase): def generate_heatmaps(self, dectections): @@ -161,7 +202,7 @@ def generate_heatmaps(self, dectections): ] #labels = tf.function(CenterNetParser(2, 200, 0.7)._generate_heatmap)( - labels = CenterNetParser(2, 200, 0.7)._generate_heatmap( + labels = CenterNetParser()._generate_heatmap( tf.constant(detections, dtype=tf.float32), [416, 416], [416, 416] ) tl_heatmaps = labels['tl_heatmaps'] diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 784ab3e53..7c6f36988 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -61,7 +61,7 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: # TODO discuss whether to return scalar or tensor # return tf.reduce_min([r1, r2, r3], axis=0) - print(r1, r2, r3) + # print(r1, r2, r3) return tf.reduce_min([r1, r2, r3], axis=0) def _gaussian_penalty(radius: int, dtype=tf.float32) -> tf.Tensor: @@ -149,9 +149,9 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) - print('heatmap ',heatmap) - print(len(heatmap)) - print('category ',category) + # print('heatmap ',heatmap) + # print(len(heatmap)) + # print('category ',category) # TODO: make sure this replicates original functionality # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] @@ -180,7 +180,7 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): heatmap_mask = heatmap_mask_ta.stack() heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, masked_gaussian * scaling_factor) - print('after ',heatmap) + # print('after ',heatmap) return heatmap # def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index f859fc9a5..11b49364b 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -5,10 +5,11 @@ from official.core import base_task from official.core import input_reader from official.core import task_factory -from official.vision.beta.evaluation import coco_evaluator -# TODO: Already added to official codebase, change to official version later -from yolo.dataloaders.decoders import tfds_coco_decoder +from official.vision.beta.evaluation import coco_evaluator +from official.vision.beta.dataloaders import tf_example_decoder +from official.vision.beta.dataloaders import tfds_detection_decoders +from official.vision.beta.dataloaders import tf_example_label_map_decoder from centernet.configs import centernet as cfg from centernet.dataloaders import centernet_input @@ -26,29 +27,13 @@ def __init__(self, params, logging_dir: str = None): def build_inputs(self, params, input_context=None): """Build input dataset.""" - decoder = tfds_coco_decoder.MSCOCODecoder() - """ - decoder_cfg = params.decoder.get() - if params.decoder.type == 'simple_decoder': - decoder = tf_example_decoder.TfExampleDecoder( - regenerate_source_id=decoder_cfg.regenerate_source_id) - elif params.decoder.type == 'label_map_decoder': - decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap( - label_map=decoder_cfg.label_map, - regenerate_source_id=decoder_cfg.regenerate_source_id) - else: - raise ValueError('Unknown decoder type: {}!'.format(params.decoder.type)) - """ - + decoder = self.get_decoder(params) model = self.task_config.model - masks, path_scales, xy_scales = self._get_masks() - anchors = self._get_boxes(gen_boxes=params.is_training) - - print(masks, path_scales, xy_scales) parser = centernet_input.CenterNetParser( num_classes=model.num_classes, - gaussian_iou=model.gaussian_iou + max_num_instances=model.max_num_instances, + gaussian_iou=model.gaussian_iou, ) if params.is_training: @@ -80,8 +65,27 @@ def build_model(self): self._loss_dict = losses return model - def build_inputs(self, params, input_context=None): - pass + def get_decoder(self, params): + if params.tfds_name: + if params.tfds_name in tfds_detection_decoders.TFDS_ID_TO_DECODER_MAP: + decoder = tfds_detection_decoders.TFDS_ID_TO_DECODER_MAP[ + params.tfds_name]() + else: + raise ValueError('TFDS {} is not supported'.format(params.tfds_name)) + else: + decoder_cfg = params.decoder.get() + if params.decoder.type == 'simple_decoder': + decoder = tf_example_decoder.TfExampleDecoder( + regenerate_source_id=decoder_cfg.regenerate_source_id) + elif params.decoder.type == 'label_map_decoder': + decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap( + label_map=decoder_cfg.label_map, + regenerate_source_id=decoder_cfg.regenerate_source_id) + else: + raise ValueError('Unknown decoder type: {}!'.format( + params.decoder.type)) + return decoder + def build_losses(self, outputs, labels, aux_losses=None): total_loss = 0.0 @@ -127,12 +131,7 @@ def build_losses(self, outputs, labels, aux_losses=None): def build_metrics(self, training=True): metrics = [] - metric_names = self._metric_names - - for name in metric_names: - metrics.append(tf.keras.metrics.Mean(name, dtype=tf.float32)) - self._metrics = metrics if not training: self.coco_metric = coco_evaluator.COCOEvaluator( annotation_file=self.task_config.annotation_file, @@ -150,8 +149,8 @@ def validation_step(self, inputs, model, metrics=None): # computer detivative and apply gradients y_pred = model(image, training=False) - y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred['raw_output']) - loss_metrics = self.build_losses(y_pred, label) + y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) + loss_metrics = self.build_losses(y_pred['raw_output'], label) logs = {self.loss: loss_metrics['total_loss']} coco_model_outputs = { diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index 464a2beda..4c2dcf774 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -11,6 +11,7 @@ from centernet.tasks.centernet import CenterNetTask from centernet.configs import centernet as exp_cfg from centernet.utils.weight_utils.load_weights import get_model_weights_as_dict, load_weights_model +from centernet.dataloaders.centernet_input import CenterNetParser import tensorflow_datasets as tfds @@ -40,32 +41,19 @@ def testCenterNetValidation(self): task = CenterNetTask(config) model = task.build_model() metrics = task.build_metrics(training=False) + strategy = tf.distribute.get_strategy() weights_dict, _ = get_model_weights_as_dict(CENTERNET_CKPT_PATH) load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') - - data_dir = 'D:\Datasets' - ds, ds_info = tfds.load(name='coco/2017', - split='validation', - shuffle_files=True, - data_dir=data_dir, - with_info=True, - download=True) - - strategy = tf.distribute.get_strategy() - dataset = orbit.utils.make_distributed_dataset(strategy, ds) + + dataset = orbit.utils.make_distributed_dataset(strategy, task.build_inputs, + config.validation_data) + #val_data = task.build_inputs(config.validation_data) iterator = iter(dataset) - print(next(iterator)) - # logs = task.validation_step(next(iterator), model, metrics=metrics) - # print(logs) - -if __name__ == '__main__': - tf.test.main() - - - - - + logs = task.validation_step(next(iterator), model, metrics=metrics) + print(logs) +if __name__ == '__main__': + tf.test.main() \ No newline at end of file diff --git a/yolo/ops/preprocessing_ops.py b/yolo/ops/preprocessing_ops.py index ca730a050..29b1844e7 100755 --- a/yolo/ops/preprocessing_ops.py +++ b/yolo/ops/preprocessing_ops.py @@ -455,34 +455,46 @@ def filter_boxes_and_classes(boxes, classes, image_info, keep_thresh=0.01): return boxes, classes -def letter_box(image, boxes, target_dim=None): +def letter_box(image, boxes, xs = 0.5, ys = 0.5, target_dim=None): height, width = get_image_shape(image) clipper = tf.math.maximum(width, height) if target_dim is None: target_dim = clipper - pad_width = clipper - width - pad_height = clipper - height - image = tf.image.pad_to_bounding_box(image, pad_height // 2, pad_width // 2, + xs = tf.convert_to_tensor(xs) + ys = tf.convert_to_tensor(ys) + pad_width_p = clipper - width + pad_height_p = clipper - height + pad_height = tf.cast(tf.cast(pad_height_p, ys.dtype) * ys, tf.int32) + pad_width = tf.cast(tf.cast(pad_width_p, xs.dtype) * xs, tf.int32) + image = tf.image.pad_to_bounding_box(image, pad_height, pad_width, clipper, clipper) boxes = box_ops.yxyx_to_xcycwh(boxes) x, y, w, h = tf.split(boxes, 4, axis=-1) - y *= tf.cast(height / clipper, tf.float32) - x *= tf.cast(width / clipper, tf.float32) + y *= tf.cast(height / clipper, y.dtype) + x *= tf.cast(width / clipper, x.dtype) - y += tf.cast((pad_height / clipper) / 2, tf.float32) - x += tf.cast((pad_width / clipper) / 2, tf.float32) + y += tf.cast((pad_height / clipper), y.dtype) + x += tf.cast((pad_width / clipper), x.dtype) - h *= tf.cast(height / clipper, tf.float32) - w *= tf.cast(width / clipper, tf.float32) + h *= tf.cast(height / clipper, h.dtype) + w *= tf.cast(width / clipper, w.dtype) boxes = tf.concat([x, y, w, h], axis=-1) boxes = box_ops.xcycwh_to_yxyx(boxes) + boxes = tf.where(h == 0, tf.zeros_like(boxes), boxes) + image = tf.image.resize(image, (target_dim, target_dim)) - return image, boxes + + scale = target_dim/clipper + pt_width = tf.cast(tf.cast(pad_width, scale.dtype) * scale, tf.int32) + pt_height = tf.cast(tf.cast(pad_height, scale.dtype) * scale, tf.int32) + pt_width_p = tf.cast(tf.cast(pad_width_p, scale.dtype) * scale, tf.int32) + pt_height_p = tf.cast(tf.cast(pad_height_p, scale.dtype) * scale, tf.int32) + return image, boxes, [pt_height, pt_width, target_dim - pt_height_p, target_dim - pt_width_p] def patch_four_images(images): From 58370a03538651f853242652106c3dda1448e9f9 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 5 Apr 2021 11:22:50 -0400 Subject: [PATCH 080/132] yaml, task, and model changes --- .vscode/settings.json | 2 +- centernet/common/registry_imports.py | 5 + centernet/configs/backbones.py | 5 +- centernet/configs/centernet.py | 113 ++++- .../configs/experiments/centernet-eval.yaml | 86 ++++ centernet/dataloaders/centernet_input.py | 3 +- centernet/dataloaders/centernet_input_test.py | 16 +- centernet/losses/__init__.py | 2 +- centernet/losses/l1_localization_loss_test.py | 3 +- .../penalty_reduced_logistic_focal_loss.py | 1 + ...enalty_reduced_logistic_focal_loss_test.py | 2 +- centernet/modeling/CenterNet.py | 26 +- centernet/modeling/CenterNet_test.py | 12 +- centernet/modeling/backbones/hourglass.py | 10 +- .../modeling/backbones/hourglass_test.py | 8 +- .../modeling/decoders/centernet_decoder.py | 63 ++- .../decoders/centernet_decoder_test.py | 4 +- .../modeling/layers/detection_generator.py | 6 +- centernet/modeling/layers/nn_blocks.py | 7 +- centernet/modeling/layers/nn_blocks_test.py | 5 +- centernet/modeling/plot_models.py | 7 +- centernet/ops/loss_ops.py | 2 +- centernet/ops/preprocessing_ops.py | 2 +- centernet/ops/preprocessing_ops_test.py | 5 +- centernet/tasks/_centernet_input.py | 15 +- centernet/tasks/centernet.py | 59 ++- .../tasks/centernet_object_detection_test.py | 10 +- centernet/tasks/centernet_test.py | 39 +- centernet/tasks/show_heatmaps.py | 4 +- centernet/train.py | 95 ++++- .../utils/weight_utils/config_classes.py | 5 +- centernet/utils/weight_utils/config_data.py | 7 +- centernet/utils/weight_utils/load_weights.py | 23 +- .../utils/weight_utils/test_load_weights.py | 15 +- centernet/utils/weight_utils/tf_to_dict.py | 3 +- fixefficientnet/train.py | 23 +- hooks/install.sh | 13 + hooks/pre-commit | 23 ++ hooks/readme.md | 55 +++ hooks/requirements.txt | 3 + hooks/screenshots/git_commit.png | Bin 0 -> 155515 bytes hooks/screenshots/mv_command.png | Bin 0 -> 8321 bytes hooks/screenshots/necessary_files.png | Bin 0 -> 28831 bytes hooks/screenshots/readme.txt | 1 + linter/yapf_contrib/fixers/fixers_api.py | 3 +- .../fixers/line_conditional_fix.py | 4 +- linter_log.txt | 0 official/common/__init__.py | 2 - official/common/dataset_fn.py | 2 +- official/common/distribute_utils.py | 1 + official/core/__init__.py | 1 - official/core/base_task.py | 5 +- official/core/base_trainer.py | 8 +- official/core/base_trainer_test.py | 6 +- official/core/config_definitions.py | 3 +- official/core/exp_factory.py | 1 - official/core/input_reader.py | 2 +- official/core/registry_test.py | 5 +- official/core/train_lib.py | 8 +- official/core/train_lib_test.py | 14 +- official/core/train_utils.py | 12 +- official/modeling/__init__.py | 1 - official/modeling/activations/__init__.py | 5 +- official/modeling/activations/gelu_test.py | 3 +- official/modeling/activations/relu_test.py | 4 +- official/modeling/activations/sigmoid_test.py | 4 +- official/modeling/activations/swish_test.py | 3 +- official/modeling/hyperparams/__init__.py | 2 - official/modeling/hyperparams/base_config.py | 4 +- .../modeling/hyperparams/base_config_test.py | 5 +- official/modeling/hyperparams/oneof.py | 2 +- official/modeling/hyperparams/oneof_test.py | 5 +- official/modeling/multitask/__init__.py | 1 - official/modeling/multitask/base_model.py | 2 +- official/modeling/multitask/configs.py | 3 +- official/modeling/multitask/evaluator.py | 6 +- official/modeling/multitask/evaluator_test.py | 8 +- official/modeling/multitask/multitask.py | 5 +- official/modeling/multitask/train_lib.py | 4 +- .../modeling/optimization/SGDAccumulated.py | 8 +- .../modeling/optimization/SGDAccumulated__.py | 8 +- official/modeling/optimization/__init__.py | 4 +- .../modeling/optimization/configs/__init__.py | 1 - .../configs/learning_rate_config.py | 2 +- .../configs/optimization_config.py | 9 +- .../configs/optimization_config_test.py | 3 +- .../optimization/configs/optimizer_config.py | 2 +- .../modeling/optimization/ema_optimizer.py | 2 +- official/modeling/optimization/lr_schedule.py | 2 +- .../optimization/optimizer_factory.py | 8 +- .../optimization/optimizer_factory_test.py | 2 +- official/modeling/progressive/policies.py | 6 +- official/modeling/progressive/train.py | 12 +- official/modeling/progressive/train_lib.py | 6 +- .../modeling/progressive/train_lib_test.py | 14 +- official/modeling/progressive/trainer.py | 9 +- official/modeling/progressive/trainer_test.py | 7 +- official/modeling/progressive/utils.py | 3 +- official/modeling/tf_utils.py | 2 +- official/nlp/albert/export_albert_tfhub.py | 8 +- .../nlp/albert/export_albert_tfhub_test.py | 4 +- official/nlp/albert/run_classifier.py | 9 +- official/nlp/albert/run_squad.py | 10 +- ...tf2_albert_encoder_checkpoint_converter.py | 8 +- official/nlp/bert/__init__.py | 1 - official/nlp/bert/bert_models.py | 3 +- official/nlp/bert/common_flags.py | 2 +- official/nlp/bert/export_tfhub.py | 13 +- official/nlp/bert/export_tfhub_test.py | 5 +- official/nlp/bert/model_saving_utils.py | 4 +- official/nlp/bert/model_training_utils.py | 3 +- .../nlp/bert/model_training_utils_test.py | 14 +- official/nlp/bert/run_classifier.py | 13 +- official/nlp/bert/run_pretraining.py | 15 +- official/nlp/bert/run_squad.py | 11 +- official/nlp/bert/run_squad_helper.py | 14 +- official/nlp/bert/serving.py | 6 +- official/nlp/bert/squad_evaluate_v1_1.py | 3 +- .../bert/tf2_encoder_checkpoint_converter.py | 11 +- official/nlp/bert/tokenization.py | 3 +- official/nlp/configs/bert.py | 3 +- official/nlp/configs/electra.py | 6 +- official/nlp/configs/encoders.py | 7 +- official/nlp/configs/experiment_configs.py | 4 +- .../nlp/configs/finetuning_experiments.py | 7 +- official/nlp/continuous_finetune_lib.py | 11 +- official/nlp/continuous_finetune_lib_test.py | 13 +- official/nlp/data/classifier_data_lib.py | 2 +- official/nlp/data/create_finetuning_data.py | 14 +- official/nlp/data/create_pretraining_data.py | 6 +- .../nlp/data/create_xlnet_pretraining_data.py | 17 +- .../create_xlnet_pretraining_data_test.py | 5 +- official/nlp/data/data_loader_factory_test.py | 1 + official/nlp/data/pretrain_dataloader.py | 9 +- official/nlp/data/pretrain_dataloader_test.py | 2 +- .../nlp/data/question_answering_dataloader.py | 6 +- .../data/sentence_prediction_dataloader.py | 5 +- .../sentence_prediction_dataloader_test.py | 4 +- official/nlp/data/sentence_retrieval_lib.py | 1 + official/nlp/data/squad_lib.py | 3 +- official/nlp/data/squad_lib_sp.py | 2 +- official/nlp/data/tagging_data_lib.py | 2 +- official/nlp/data/tagging_data_lib_test.py | 2 +- official/nlp/data/tagging_dataloader.py | 6 +- official/nlp/data/tagging_dataloader_test.py | 2 +- official/nlp/data/train_sentencepiece.py | 6 +- official/nlp/data/wmt_dataloader.py | 6 +- official/nlp/data/wmt_dataloader_test.py | 4 +- official/nlp/keras_nlp/__init__.py | 3 +- .../nlp/keras_nlp/encoders/bert_encoder.py | 3 +- .../keras_nlp/encoders/bert_encoder_test.py | 5 +- official/nlp/keras_nlp/layers/__init__.py | 3 +- .../layers/on_device_embedding_test.py | 3 +- .../layers/position_embedding_test.py | 3 +- .../layers/transformer_encoder_block_test.py | 8 +- official/nlp/keras_nlp/setup.py | 3 +- official/nlp/modeling/__init__.py | 5 +- official/nlp/modeling/layers/__init__.py | 33 +- .../nlp/modeling/layers/attention_test.py | 3 +- official/nlp/modeling/layers/dense_einsum.py | 1 - .../nlp/modeling/layers/dense_einsum_test.py | 3 +- .../modeling/layers/gated_feedforward_test.py | 5 +- official/nlp/modeling/layers/masked_lm.py | 1 - .../nlp/modeling/layers/masked_lm_test.py | 4 +- .../modeling/layers/masked_softmax_test.py | 3 +- .../modeling/layers/mat_mul_with_margin.py | 1 + .../layers/mat_mul_with_margin_test.py | 3 +- .../layers/mobile_bert_layers_test.py | 3 +- .../layers/multi_channel_attention.py | 1 + .../modeling/layers/on_device_embedding.py | 1 - .../layers/position_embedding_test.py | 5 +- .../nlp/modeling/layers/relative_attention.py | 2 +- .../layers/relative_attention_test.py | 5 +- .../layers/rezero_transformer_test.py | 3 +- .../layers/talking_heads_attention_test.py | 5 +- official/nlp/modeling/layers/text_layers.py | 2 +- .../nlp/modeling/layers/text_layers_test.py | 2 +- .../nlp/modeling/layers/tn_expand_condense.py | 3 +- .../layers/tn_expand_condense_test.py | 3 +- .../modeling/layers/tn_transformer_test.py | 8 +- official/nlp/modeling/layers/transformer.py | 3 +- .../modeling/layers/transformer_scaffold.py | 2 +- .../layers/transformer_scaffold_test.py | 6 +- .../nlp/modeling/layers/transformer_test.py | 3 +- .../nlp/modeling/layers/transformer_xl.py | 3 +- .../modeling/layers/transformer_xl_test.py | 4 +- official/nlp/modeling/losses/__init__.py | 3 +- ...ed_sparse_categorical_crossentropy_test.py | 10 +- official/nlp/modeling/models/__init__.py | 9 +- .../nlp/modeling/models/bert_classifier.py | 4 +- .../modeling/models/bert_classifier_test.py | 5 +- .../nlp/modeling/models/bert_pretrainer.py | 5 +- .../modeling/models/bert_pretrainer_test.py | 8 +- .../nlp/modeling/models/bert_span_labeler.py | 1 + .../modeling/models/bert_span_labeler_test.py | 5 +- .../modeling/models/bert_token_classifier.py | 1 + .../models/bert_token_classifier_test.py | 5 +- official/nlp/modeling/models/dual_encoder.py | 1 + .../nlp/modeling/models/dual_encoder_test.py | 5 +- .../models/electra_pretrainer_test.py | 3 +- .../modeling/models/seq2seq_transformer.py | 1 + .../models/seq2seq_transformer_test.py | 7 +- official/nlp/modeling/models/xlnet.py | 4 +- official/nlp/modeling/models/xlnet_test.py | 6 +- official/nlp/modeling/networks/__init__.py | 11 +- .../nlp/modeling/networks/albert_encoder.py | 1 + .../modeling/networks/albert_encoder_test.py | 9 +- .../nlp/modeling/networks/bert_encoder.py | 1 + .../modeling/networks/bert_encoder_test.py | 7 +- .../nlp/modeling/networks/classification.py | 1 + .../modeling/networks/classification_test.py | 9 +- .../nlp/modeling/networks/encoder_scaffold.py | 2 +- .../networks/encoder_scaffold_test.py | 5 +- .../networks/mobile_bert_encoder_test.py | 4 +- .../networks/packed_sequence_embedding.py | 1 + .../packed_sequence_embedding_test.py | 2 +- .../nlp/modeling/networks/span_labeling.py | 1 + .../modeling/networks/span_labeling_test.py | 3 +- official/nlp/modeling/networks/xlnet_base.py | 3 +- .../nlp/modeling/networks/xlnet_base_test.py | 9 +- official/nlp/modeling/ops/__init__.py | 4 +- official/nlp/modeling/ops/beam_search_test.py | 2 +- official/nlp/modeling/ops/decoding_module.py | 4 - .../nlp/modeling/ops/decoding_module_test.py | 1 + official/nlp/modeling/ops/sampling_module.py | 7 - .../modeling/ops/segment_extractor_test.py | 2 +- official/nlp/nhnet/configs.py | 3 +- official/nlp/nhnet/configs_test.py | 1 + official/nlp/nhnet/decoder.py | 1 + official/nlp/nhnet/decoder_test.py | 5 +- official/nlp/nhnet/evaluation.py | 8 +- official/nlp/nhnet/models.py | 9 +- official/nlp/nhnet/models_test.py | 13 +- official/nlp/nhnet/raw_data_process.py | 4 +- official/nlp/nhnet/trainer.py | 12 +- official/nlp/nhnet/trainer_test.py | 10 +- official/nlp/nhnet/utils.py | 6 +- official/nlp/optimization.py | 2 +- official/nlp/projects/triviaqa/dataset.py | 4 +- .../projects/triviaqa/download_and_prepare.py | 9 +- official/nlp/projects/triviaqa/evaluate.py | 4 +- official/nlp/projects/triviaqa/inputs.py | 3 +- official/nlp/projects/triviaqa/predict.py | 10 +- official/nlp/projects/triviaqa/preprocess.py | 13 +- .../projects/triviaqa/sentencepiece_pb2.py | 1 + official/nlp/projects/triviaqa/train.py | 12 +- official/nlp/tasks/electra_task.py | 8 +- official/nlp/tasks/electra_task_test.py | 4 +- official/nlp/tasks/masked_lm.py | 7 +- official/nlp/tasks/masked_lm_test.py | 3 +- official/nlp/tasks/question_answering.py | 11 +- official/nlp/tasks/question_answering_test.py | 8 +- official/nlp/tasks/sentence_prediction.py | 10 +- .../nlp/tasks/sentence_prediction_test.py | 8 +- official/nlp/tasks/tagging.py | 8 +- official/nlp/tasks/translation.py | 6 +- official/nlp/tasks/translation_test.py | 4 +- official/nlp/tasks/utils.py | 3 +- official/nlp/train.py | 13 +- official/nlp/train_ctl_continuous_finetune.py | 7 +- official/nlp/transformer/attention_layer.py | 1 + official/nlp/transformer/beam_search_v1.py | 1 + official/nlp/transformer/compute_bleu.py | 8 +- official/nlp/transformer/data_download.py | 14 +- official/nlp/transformer/data_pipeline.py | 2 +- official/nlp/transformer/misc.py | 2 +- official/nlp/transformer/model_params.py | 1 - official/nlp/transformer/transformer.py | 8 +- .../transformer/transformer_forward_test.py | 5 +- .../transformer/transformer_layers_test.py | 6 +- official/nlp/transformer/transformer_main.py | 16 +- .../nlp/transformer/transformer_main_test.py | 9 +- official/nlp/transformer/transformer_test.py | 3 +- official/nlp/transformer/translate.py | 4 +- official/nlp/transformer/utils/metrics.py | 6 +- official/nlp/transformer/utils/tokenizer.py | 9 +- official/nlp/xlnet/__init__.py | 1 - official/nlp/xlnet/data_utils.py | 3 +- official/nlp/xlnet/optimization.py | 3 +- .../xlnet/preprocess_classification_data.py | 12 +- .../nlp/xlnet/preprocess_pretrain_data.py | 12 +- official/nlp/xlnet/preprocess_squad_data.py | 8 +- official/nlp/xlnet/run_classifier.py | 14 +- official/nlp/xlnet/run_pretrain.py | 14 +- official/nlp/xlnet/run_squad.py | 18 +- official/nlp/xlnet/squad_utils.py | 9 +- official/nlp/xlnet/training_utils.py | 5 +- official/pip_package/setup.py | 3 +- official/recommendation/create_ncf_data.py | 10 +- official/recommendation/data_pipeline.py | 12 +- official/recommendation/data_preprocessing.py | 21 +- official/recommendation/data_test.py | 13 +- official/recommendation/movielens.py | 14 +- official/recommendation/ncf_common.py | 12 +- official/recommendation/ncf_input_pipeline.py | 7 +- official/recommendation/ncf_keras_main.py | 21 +- official/recommendation/ncf_test.py | 11 +- official/recommendation/neumf_model.py | 12 +- official/recommendation/stat_utils.py | 4 +- official/staging/training/grad_utils.py | 3 +- official/utils/docs/build_docs.py | 10 +- official/utils/flags/_base.py | 7 +- official/utils/flags/_benchmark.py | 4 +- official/utils/flags/_conventions.py | 6 +- official/utils/flags/_device.py | 7 +- official/utils/flags/_distribution.py | 6 +- official/utils/flags/_misc.py | 4 +- official/utils/flags/_performance.py | 6 +- official/utils/flags/core.py | 16 +- official/utils/flags/flags_test.py | 2 +- official/utils/hyperparams_flags.py | 5 +- official/utils/misc/distribution_utils.py | 1 - official/utils/misc/keras_utils.py | 3 +- official/utils/misc/model_helpers.py | 8 +- official/utils/misc/model_helpers_test.py | 4 +- official/utils/testing/integration.py | 4 +- official/utils/testing/mock_task.py | 4 +- official/vision/beta/__init__.py | 3 +- official/vision/beta/configs/__init__.py | 8 +- official/vision/beta/configs/backbones.py | 3 +- official/vision/beta/configs/backbones_3d.py | 3 +- official/vision/beta/configs/decoders.py | 3 +- .../beta/configs/image_classification.py | 9 +- .../beta/configs/image_classification_test.py | 2 +- official/vision/beta/configs/maskrcnn.py | 10 +- official/vision/beta/configs/retinanet.py | 10 +- .../beta/configs/semantic_segmentation.py | 9 +- .../configs/semantic_segmentation_test.py | 2 +- .../beta/configs/video_classification.py | 9 +- .../beta/configs/video_classification_test.py | 2 +- .../vision/beta/data/create_coco_tf_record.py | 8 +- official/vision/beta/data/tfrecord_lib.py | 8 +- .../vision/beta/data/tfrecord_lib_test.py | 3 +- .../beta/dataloaders/classification_input.py | 7 +- .../vision/beta/dataloaders/maskrcnn_input.py | 7 +- .../beta/dataloaders/retinanet_input.py | 7 +- .../beta/dataloaders/segmentation_input.py | 4 +- .../dataloaders/tf_example_decoder_test.py | 6 +- .../tf_example_label_map_decoder.py | 1 + .../tf_example_label_map_decoder_test.py | 6 +- .../tfds_classification_decoders.py | 1 + .../dataloaders/tfds_detection_decoders.py | 1 + .../dataloaders/tfds_segmentation_decoders.py | 1 + .../vision/beta/dataloaders/video_input.py | 5 +- .../beta/dataloaders/video_input_test.py | 3 +- .../vision/beta/evaluation/coco_evaluator.py | 7 +- official/vision/beta/evaluation/coco_utils.py | 9 +- .../vision/beta/losses/maskrcnn_losses.py | 1 - .../beta/modeling/backbones/__init__.py | 3 +- .../beta/modeling/backbones/efficientnet.py | 5 +- .../modeling/backbones/efficientnet_test.py | 2 +- .../vision/beta/modeling/backbones/factory.py | 1 - .../beta/modeling/backbones/factory_test.py | 4 +- .../beta/modeling/backbones/mobilenet.py | 11 +- .../beta/modeling/backbones/mobilenet_test.py | 7 +- .../vision/beta/modeling/backbones/resnet.py | 4 +- .../beta/modeling/backbones/resnet_3d.py | 1 + .../beta/modeling/backbones/resnet_3d_test.py | 2 +- .../beta/modeling/backbones/resnet_deeplab.py | 4 +- .../modeling/backbones/resnet_deeplab_test.py | 7 +- .../beta/modeling/backbones/resnet_test.py | 7 +- .../vision/beta/modeling/backbones/revnet.py | 3 +- .../beta/modeling/backbones/revnet_test.py | 2 +- .../beta/modeling/backbones/spinenet.py | 6 +- .../beta/modeling/backbones/spinenet_test.py | 2 +- .../modeling/classification_model_test.py | 10 +- .../beta/modeling/decoders/aspp_test.py | 2 +- .../vision/beta/modeling/decoders/fpn_test.py | 2 +- .../vision/beta/modeling/decoders/nasfpn.py | 3 +- .../beta/modeling/decoders/nasfpn_test.py | 2 +- official/vision/beta/modeling/factory.py | 28 +- official/vision/beta/modeling/factory_3d.py | 6 +- official/vision/beta/modeling/factory_test.py | 14 +- .../heads/dense_prediction_heads_test.py | 4 +- .../modeling/heads/instance_heads_test.py | 4 +- .../modeling/heads/segmentation_heads_test.py | 4 +- .../modeling/layers/detection_generator.py | 3 +- .../layers/detection_generator_test.py | 2 +- .../vision/beta/modeling/layers/nn_blocks.py | 4 +- .../beta/modeling/layers/nn_blocks_3d_test.py | 2 +- .../beta/modeling/layers/nn_blocks_test.py | 6 +- .../vision/beta/modeling/layers/nn_layers.py | 8 +- .../beta/modeling/layers/roi_generator.py | 3 +- .../beta/modeling/maskrcnn_model_test.py | 16 +- .../beta/modeling/retinanet_model_test.py | 8 +- .../beta/modeling/segmentation_model_test.py | 5 +- .../modeling/video_classification_model.py | 1 + .../video_classification_model_test.py | 7 +- official/vision/beta/ops/anchor.py | 7 +- official/vision/beta/ops/anchor_test.py | 5 +- official/vision/beta/ops/augment.py | 7 +- official/vision/beta/ops/augment_test.py | 6 +- official/vision/beta/ops/box_ops.py | 1 - official/vision/beta/ops/mask_ops.py | 2 +- official/vision/beta/ops/mask_ops_test.py | 1 + official/vision/beta/ops/nms.py | 1 - official/vision/beta/ops/preprocess_ops.py | 4 +- official/vision/beta/ops/preprocess_ops_3d.py | 1 + .../vision/beta/ops/preprocess_ops_3d_test.py | 3 +- .../vision/beta/ops/preprocess_ops_test.py | 5 +- .../beta/projects/yolo/configs/backbones.py | 1 - .../yolo/configs/darknet_classification.py | 3 +- .../classification_tfds_decoder.py | 2 - .../yolo/modeling/backbones/darknet_test.py | 5 +- .../yolo/modeling/layers/nn_blocks.py | 2 + .../yolo/modeling/layers/nn_blocks_test.py | 2 +- .../yolo/tasks/image_classification.py | 11 +- official/vision/beta/projects/yolo/train.py | 10 +- official/vision/beta/serving/detection.py | 4 +- .../vision/beta/serving/detection_test.py | 4 +- official/vision/beta/serving/export_base.py | 1 + .../vision/beta/serving/export_saved_model.py | 3 +- .../beta/serving/export_saved_model_lib.py | 5 +- official/vision/beta/serving/export_tfhub.py | 7 +- .../beta/serving/image_classification.py | 1 - .../beta/serving/image_classification_test.py | 4 +- .../beta/serving/semantic_segmentation.py | 1 - .../serving/semantic_segmentation_test.py | 4 +- official/vision/beta/tasks/__init__.py | 8 +- .../vision/beta/tasks/image_classification.py | 11 +- official/vision/beta/tasks/maskrcnn.py | 13 +- official/vision/beta/tasks/retinanet.py | 15 +- .../beta/tasks/semantic_segmentation.py | 11 +- .../vision/beta/tasks/video_classification.py | 7 +- official/vision/beta/train.py | 11 +- .../vision/beta/train_spatial_partitioning.py | 8 +- official/vision/detection/configs/factory.py | 6 +- .../detection/configs/maskrcnn_config.py | 1 - .../detection/configs/olnmask_config.py | 1 - .../detection/configs/retinanet_config.py | 1 - .../vision/detection/dataloader/anchor.py | 13 +- .../vision/detection/dataloader/factory.py | 12 +- .../detection/dataloader/input_reader.py | 7 +- .../detection/dataloader/maskrcnn_parser.py | 5 +- .../vision/detection/dataloader/mode_keys.py | 5 +- .../detection/dataloader/olnmask_parser.py | 7 +- .../detection/dataloader/retinanet_parser.py | 3 +- .../detection/dataloader/shapemask_parser.py | 6 +- .../detection/evaluation/coco_evaluator.py | 8 +- .../vision/detection/evaluation/coco_utils.py | 13 +- .../vision/detection/evaluation/factory.py | 4 +- .../detection/executor/detection_executor.py | 9 +- .../executor/distributed_executor.py | 14 +- official/vision/detection/main.py | 7 +- .../modeling/architecture/factory.py | 15 +- .../detection/modeling/architecture/fpn.py | 7 +- .../detection/modeling/architecture/heads.py | 7 +- .../modeling/architecture/identity.py | 5 +- .../modeling/architecture/keras_utils.py | 5 +- .../modeling/architecture/nn_blocks.py | 4 +- .../detection/modeling/architecture/nn_ops.py | 4 +- .../detection/modeling/architecture/resnet.py | 10 +- .../modeling/architecture/spinenet.py | 6 +- .../vision/detection/modeling/base_model.py | 10 +- .../detection/modeling/checkpoint_utils.py | 7 +- official/vision/detection/modeling/factory.py | 7 +- .../detection/modeling/learning_rates.py | 5 +- official/vision/detection/modeling/losses.py | 6 +- .../detection/modeling/maskrcnn_model.py | 20 +- .../detection/modeling/olnmask_model.py | 17 +- .../vision/detection/modeling/optimizers.py | 4 +- .../detection/modeling/retinanet_model.py | 11 +- .../detection/modeling/shapemask_model.py | 14 +- official/vision/detection/ops/nms.py | 4 +- .../vision/detection/ops/postprocess_ops.py | 4 +- official/vision/detection/ops/roi_ops.py | 4 +- .../detection/ops/spatial_transform_ops.py | 4 +- official/vision/detection/ops/target_ops.py | 7 +- official/vision/detection/utils/box_utils.py | 4 +- official/vision/detection/utils/mask_utils.py | 6 +- .../utils/object_detection/argmax_matcher.py | 4 +- .../balanced_positive_negative_sampler.py | 4 +- .../utils/object_detection/box_coder.py | 4 +- .../utils/object_detection/box_list_ops.py | 9 +- .../object_detection/faster_rcnn_box_coder.py | 4 +- .../utils/object_detection/matcher.py | 3 +- .../object_detection/minibatch_sampler.py | 3 +- .../utils/object_detection/preprocessor.py | 3 +- .../region_similarity_calculator.py | 3 +- .../utils/object_detection/target_assigner.py | 4 +- .../object_detection/visualization_utils.py | 6 +- .../vision/image_classification/augment.py | 10 +- .../image_classification/augment_test.py | 7 +- .../vision/image_classification/callbacks.py | 6 +- .../classifier_trainer.py | 23 +- .../classifier_trainer_test.py | 18 +- .../classifier_trainer_util_test.py | 11 +- .../configs/base_configs.py | 3 +- .../image_classification/configs/configs.py | 7 +- .../image_classification/dataset_factory.py | 13 +- .../efficientnet/common_modules.py | 8 +- .../efficientnet/efficientnet_config.py | 7 +- .../efficientnet/efficientnet_model.py | 8 +- .../efficientnet/tfhub_export.py | 11 +- .../image_classification/learning_rate.py | 4 +- .../learning_rate_test.py | 4 +- .../vision/image_classification/mnist_main.py | 11 +- .../vision/image_classification/mnist_test.py | 10 +- .../image_classification/optimizer_factory.py | 6 +- .../optimizer_factory_test.py | 6 +- .../image_classification/preprocessing.py | 8 +- .../image_classification/resnet/common.py | 9 +- .../resnet/imagenet_preprocessing.py | 6 +- .../resnet/resnet_config.py | 4 +- .../resnet/resnet_ctl_imagenet_main.py | 15 +- .../resnet/resnet_model.py | 9 +- .../resnet/resnet_runnable.py | 7 +- .../resnet/tfhub_export.py | 14 +- .../vision/image_classification/test_utils.py | 8 +- official/vision/keras_cv/__init__.py | 5 +- .../vision/keras_cv/layers/deeplab_test.py | 2 +- .../keras_cv/ops/anchor_generator_test.py | 3 +- official/vision/keras_cv/setup.py | 3 +- orbit/__init__.py | 13 +- orbit/controller.py | 7 +- orbit/controller_test.py | 10 +- orbit/runner.py | 2 - orbit/standard_runner.py | 6 +- orbit/standard_runner_test.py | 6 +- orbit/utils/__init__.py | 14 +- orbit/utils/common_test.py | 4 +- orbit/utils/loop_fns.py | 4 +- orbit/utils/tpu_summaries_test.py | 5 +- panoptic/data/create_cityscaped_tf_record.py | 11 +- .../dataloaders/decoders/coco_panoptic.py | 8 +- .../decoders/tfds_panoptic_coco_decoder.py | 2 + panoptic/dataloaders/encoders/coco.py | 5 +- .../encoders/coco_record_writer.py | 9 +- panoptic/dataloaders/specs/tfds_load.py | 2 +- panoptic/dataloaders/visual_test.py | 9 +- panoptic/train.py | 23 +- pylint.sh | 171 ++++++++ pylintrc | 391 ++++++++++++------ test_folder_linter/test_file.py | 11 +- utils/demos/coco.py | 4 +- utils/demos/coco_dataset_converter.py | 7 +- utils/demos/utils.py | 15 +- utils/downloads/__init__.py | 1 - utils/downloads/file_manager.py | 1 - utils/export/get_cfg_yaml.py | 3 +- utils/export/tensor_rt.py | 6 +- utils/register.py | 10 +- utils/training/keras_task_trainer.py | 3 +- yolo/common/registry_imports.py | 14 +- yolo/common/train_lib.py | 8 +- yolo/configs/backbones.py | 1 + yolo/configs/darknet_classification.py | 7 +- yolo/configs/yolo.py | 9 +- yolo/dataloaders/classification_input.py | 2 +- yolo/dataloaders/classification_vision.py | 5 +- .../decoders/classification_tfds_decoder.py | 3 +- .../dataloaders/decoders/tfds_coco_decoder.py | 1 + yolo/dataloaders/priming_input.py | 2 +- yolo/dataloaders/yolo_input.py | 6 +- yolo/dataloaders/yolo_input_test.py | 26 +- yolo/demos/demo_lib/readers.py | 5 +- yolo/demos/demo_lib/video_detect_gpu.py | 23 +- yolo/demos/demo_lib/writer.py | 5 +- yolo/demos/examples/server/app.py | 28 +- yolo/demos/model_val.py | 2 +- yolo/demos/routed_webcam.py | 20 +- yolo/demos/three_servers/model_server.py | 21 +- yolo/demos/three_servers/udp_server.py | 8 +- yolo/demos/three_servers/video_server.py | 7 +- yolo/demos/video_detect_cpu.py | 8 +- yolo/demos/video_detect_gpu.py | 25 +- yolo/losses/gradient_aggregator.py | 3 +- yolo/losses/yolo_loss.py | 6 +- yolo/modeling/Yolo.py | 12 +- yolo/modeling/backbones/darknet.py | 1 + yolo/modeling/backbones/darknet_test.py | 5 +- yolo/modeling/backbones/spinenet.py | 6 +- yolo/modeling/classification_model.py | 1 + yolo/modeling/decoders/yolo_decoder.py | 1 + yolo/modeling/decoders/yolo_decoder_.py | 1 + yolo/modeling/decoders/yolo_decoder_test.py | 7 +- yolo/modeling/heads/yolo_head.py | 1 + yolo/modeling/heads/yolo_head_test.py | 7 +- yolo/modeling/layers/detection_generator.py | 5 +- .../layers/detection_generator_test.py | 7 +- yolo/modeling/layers/nn_blocks.py | 6 +- yolo/modeling/layers/nn_blocks_test.py | 2 +- yolo/modeling/layers/subnormalization.py | 29 +- yolo/modeling/optimizers/AdamAccumulated.py | 4 +- yolo/modeling/optimizers/SGDAccumulated.py | 4 +- yolo/ops/box_ops.py | 6 +- yolo/ops/kmeans_anchors.py | 5 +- yolo/ops/math_ops.py | 3 +- yolo/ops/preprocessing_ops.py | 7 +- yolo/run.py | 34 +- yolo/run_image.py | 30 +- yolo/tasks/backbone_test.py | 8 +- yolo/tasks/image_classification.py | 14 +- yolo/tasks/yolo.py | 20 +- yolo/tasks/yolo_subdiv.py | 17 +- yolo/tasks/yolo_test.py | 9 +- yolo/test.py | 2 +- yolo/train.py | 28 +- yolo/train_gilbreth.py | 17 +- yolo/train_vm.py | 17 +- yolo/utils/_darknet2tf/__init__.py | 3 +- yolo/utils/_darknet2tf/__main__.py | 11 +- yolo/utils/_darknet2tf/config_classes.py | 4 +- yolo/utils/_darknet2tf/dn2dicts.py | 1 - yolo/utils/_darknet2tf/dnnum.py | 3 +- yolo/utils/_darknet2tf/load_weights copy.py | 5 +- yolo/utils/_darknet2tf/load_weights.py | 3 + yolo/utils/_darknet2tf/load_weights2.py | 4 +- yolo/utils/_darknet2tf/read_weights.py | 3 +- yolo/utils/demos/coco.py | 4 +- yolo/utils/demos/coco_dataset_converter.py | 7 +- yolo/utils/demos/utils.py | 16 +- yolo/utils/downloads/__init__.py | 1 - yolo/utils/downloads/file_manager.py | 1 - yolo/utils/export/get_cfg_yaml.py | 3 +- yolo/utils/export/tensor_rt.py | 6 +- yolo/utils/export/tflite_convert.py | 9 +- yolo/utils/export/tflite_convert_gpu.py | 8 +- yolo/utils/export/tflite_gpu_test.py | 3 +- yolo/utils/export/tflite_inspect.py | 2 + yolo/utils/export/tflite_test.py | 3 +- yolo/utils/tests/test_darknet2tf.py | 2 +- yolo/utils/training/keras_task_trainer.py | 3 +- 623 files changed, 2590 insertions(+), 2392 deletions(-) create mode 100644 centernet/configs/experiments/centernet-eval.yaml create mode 100644 hooks/install.sh create mode 100644 hooks/pre-commit create mode 100644 hooks/readme.md create mode 100644 hooks/requirements.txt create mode 100644 hooks/screenshots/git_commit.png create mode 100644 hooks/screenshots/mv_command.png create mode 100644 hooks/screenshots/necessary_files.png create mode 100644 hooks/screenshots/readme.txt create mode 100644 linter_log.txt create mode 100644 pylint.sh diff --git a/.vscode/settings.json b/.vscode/settings.json index 14d906b92..49a4a6073 100755 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,3 @@ { - "python.pythonPath": "/usr/bin/python3.8" + "python.pythonPath": "C:\\Users\\david\\AppData\\Local\\Programs\\Python\\Python37\\python.exe" } \ No newline at end of file diff --git a/centernet/common/registry_imports.py b/centernet/common/registry_imports.py index bd7398f8b..101cd8be9 100755 --- a/centernet/common/registry_imports.py +++ b/centernet/common/registry_imports.py @@ -14,5 +14,10 @@ # ============================================================================== """All necessary imports for registration.""" +import centernet +from centernet.configs import centernet +from centernet.configs.centernet import CenterNetTask +from centernet.modeling.backbones import hourglass +from centernet.tasks.centernet import CenterNetTask # pylint: disable=unused-import from official.common import registry_imports diff --git a/centernet/configs/backbones.py b/centernet/configs/backbones.py index c7bb53388..a15d42972 100644 --- a/centernet/configs/backbones.py +++ b/centernet/configs/backbones.py @@ -14,10 +14,9 @@ # limitations under the License. # ============================================================================== """Backbones configurations.""" -from typing import List - # Import libraries import dataclasses +from typing import List from official.modeling import hyperparams from official.vision.beta.configs import backbones @@ -36,4 +35,4 @@ class Hourglass(hyperparams.Config): @dataclasses.dataclass class Backbone(backbones.Backbone): - hourglass: Hourglass = Hourglass() \ No newline at end of file + hourglass: Hourglass = Hourglass() diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 36e3dafaa..bc982ec02 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -14,15 +14,15 @@ # limitations under the License. # ============================================================================== """CenterNet configuration definition.""" -from typing import ClassVar, Dict, List, Optional, Tuple, Union - # Import libraries import dataclasses +from typing import ClassVar, Dict, List, Optional, Tuple, Union -from official.modeling import hyperparams +from centernet.configs import backbones +from official.core import exp_factory +from official.modeling import hyperparams, optimization from official.modeling.hyperparams import config_definitions as cfg from official.vision.beta.configs import common -from centernet.configs import backbones @dataclasses.dataclass @@ -66,22 +66,20 @@ class Parser(hyperparams.Config): @dataclasses.dataclass class DataConfig(cfg.DataConfig): """Input config for training.""" - input_path: str = 'D:/Datasets/coco/2017/1.1.0/val' #'gs://tensorflow2/coco_records/train/2017*' + input_path: str = 'D:\\Datasets\\coco\\2017\\1.1.0*' #'gs://tensorflow2/coco_records/train/2017*' tfds_name: str = None #'coco' - tfds_split: str = 'validation' #'train' + tfds_split: str = 'val' #'train' global_batch_size: int = 32 is_training: bool = True dtype: str = 'float16' decoder: DataDecoder = DataDecoder() parser: Parser = Parser() shuffle_buffer_size: int = 10000 - tfds_download: bool = True + tfds_download: bool = False -@dataclasses.dataclass class Loss(hyperparams.Config): pass - @dataclasses.dataclass class DetectionLoss(Loss): detection_weight: float = 1.0 @@ -105,6 +103,15 @@ class Losses(hyperparams.Config): class CenterNetDecoder(hyperparams.Config): heatmap_bias: float = -2.19 +@dataclasses.dataclass +class CenterNetLayer(hyperparams.Config): + max_detections: int = 100 + peak_error: float = 1e-6 + peak_extract_kernel_size: int = 3 + use_nms: bool = False + center_thresh: float = 0.1 + iou_thresh: float = 0.4 + class_offset: int = 1 @dataclasses.dataclass class CenterNetDetection(cfg.TaskConfig): @@ -112,7 +119,6 @@ class CenterNetDetection(cfg.TaskConfig): use_corners: bool = False predict_3d: bool = False - @dataclasses.dataclass class CenterNetSubTasks(cfg.TaskConfig): detection: CenterNetDetection = CenterNetDetection() @@ -126,15 +132,20 @@ class CenterNetSubTasks(cfg.TaskConfig): class CenterNetBase(hyperparams.OneOfConfig): backbone: backbones.Backbone = backbones.Backbone(type='hourglass') decoder: CenterNetDecoder = CenterNetDecoder() + odapi_weights: str = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' + extremenet_weights: str = 'D:\\weights\extremenet' + backbone_name: str = 'hourglass104_512' + decoder_name: str = 'detection_2d' @dataclasses.dataclass class CenterNet(hyperparams.Config): + base: Union[str, CenterNetBase] = CenterNetBase() num_classes: int = 90 gaussian_iou: float = 0.7 max_num_instances: int = 200 input_size: Optional[List[int]] = dataclasses.field( default_factory=lambda: [None, None, 3]) - base: Union[str, CenterNetBase] = CenterNetBase() + filter: CenterNetLayer = CenterNetLayer() @dataclasses.dataclass class CenterNetTask(cfg.TaskConfig): @@ -144,10 +155,14 @@ class CenterNetTask(cfg.TaskConfig): subtasks: CenterNetSubTasks = CenterNetSubTasks() losses: Losses = Losses() - annotation_file: Optional[str] = None per_category_metrics: bool = False - weight_decay: float = 5e-4 + + init_checkpoint: str = None + annotation_file: Optional[str] = None + gradient_clip_norm: float = 0.0 + load_odapi_weights: bool = True + load_extremenet_weights: bool = False def _get_output_length_dict(self): lengths = {} @@ -193,4 +208,74 @@ def _get_output_length_dict(self): # 'joint_offset': 2 # }) - return lengths \ No newline at end of file + return lengths + +@exp_factory.register_config_factory('centernet_custom') +def centernet_custom() -> cfg.ExperimentConfig: + """COCO object detection with CenterNet.""" + train_batch_size = 1 + eval_batch_size = 1 + base_default = 1200000 + num_batches = 1200000 * 64 / train_batch_size + + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig( + # mixed_precision_dtype='float16', + # loss_scale='dynamic', + num_gpus=2), + task=CenterNetTask( + model=CenterNet(), + train_data=DataConfig( # input_path=os.path.join( + # COCO_INPUT_PATH_BASE, 'train*'), + is_training=True, + global_batch_size=train_batch_size, + parser=Parser(), + shuffle_buffer_size=2), + validation_data=DataConfig( + # input_path=os.path.join(COCO_INPUT_PATH_BASE, + # 'val*'), + is_training=False, + global_batch_size=eval_batch_size, + shuffle_buffer_size=2)), + trainer=cfg.TrainerConfig( + steps_per_loop=2000, + summary_interval=8000, + checkpoint_interval=10000, + train_steps=num_batches, + validation_steps=1000, + validation_interval=10, + optimizer_config=optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [ + int(400000 / base_default * num_batches), + int(450000 / base_default * num_batches) + ], + 'values': [ + 0.00261 * train_batch_size / 64, + 0.000261 * train_batch_size / 64, + 0.0000261 * train_batch_size / 64 + ] + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 1000 * 64 // num_batches, + 'warmup_learning_rate': 0 + } + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + + return config diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml new file mode 100644 index 000000000..8099641fc --- /dev/null +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -0,0 +1,86 @@ +runtime: + distribution_strategy: mirrored + mixed_precision_dtype: float16 + loss_scale: "dynamic" + num_gpus: 1 +task: + model: + base: + backbone: + type: hourglass + decoder: + heatmap_bias: -2.19 + odapi_weights: 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' + extremenet_weights: 'D:\\weights\extremenet' + backbone_name: hourglass104_512 + decoder_name: detection_2d + num_classes: 90 + gaussian_iou: 0.7 + input_size: [512, 512, 3] # null + filter: + max_detections: 100 + peak_error: 0.000001 + peak_extract_kernel_size: 3 + use_nms: false + center_thresh: 0.1 + iou_thresh: 0.4 + class_offset: 1 + validation_data: + tfds_data_dir: 'D:\\Datasets' + tfds_name: coco + tfds_split: validation + global_batch_size: 8 + tfds_download: true + dtype: float16 + is_training: false + drop_remainder: true + parser: + fixed_size: true + image_h: 512 + image_w: 512 + letter_box: true + use_tie_breaker: true + shuffle_buffer_size: 10000 + subtasks: + detection: + use_centers: true + use_corners: false + predict_3d: false + segmentation: false + losses: + detection: + detection_weight: 1.0 + corner_pull_weight: 0.1 + corner_push_weight: 0.1 + offset_weight: 1.0 + segmentation: + per_category_metrics: false + weight_decay: 0.0005 + init_checkpoint: '' + annotation_file: null + gradient_clip_norm: 0.0 + load_odapi_weights: true + load_extremenet_weights: false +trainer: + train_steps: 10000 # 160 epochs at 64 batchsize -> 500500 * 64/16 + validation_steps: 625 # 5063 #625 + steps_per_loop: 10000 + validation_interval: 1 + summary_interval: 10000 + checkpoint_interval: 10000 + optimizer_config: + learning_rate: + type: stepwise + stepwise: + boundaries: [800000, 900000] # [400000, 450000] + name: PiecewiseConstantDecay + values: [0.00065, 0.000065, 0.0000065] #[0.0013, 0.00013, 0.000013] + optimizer: + type: sgd + sgd: + momentum: 0.949 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 1000 #learning rate rises from 0 to 0.0013 over 1000 steps \ No newline at end of file diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index ce9f27230..a2808334b 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -1,9 +1,10 @@ import tensorflow as tf -from official.vision.beta.dataloaders import parser from centernet.ops import preprocessing_ops +from official.vision.beta.dataloaders import parser from yolo.ops import preprocessing_ops as yolo_preprocessing_ops + class CenterNetParser(parser.Parser): def __init__(self, image_w: int = 512, diff --git a/centernet/dataloaders/centernet_input_test.py b/centernet/dataloaders/centernet_input_test.py index 753cf80a6..60e2cd85f 100755 --- a/centernet/dataloaders/centernet_input_test.py +++ b/centernet/dataloaders/centernet_input_test.py @@ -1,14 +1,16 @@ -from yolo.tasks import yolo -from yolo.configs import yolo as yolocfg +import dataclasses + +import matplotlib.pyplot as plt +import tensorflow as tf + +from official.core import config_definitions as cfg from official.core import input_reader +from official.modeling import hyperparams +from yolo.configs import yolo as yolocfg from yolo.dataloaders import yolo_input as YOLO_Detection_Input from yolo.dataloaders.decoders import tfds_coco_decoder from yolo.ops import box_ops -import matplotlib.pyplot as plt -import dataclasses -from official.modeling import hyperparams -from official.core import config_definitions as cfg -import tensorflow as tf +from yolo.tasks import yolo @dataclasses.dataclass diff --git a/centernet/losses/__init__.py b/centernet/losses/__init__.py index 9bf1c1685..259e5a25f 100755 --- a/centernet/losses/__init__.py +++ b/centernet/losses/__init__.py @@ -1,3 +1,3 @@ from centernet.losses.l1_localization_loss import L1LocalizationLoss from centernet.losses.penalty_reduced_logistic_focal_loss import \ - PenaltyReducedLogisticFocalLoss + PenaltyReducedLogisticFocalLoss diff --git a/centernet/losses/l1_localization_loss_test.py b/centernet/losses/l1_localization_loss_test.py index e85943857..6e1aa031e 100644 --- a/centernet/losses/l1_localization_loss_test.py +++ b/centernet/losses/l1_localization_loss_test.py @@ -1,8 +1,9 @@ -import tensorflow as tf import numpy as np +import tensorflow as tf from centernet import losses + class L1LocalizationLossTest(tf.test.TestCase): def test_returns_correct_loss(self): diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss.py b/centernet/losses/penalty_reduced_logistic_focal_loss.py index 8dca75efb..78dda198c 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss.py @@ -1,6 +1,7 @@ # may be a dupe of retinanet_losses. need to look later import tensorflow as tf + class PenaltyReducedLogisticFocalLoss(tf.keras.losses.Loss): """Penalty-reduced pixelwise logistic regression with focal loss. The loss is defined in Equation (1) of the Objects as Points[1] paper. diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py index 96cbd96ca..1f6d1f402 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss_test.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss_test.py @@ -1,5 +1,5 @@ -import tensorflow as tf import numpy as np +import tensorflow as tf import centernet.losses.penalty_reduced_logistic_focal_loss as losses diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index bcbc6877c..af8689c94 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -1,12 +1,11 @@ -from official.core import registry -from official.vision.beta.modeling.backbones import factory import tensorflow as tf import tensorflow.keras as ks -from centernet.modeling.backbones.hourglass import Hourglass -from centernet.modeling.backbones.hourglass import build_hourglass +from centernet.modeling.backbones.hourglass import Hourglass, build_hourglass from centernet.modeling.decoders.centernet_decoder import CenterNetDecoder from centernet.modeling.layers.detection_generator import CenterNetLayer +from official.core import registry +from official.vision.beta.modeling.backbones import factory # TODO: import prediction and filtering layers when made @@ -25,12 +24,6 @@ def __init__(self, self._head = head self._filter = filter return - - def build(self, input_shape): - self._backbone.build(input_shape) - nshape = self._backbone.output_specs - self._decoder.build(nshape) - super().build(input_shape) def call(self, inputs, training=False): features = self._backbone(inputs) @@ -67,15 +60,21 @@ def build_centernet_decoder(input_specs, task_config, num_inputs): # task specific task_outputs = task_config._get_output_length_dict() model = CenterNetDecoder( + input_specs = input_specs task_outputs=task_outputs, heatmap_bias=heatmap_bias, num_inputs=num_inputs) - model.build(input_specs) return model def build_centernet_filter(model_config): - return CenterNetLayer() + return CenterNetLayer(max_detections=model_config.filter.max_detections, + peak_error=model_config.filter.peak_error, + peak_extract_kernel_size=model_config.filter.peak_extract_kernel_size, + use_nms=model_config.filter.use_nms, + center_thresh=model_config.filter.center_thresh, + iou_thresh=model_config.filter.iou_thresh, + class_offset=model_config.filter.class_offset) def build_centernet_head(model_config): return None @@ -88,7 +87,7 @@ def build_centernet(input_specs, task_config, l2_regularization): backbone = factory.build_backbone(input_specs, model_config.base, l2_regularization) - decoder = build_centernet_decoder(backbone.output_specs.as_list(), task_config, backbone._num_hourglasses) + decoder = build_centernet_decoder(backbone.output_specs, task_config, backbone._num_hourglasses) head = build_centernet_head(model_config) filter = build_centernet_filter(model_config) @@ -99,4 +98,3 @@ def build_centernet(input_specs, task_config, l2_regularization): # losses = filter.losses losses = None return model, losses - diff --git a/centernet/modeling/CenterNet_test.py b/centernet/modeling/CenterNet_test.py index 2b207aa74..9996eefce 100644 --- a/centernet/modeling/CenterNet_test.py +++ b/centernet/modeling/CenterNet_test.py @@ -1,13 +1,13 @@ -from absl.testing import parameterized -import tensorflow as tf -import numpy as np import dataclasses -from official.modeling import hyperparams -from official.vision.beta.configs import backbones +import numpy as np +import tensorflow as tf +from absl.testing import parameterized -from centernet.modeling.CenterNet import build_centernet from centernet.configs import centernet +from centernet.modeling.CenterNet import build_centernet +from official.modeling import hyperparams +from official.vision.beta.configs import backbones class CenterNetTest(parameterized.TestCase, tf.test.TestCase): diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index a8afaddd9..de3ffd45e 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -1,14 +1,14 @@ +from typing import List + import tensorflow as tf from centernet.configs import backbones as cfg from centernet.modeling.layers import nn_blocks - # from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks +from official.vision.beta.modeling.layers import \ + nn_blocks as official_nn_blocks from utils import register -from typing import List - @tf.keras.utils.register_keras_serializable(package='centernet') class Hourglass(tf.keras.Model): @@ -140,7 +140,7 @@ def build_hourglass( input_specs: tf.keras.layers.InputSpec, model_config, l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: - """Builds ResNet backbone from a config.""" + """Builds Hourglass backbone from a config.""" backbone_type = model_config.backbone.type backbone_cfg = model_config.backbone.get() assert backbone_type == 'hourglass', (f'Inconsistent backbone type ' diff --git a/centernet/modeling/backbones/hourglass_test.py b/centernet/modeling/backbones/hourglass_test.py index 71be62258..ba6c598e1 100644 --- a/centernet/modeling/backbones/hourglass_test.py +++ b/centernet/modeling/backbones/hourglass_test.py @@ -1,11 +1,11 @@ -from absl.testing import parameterized -import tensorflow as tf +import dataclasses + import numpy as np +import tensorflow as tf +from absl.testing import parameterized from centernet.configs import backbones as cfg from centernet.modeling.backbones import hourglass - -import dataclasses from official.modeling import hyperparams from official.vision.beta.configs import backbones diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index aa9978a61..0862a60e7 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -1,13 +1,16 @@ import tensorflow as tf -from centernet.modeling.layers.nn_blocks import CenterNetDecoderConv + from centernet.configs import centernet as cfg +from centernet.modeling.layers.nn_blocks import CenterNetDecoderConv +@tf.keras.utils.register_keras_serializable(package='centernet') class CenterNetDecoder(tf.keras.Model): """ CenterNet Decoder """ def __init__(self, + input_specs task_outputs: dict, heatmap_bias: float = -2.19, num_inputs: int = 2, @@ -24,31 +27,43 @@ def __init__(self, dictionary where the keys-value pairs denote the names of the output and the respective output tensor """ - super().__init__(**kwargs) + # super().__init__(**kwargs) - self._task_outputs = task_outputs - self._heatmap_bias = heatmap_bias - self._num_inputs = num_inputs + # self._task_outputs = task_outputs + # self._heatmap_bias = heatmap_bias + # self._num_inputs = num_inputs - self.outputs_layers = {} + # self.outputs_layers = {} - def build(self, inputs): - for key in self._task_outputs: - num_filters = self._task_outputs[key] + inputs = [tf.keras.layers.Input(shape=value[1:]) for value in input_specs] + outputs = dict() + for key in task_outputs: + num_filters = self.task_outputs[key] bias = 0 if 'heatmaps' in key: bias = self._heatmap_bias + outputs[key] = [CenterNetDecoderConv(output_filters=num_filters, + name=key, bias_init=bias)() for _ in range(self.num_inputs)] - self.outputs_layers[key] = [CenterNetDecoderConv(output_filters=num_filters, - name=key, bias_init=bias) for _ in range(self._num_inputs)] + super().__init__(inputs=inputs, outputs=outputs, name='CenterNetDecoder') - def call(self, inputs): - outputs = {} + # def build(self, inputs): + # for key in self._task_outputs: + # num_filters = self._task_outputs[key] + # bias = 0 + # if 'heatmaps' in key: + # bias = self._heatmap_bias - for key in self._task_outputs: - outputs[key] = [self.outputs_layers[key][i](inputs[i]) for i in range(self._num_inputs)] + # self.outputs_layers[key] = [CenterNetDecoderConv(output_filters=num_filters, + # name=key, bias_init=bias) for _ in range(self._num_inputs)] + + # def call(self, inputs): + # outputs = dict() + + # for key in self._task_outputs: + # outputs[key] = [self.outputs_layers[key][i](inputs[i]) for i in range(self._num_inputs)] - return outputs + # return outputs def get_config(self): layer_config = { @@ -58,19 +73,3 @@ def get_config(self): #layer_config.update(super().get_config()) return layer_config - -def build_centernet_decoder(input_specs, task_config, num_inputs): - # NOTE: For now just support the default config - - # model specific - heatmap_bias = task_config.model.base.decoder.heatmap_bias - - # task specific - task_outputs = task_config._get_output_length_dict() - model = CenterNetDecoder( - task_outputs=task_outputs, - heatmap_bias=heatmap_bias, - num_inputs=num_inputs) - - model.build(input_specs) - return model \ No newline at end of file diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index 6a8d65464..e4dc819f3 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -1,6 +1,6 @@ -from absl.testing import parameterized -import tensorflow as tf import numpy as np +import tensorflow as tf +from absl.testing import parameterized from centernet.configs import centernet as cfg from centernet.modeling.decoders import centernet_decoder diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index bd90682ae..be9ae8e38 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -1,8 +1,10 @@ -from tensorflow import keras as ks import tensorflow as tf +from tensorflow import keras as ks from yolo.ops.nms_ops import nms + +@tf.keras.utils.register_keras_serializable(package='centernet') class CenterNetLayer(ks.Model): """ CenterNet Detection Generator @@ -323,4 +325,4 @@ def call(self, inputs): 'classes': classes, 'confidence': scores, 'num_dets': num_det - } \ No newline at end of file + } diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index db8bcf79f..05412e018 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -1,6 +1,8 @@ import tensorflow as tf + from official.modeling import tf_utils -from official.vision.beta.modeling.layers import nn_blocks as official_nn_blocks +from official.vision.beta.modeling.layers import \ + nn_blocks as official_nn_blocks @tf.keras.utils.register_keras_serializable(package='centernet') @@ -301,6 +303,7 @@ def get_config(self): layer_config.update(super().get_config()) return layer_config +@tf.keras.utils.register_keras_serializable(package='centernet') class CenterNetDecoderConv(tf.keras.layers.Layer): """ Convolution block for the CenterNet head. This is used to generate @@ -340,4 +343,4 @@ def call(self, x): x = self.conv1(x) x = self.relu(x) x = self.conv2(x) - return x \ No newline at end of file + return x diff --git a/centernet/modeling/layers/nn_blocks_test.py b/centernet/modeling/layers/nn_blocks_test.py index 66060e72e..87f61cad5 100644 --- a/centernet/modeling/layers/nn_blocks_test.py +++ b/centernet/modeling/layers/nn_blocks_test.py @@ -1,9 +1,8 @@ -from absl.testing import parameterized -import tensorflow as tf import numpy as np +import tensorflow as tf +from absl.testing import parameterized from centernet.modeling.layers import nn_blocks - # the following is a literal translation of the PyTorch implementation # https://github.com/xingyizhou/CenterNet/blob/master/src/lib/models/networks/large_hourglass.py from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock diff --git a/centernet/modeling/plot_models.py b/centernet/modeling/plot_models.py index a981cbc48..1a42b25bb 100644 --- a/centernet/modeling/plot_models.py +++ b/centernet/modeling/plot_models.py @@ -2,10 +2,9 @@ from centernet.configs import centernet as cfg from centernet.modeling.backbones import hourglass -from centernet.modeling.layers import nn_blocks -from centernet.modeling.decoders import centernet_decoder from centernet.modeling.CenterNet import build_centernet - +from centernet.modeling.decoders import centernet_decoder +from centernet.modeling.layers import nn_blocks BACKBONE_OUT = "./centernet/modeling/backbone.png" DECODER_OUT = "./centernet/modeling/decoder.png" @@ -38,5 +37,3 @@ hg_block.summary() model.summary() - - \ No newline at end of file diff --git a/centernet/ops/loss_ops.py b/centernet/ops/loss_ops.py index 8aaa1f9fb..227fd0ebc 100644 --- a/centernet/ops/loss_ops.py +++ b/centernet/ops/loss_ops.py @@ -1,5 +1,6 @@ import tensorflow as tf + def _to_float32(x): return tf.cast(x, tf.float32) @@ -65,4 +66,3 @@ def get_batch_predictions_from_indices(batch_predictions, indices): values at the given indices. """ return tf.gather_nd(batch_predictions, indices) - diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 7c6f36988..d9fc4891e 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -1,7 +1,7 @@ # Moved groundtruth.py here import tensorflow as tf -from yolo.ops import preprocessing_ops +from yolo.ops import preprocessing_ops LARGE_NUM = 1. / tf.keras.backend.epsilon() diff --git a/centernet/ops/preprocessing_ops_test.py b/centernet/ops/preprocessing_ops_test.py index 44894df35..9245e2e15 100644 --- a/centernet/ops/preprocessing_ops_test.py +++ b/centernet/ops/preprocessing_ops_test.py @@ -1,10 +1,11 @@ -from absl.testing import parameterized -import tensorflow as tf import numpy as np +import tensorflow as tf +from absl.testing import parameterized import centernet.utils.groundtruth as preprocessing_ops from yolo.ops import box_ops + def image_shape_to_grids(height, width): """Computes xy-grids given the shape of the image. Args: diff --git a/centernet/tasks/_centernet_input.py b/centernet/tasks/_centernet_input.py index 8321b1b23..c07630cdd 100644 --- a/centernet/tasks/_centernet_input.py +++ b/centernet/tasks/_centernet_input.py @@ -1,20 +1,15 @@ import tensorflow as tf -from tensorflow.keras.mixed_precision import experimental as mixed_precision - from absl import logging -from official.core import base_task -from official.core import input_reader -from official.core import task_factory -from yolo.configs import yolo as exp_cfg +from tensorflow.keras.mixed_precision import experimental as mixed_precision +from centernet.dataloaders import centernet_input +from official.core import base_task, input_reader, task_factory from official.vision.beta.evaluation import coco_evaluator - +from yolo.configs import yolo as exp_cfg from yolo.dataloaders import yolo_input from yolo.dataloaders.decoders import tfds_coco_decoder -from yolo.ops.kmeans_anchors import BoxGenInputReader from yolo.ops.box_ops import xcycwh_to_yxyx - -from centernet.dataloaders import centernet_input +from yolo.ops.kmeans_anchors import BoxGenInputReader @task_factory.register_task_cls(exp_cfg.YoloTask) diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 11b49364b..1645dfc57 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -1,29 +1,29 @@ import tensorflow as tf -from tensorflow.keras.mixed_precision import experimental as mixed_precision - from absl import logging -from official.core import base_task -from official.core import input_reader -from official.core import task_factory - -from official.vision.beta.evaluation import coco_evaluator -from official.vision.beta.dataloaders import tf_example_decoder -from official.vision.beta.dataloaders import tfds_detection_decoders -from official.vision.beta.dataloaders import tf_example_label_map_decoder +from tensorflow.keras.mixed_precision import experimental as mixed_precision -from centernet.configs import centernet as cfg +from centernet.configs import centernet as exp_cfg from centernet.dataloaders import centernet_input -from centernet.modeling.CenterNet import build_centernet +from centernet.losses import (l1_localization_loss, + penalty_reduced_logistic_focal_loss) from centernet.ops import loss_ops -from centernet.losses import penalty_reduced_logistic_focal_loss -from centernet.losses import l1_localization_loss +from official.core import base_task, input_reader, task_factory +from official.vision.beta.dataloaders import (tf_example_decoder, + tf_example_label_map_decoder, + tfds_detection_decoders) +from official.vision.beta.evaluation import coco_evaluator -@task_factory.register_task_cls(cfg.CenterNetTask) +@task_factory.register_task_cls(exp_cfg.CenterNetTask) class CenterNetTask(base_task.Task): def __init__(self, params, logging_dir: str = None): super().__init__(params, logging_dir) + self._loss_dict = None + + self.coco_metric = None + self._metric_names = [] + self._metrics = [] def build_inputs(self, params, input_context=None): """Build input dataset.""" @@ -52,6 +52,7 @@ def build_inputs(self, params, input_context=None): def build_model(self): """get an instance of CenterNet""" + from centernet.modeling.CenterNet import build_centernet params = self.task_config.train_data model_base_cfg = self.task_config.model l2_weight_decay = self.task_config.weight_decay / 2.0 @@ -65,6 +66,31 @@ def build_model(self): self._loss_dict = losses return model + def initialize(self, model: tf.keras.Model): + """Initializes CenterNet model by loading pretrained weights """ + if self.task_config.load_odapi_weights and self.task_config.load_extremenet_weights: + raise ValueError('Only 1 of odapi or extremenet weights should be loaded') + + if self.task_config.load_odapi_weights or self.task_config.load_extremenet_weights: + from centernet.utils.weight_utils.tf_to_dict import get_model_weights_as_dict + from centernet.utils.weight_utils.load_weights import load_weights_model + from centernet.utils.weight_utils.load_weights import load_weights_backbone + + # Load entire model weights from the ODAPI checkpoint + if self.task_config.load_odapi_weights: + weights_file = self.task_config.model.base.odapi_weights + weights_dict, n_weights = get_model_weights_as_dict(weights_file) + load_weights_model(model, weights_dict, + backbone_name=self.task_config.model.base.backbone_name, + decoder_name=self.task_config.model.base.decoder_name) + + # Load backbone weights from ExtremeNet + else: + weights_file = self.task_config.model.base.extremenet_weights + weights_dict, n_weights = get_model_weights_as_dict(weights_file) + load_weights_backbone(model.backbone, weights_dict['feature_extractor'], + backbone_name=self.task_config.model.base.backbone_name) + def get_decoder(self, params): if params.tfds_name: if params.tfds_name in tfds_detection_decoders.TFDS_ID_TO_DECODER_MAP: @@ -185,6 +211,3 @@ def _get_masks(self, exp_base=2, xy_scale_base='default_value'): pass - - def initialize(self, model: tf.keras.Model): - pass diff --git a/centernet/tasks/centernet_object_detection_test.py b/centernet/tasks/centernet_object_detection_test.py index 1db8720ef..c94df866e 100644 --- a/centernet/tasks/centernet_object_detection_test.py +++ b/centernet/tasks/centernet_object_detection_test.py @@ -1,11 +1,13 @@ -import tensorflow as tf import numpy as np -import centernet.tasks as tasks +import tensorflow as tf + import centernet.ops.loss_ops as utils -from centernet.losses import penalty_reduced_logistic_focal_loss -from centernet.losses import l1_localization_loss +import centernet.tasks as tasks +from centernet.losses import (l1_localization_loss, + penalty_reduced_logistic_focal_loss) from centernet.tasks.centernet import CenterNetTask + def gaussian2D(shape, sigma=1): m, n = [(ss - 1.) / 2. for ss in shape] y, x = np.ogrid[-m:m+1,-n:n+1] diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index 4c2dcf774..e53f9e6ac 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -1,19 +1,18 @@ -from absl.testing import parameterized -import tensorflow as tf -import numpy as np import dataclasses -import orbit - -from official.modeling import hyperparams -from official.vision.beta.configs import backbones +import numpy as np +import tensorflow as tf +import tensorflow_datasets as tfds +from absl.testing import parameterized -from centernet.tasks.centernet import CenterNetTask +import orbit from centernet.configs import centernet as exp_cfg -from centernet.utils.weight_utils.load_weights import get_model_weights_as_dict, load_weights_model from centernet.dataloaders.centernet_input import CenterNetParser - -import tensorflow_datasets as tfds +from centernet.tasks.centernet import CenterNetTask +from centernet.utils.weight_utils.load_weights import ( + get_model_weights_as_dict, load_weights_model) +from official.modeling import hyperparams +from official.vision.beta.configs import backbones CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' @@ -40,20 +39,22 @@ def testCenterNetValidation(self): task = CenterNetTask(config) model = task.build_model() + task.initialize(model) metrics = task.build_metrics(training=False) strategy = tf.distribute.get_strategy() - - weights_dict, _ = get_model_weights_as_dict(CENTERNET_CKPT_PATH) - load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') dataset = orbit.utils.make_distributed_dataset(strategy, task.build_inputs, config.validation_data) - #val_data = task.build_inputs(config.validation_data) - iterator = iter(dataset) - logs = task.validation_step(next(iterator), model, metrics=metrics) - print(logs) + # tf.print("Dataset: ") + # tf.print(dataset) + # iterator = iter(dataset) + # tf.print("Getting logs: ") + # logs = task.validation_step(next(iterator), model, metrics=metrics) + # tf.print("logs: ") + # tf.print(logs) + if __name__ == '__main__': - tf.test.main() \ No newline at end of file + tf.test.main() diff --git a/centernet/tasks/show_heatmaps.py b/centernet/tasks/show_heatmaps.py index 69a5f23e3..6a138d686 100644 --- a/centernet/tasks/show_heatmaps.py +++ b/centernet/tasks/show_heatmaps.py @@ -1,8 +1,10 @@ -import tensorflow as tf import numpy as np +import tensorflow as tf + import centernet.tasks as tasks import centernet.utils as utils + def gaussian2D(shape, sigma=1): m, n = [(ss - 1.) / 2. for ss in shape] y, x = np.ogrid[-m:m+1,-n:n+1] diff --git a/centernet/train.py b/centernet/train.py index b00fc5986..5860accfa 100755 --- a/centernet/train.py +++ b/centernet/train.py @@ -1,4 +1,4 @@ -# Lint as: python3 +# Lint as: python3 # Copyright 2020 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,42 +14,101 @@ # limitations under the License. # ============================================================================== """TensorFlow Model Garden Vision training driver.""" -from yolo.utils.run_utils import prep_gpu -try: - prep_gpu() -except BaseException: - print('GPUs ready') +import pprint +import sys -from absl import app -from absl import flags import gin +from absl import app, flags -from official.core import train_utils # pylint: disable=unused-import -from yolo.common import registry_imports +from centernet.common import registry_imports # pylint: enable=unused-import from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib +from official.core import task_factory, train_lib, train_utils from official.modeling import performance +from yolo.utils.run_utils import prep_gpu + +try: + prep_gpu() +except BaseException: + print('GPUs ready') + + FLAGS = flags.FLAGS """ -python3 -m yolo.train --mode=train_and_eval --experiment=darknet_classification --model_dir=training_dir --config_file=yolo/configs/experiments/darknet53.yaml +export LD_LIBRARY_PATH=/usr/local/cuda/lib64 """ """ -python3 -m yolo.train --mode=train_and_eval --experiment=yolo_v4_coco --model_dir=training_dir --config_file=yolo/configs/experiments/yolov4.yaml +get the cache file: +scp -i ./jaeyounkim-purdue-1 cache.zip purdue@34.105.118.198:~/ +train darknet: +python3 -m yolo.train --mode=train_and_eval --experiment=darknet_classification --model_dir=../checkpoints/darknet53 --config_file=yolo/configs/experiments/darknet53.yaml +python3.8 -m yolo.train --mode=train_and_eval --experiment=darknet_classification --model_dir=../checkpoints/dilated_darknet53 --config_file=yolo/configs/experiments/dilated_darknet53.yaml +finetune darknet: +nohup python3 -m yolo.train --mode=train_and_eval --experiment=darknet_classification --model_dir=../checkpoints/darknet53_remap_fn --config_file=yolo/configs/experiments/darknet53_leaky_fn_tune.yaml >> darknet53.log & tail -f darknet53.log +train yolo-v4: +nohup python3 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov4- --config_file=yolo/configs/experiments/yolov4.yaml >> yolov4.log & tail -f yolov4.log +nohup python3.8 -m yolo.train --mode=train_and_eval --experiment=yolo_subdiv_custom --model_dir=../checkpoints/yolov4_subdiv64 --config_file=yolo/configs/experiments/yolov4_subdiv.yaml >> yolov4.log & tail -f yolov4.log +nohup python3 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov4- --config_file=yolo/configs/experiments/yolov4-1gpu.yaml >> yolov4-1gpu.log & tail -f yolov4-1gpu.log +nohup python3.8 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov3-1gpu_mosaic --config_file=yolo/configs/experiments/yolov3-1gpu.yaml >> yolov3-1gpu.log & tail -f yolov3-1gpu.log +evalaute CenterNet: +nohup python -m centernet.train --mode=train_and_eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval.yaml >> centernet-eval.log & tail -f centernet-eval.log """ +def subdivison_adjustment(params): + + if hasattr(params.task.model, + 'subdivisions') and params.task.model.subdivisions > 1: + print('adjustment is needed') + subdivisons = params.task.model.subdivisions + params.task.train_data.global_batch_size //= subdivisons + # params.task.validation_data.global_batch_size //= subdivisons + params.trainer.train_steps *= subdivisons + # params.trainer.validation_steps *= subdivisons + params.trainer.validation_interval = (params.trainer.validation_interval // + subdivisons) * subdivisons + params.trainer.checkpoint_interval = (params.trainer.checkpoint_interval // + subdivisons) * subdivisons + params.trainer.steps_per_loop = (params.trainer.steps_per_loop // + subdivisons) * subdivisons + params.trainer.summary_interval = (params.trainer.summary_interval // + subdivisons) * subdivisons + + if params.trainer.optimizer_config.learning_rate.type == 'stepwise': + bounds = params.trainer.optimizer_config.learning_rate.stepwise.boundaries + params.trainer.optimizer_config.learning_rate.stepwise.boundaries = [ + subdivisons * bound for bound in bounds + ] + + if params.trainer.optimizer_config.learning_rate.type == 'polynomial': + params.trainer.optimizer_config.learning_rate.polynomial.decay_steps *= subdivisons + + if params.trainer.optimizer_config.optimizer.type == 'sgd': + print(params.trainer.optimizer_config.optimizer.type) + params.trainer.optimizer_config.optimizer.type = 'sgd_accum' + params.trainer.optimizer_config.optimizer.sgd_accum.accumulation_steps = subdivisons + params.trainer.optimizer_config.optimizer.sgd_accum.momentum = params.trainer.optimizer_config.optimizer.sgd.momentum + params.trainer.optimizer_config.optimizer.sgd_accum.decay = params.trainer.optimizer_config.optimizer.sgd.decay + + if params.trainer.optimizer_config.warmup.type == 'linear': + params.trainer.optimizer_config.warmup.linear.warmup_steps *= subdivisons + + print(params.as_dict()) + # sys.exit() + return params + + def main(_): gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params) print(FLAGS.experiment) params = train_utils.parse_configuration(FLAGS) + params = subdivison_adjustment(params) model_dir = FLAGS.model_dir - if 'train' in FLAGS.mode: + if 'train' in FLAGS.mode and model_dir != None: # Pure eval modes do not output yaml files. Otherwise continuous eval job # may race against the train job for writing the same file. train_utils.serialize_config(params, model_dir) @@ -61,11 +120,17 @@ def main(_): if params.runtime.mixed_precision_dtype: performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype, params.runtime.loss_scale) + if params.runtime.worker_hosts != '' and params.runtime.worker_hosts is not None: + num_workers = distribute_utils.configure_cluster( + worker_hosts=params.runtime.worker_hosts, + task_index=params.runtime.task_index) + print(num_workers) distribution_strategy = distribute_utils.get_distribution_strategy( distribution_strategy=params.runtime.distribution_strategy, all_reduce_alg=params.runtime.all_reduce_alg, num_gpus=params.runtime.num_gpus, tpu_address=params.runtime.tpu) + with distribution_strategy.scope(): task = task_factory.get_task(params.task, logging_dir=model_dir) diff --git a/centernet/utils/weight_utils/config_classes.py b/centernet/utils/weight_utils/config_classes.py index 06a878d92..1ffc5a565 100644 --- a/centernet/utils/weight_utils/config_classes.py +++ b/centernet/utils/weight_utils/config_classes.py @@ -1,8 +1,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field +from typing import ClassVar, Dict, List, Tuple + import numpy as np -from typing import Tuple, List, Dict, ClassVar class Config(ABC): def get_weights(self): @@ -201,4 +202,4 @@ def get_weights(self): self.conv_1_bias, self.conv_2_weights, self.conv_2_bias - ] \ No newline at end of file + ] diff --git a/centernet/utils/weight_utils/config_data.py b/centernet/utils/weight_utils/config_data.py index 081a5dfd0..de3f82c42 100644 --- a/centernet/utils/weight_utils/config_data.py +++ b/centernet/utils/weight_utils/config_data.py @@ -1,7 +1,11 @@ from dataclasses import dataclass, field from typing import Dict -from centernet.utils.weight_utils.config_classes import convBnCFG, residualBlockCFG, hourglassCFG, decoderConvCFG +from centernet.utils.weight_utils.config_classes import (convBnCFG, + decoderConvCFG, + hourglassCFG, + residualBlockCFG) + @dataclass class BackboneConfigData(): @@ -63,4 +67,3 @@ def __post_init__(self): def get_cfg_list(self, name): if name == 'detection_2d': return self.detection_2d - diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index e9947bf34..799310037 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -1,15 +1,18 @@ -import tensorflow as tf import numpy as np +import tensorflow as tf -from centernet.modeling.CenterNet import build_centernet from centernet.configs.centernet import CenterNetTask - -from centernet.utils.weight_utils.tf_to_dict import get_model_weights_as_dict, write_dict_as_tree -from centernet.utils.weight_utils.config_classes import convBnCFG, residualBlockCFG, hourglassCFG, decoderConvCFG -from centernet.utils.weight_utils.config_data import BackboneConfigData, DecoderConfigData - -from centernet.modeling.layers.nn_blocks import ConvBN, HourglassBlock, CenterNetDecoderConv - +from centernet.modeling.CenterNet import build_centernet +from centernet.modeling.layers.nn_blocks import (CenterNetDecoderConv, ConvBN, + HourglassBlock) +from centernet.utils.weight_utils.config_classes import (convBnCFG, + decoderConvCFG, + hourglassCFG, + residualBlockCFG) +from centernet.utils.weight_utils.config_data import (BackboneConfigData, + DecoderConfigData) +from centernet.utils.weight_utils.tf_to_dict import ( + get_model_weights_as_dict, write_dict_as_tree) from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' @@ -146,4 +149,4 @@ def load_weights_decoder(decoder, weights_dict, decoder_name): # that are not weights to the model # Uncomment line below to write weights dict key names to a file - # write_dict_as_tree(weights_dict, filename="centernet/utils/weight_utils/MODEL_VARS.txt") \ No newline at end of file + # write_dict_as_tree(weights_dict, filename="centernet/utils/weight_utils/MODEL_VARS.txt") diff --git a/centernet/utils/weight_utils/test_load_weights.py b/centernet/utils/weight_utils/test_load_weights.py index 4f7b79776..4fd9dc4ba 100644 --- a/centernet/utils/weight_utils/test_load_weights.py +++ b/centernet/utils/weight_utils/test_load_weights.py @@ -1,13 +1,11 @@ -from yolo.demos.video_detect_gpu import FastVideo -from yolo.demos.video_detect_cpu import runner +import tensorflow as tf -from centernet.modeling.CenterNet import build_centernet from centernet.configs.centernet import CenterNetTask - -from centernet.utils.weight_utils.load_weights import get_model_weights_as_dict, load_weights_model -from centernet.utils.weight_utils.load_weights import load_weights_backbone - -import tensorflow as tf +from centernet.modeling.CenterNet import build_centernet +from centernet.utils.weight_utils.load_weights import ( + get_model_weights_as_dict, load_weights_backbone, load_weights_model) +from yolo.demos.video_detect_cpu import runner +from yolo.demos.video_detect_gpu import FastVideo CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' EXTREMENET_CKPT_PATH = 'D:\\weights\extremenet' @@ -45,4 +43,3 @@ # wait_time='dynamic') # cap.run() # runner(model, 0, 512, 512) - diff --git a/centernet/utils/weight_utils/tf_to_dict.py b/centernet/utils/weight_utils/tf_to_dict.py index 4b6465181..83481d014 100644 --- a/centernet/utils/weight_utils/tf_to_dict.py +++ b/centernet/utils/weight_utils/tf_to_dict.py @@ -1,5 +1,6 @@ import tensorflow as tf + def update_weights_dict(weights_dict, variable_key, value): """ Inserts weight value into a weight dictionary. @@ -88,4 +89,4 @@ def print_layer_weights_and_shape(layer): variables = layer.variables for i in range(len(weights)): - tf.print(np.shape(weights[i]), variables[i].name) \ No newline at end of file + tf.print(np.shape(weights[i]), variables[i].name) diff --git a/fixefficientnet/train.py b/fixefficientnet/train.py index b00fc5986..111646212 100755 --- a/fixefficientnet/train.py +++ b/fixefficientnet/train.py @@ -14,25 +14,24 @@ # limitations under the License. # ============================================================================== """TensorFlow Model Garden Vision training driver.""" +import gin +from absl import app, flags + +# pylint: enable=unused-import +from official.common import distribute_utils +from official.common import flags as tfm_flags +from official.core import task_factory, train_lib, train_utils +from official.modeling import performance +# pylint: disable=unused-import +from yolo.common import registry_imports from yolo.utils.run_utils import prep_gpu + try: prep_gpu() except BaseException: print('GPUs ready') -from absl import app -from absl import flags -import gin -from official.core import train_utils -# pylint: disable=unused-import -from yolo.common import registry_imports -# pylint: enable=unused-import -from official.common import distribute_utils -from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib -from official.modeling import performance FLAGS = flags.FLAGS """ diff --git a/hooks/install.sh b/hooks/install.sh new file mode 100644 index 000000000..f5c907b93 --- /dev/null +++ b/hooks/install.sh @@ -0,0 +1,13 @@ +#!bin/bash +# +# Script that installs the requirements for custom +# git precommit hook and creates symbolic link between +# hook in the repository and local git hook + +printf ">>> Installing required python packages...\n" +python -m pip install -r requirements.txt + +printf "\n>>> Creating symbolic link between pre-commit file and local git pre-commit hook..." +ln -s -f ../../hooks/pre-commit .git/hooks/pre-commit + +printf "\n\nInstallation complete." \ No newline at end of file diff --git a/hooks/pre-commit b/hooks/pre-commit new file mode 100644 index 000000000..cffbfcb96 --- /dev/null +++ b/hooks/pre-commit @@ -0,0 +1,23 @@ + +#!/bin/bash +# +# Git hook that runs whenever a user types "git commit"; +# It runs pylint on any files that were changed, saves the linter +# output into a .txt file, prints the contents of the file out, +# and adds the file to the commit. + +echo ">>> Running isort on your project repo..." +isort --recursive . + +echo +echo ">>> Running mypy on your project repo..." +mypy . + +echo +echo ">>> Running pylint on your changed python files -- please wait..." +bash pylint.sh > linter_log.txt +cat linter_log.txt + +git add . + +exit 0 \ No newline at end of file diff --git a/hooks/readme.md b/hooks/readme.md new file mode 100644 index 000000000..437b5c63f --- /dev/null +++ b/hooks/readme.md @@ -0,0 +1,55 @@ +# About +This folder contains a pre-commit hook that runs the following +modules in bash when a user types "git commit": +- isort +- mypy +- pylint.sh + +# Dependencies +You will need to have the following installed on your machine: +- isort +- mypy +- pylint + +They will be installed through the ```install.sh``` script, or you can +install them manually. + +# Setup +## Required Files +You will need to have the following in the root directory of your project: +- ```hooks``` directory, which contains the ```pre-commit``` hook script +- ```pylint.sh``` script +- ```pylintrc``` config file + +The files are located [here](https://github.com/PurdueCAM2Project/TensorFlowModels/tree/yolo_debug_linter) +in the root folder of the ```yolo_linter_debug``` branch: + +![necessary_files](./screenshots/necessary_files.png) + +## install.sh +You can easily set everything up by navigating into the ```hooks``` folder and running the +install script with the following command in your terminal: ```bash install.sh```. + +__NOTE__: make sure your ```install.sh``` file is set up as an executable; if it isn't, run +the command ```chmod +x install.sh```. + +## Manual Installation +If you want to set things up manually, what you can do is the following: +- copy and paste the ```pre-commit``` script into your local ```.git/hooks``` directory like so: +![mv_command](./screenshots/mv_command.png) +, OR +- create a symbolic link from the ```hooks``` directory to the ```.git/hooks``` directory +using the following command (run the command in project root directory): +```ln -s -f ../../hooks/pre-commit .git/hooks/pre-commit``` + +And that's it! Now when you run ```git commit``` command on your local machine, +you will see the following output: + +![git_commit](./screenshots/git_commit.png) + +# Important Notes +- You need to make sure the ```pre-commit``` and the ```pylint.sh``` files are +executables; if they're not, use the command ```sudo chmod +x {FILENAME}``` to +make them executables +- If you run into permission issues with "git add . " command when running the pre-commit hook, +use the following command to fix the issue: ```sudo chown -R "${USER:-$(id -un)}" . ``` \ No newline at end of file diff --git a/hooks/requirements.txt b/hooks/requirements.txt new file mode 100644 index 000000000..df907d2c9 --- /dev/null +++ b/hooks/requirements.txt @@ -0,0 +1,3 @@ +isort==4.3.21 +mypy==0.812 +pylint==2.4.2 \ No newline at end of file diff --git a/hooks/screenshots/git_commit.png b/hooks/screenshots/git_commit.png new file mode 100644 index 0000000000000000000000000000000000000000..3e11ee648700dae58f1b84766bb0a7b280e7dcef GIT binary patch literal 155515 zcmV)LK)Jt(P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGmbN~PnbOGLG zA9w%&02y>eSaefwW^{L9a%BK#Zf|X6EpuaXWo2%2Xm51y1HV210RM1FL_t(|UhJI* zU=&3c$NyJyx#ZF-354Ez6QqMQMa7C>7wn4tv-jTh^RuC%SU^QlRC<*nMS3rxlaN9} zdUCnTm2YNmH`%-F-R&h8ioPGb%iheJH*aQVcjw;C?P^-3wRPgOaGI(K7>)SXg{In1 z_jEOQTJlHWHPoPSE3D2^UA5Husm>+$47Il9crQ~s$ESTSnf|Q!w5fngUqvY&DuLFi z47cL@IMn#9$@}r)WdHovaJuk!9Q*R}>}kb60e>T^?zmf4tVf>6cB#hw#3@oUR-`fP z#hMI%-6w7a0B)7 zrViBwfcpA9hkz^Hm0B~;QlkEYw(b^um#n{MF1VfA{lvC>Rlo}<qs+7+Tv?KWXM26tSPj?mQ;llmp2+PfdFGiGOq6W< z^2axlwrD|kTVo+GTguee2A^Zp2d7(c-?oC$UI}Ge>T9T_siJ7@zVbFQd#AF3`ud^1ynjuf8g+2AzHO86ZKv+FxF1h}Q@`RhrR(WN!0iaUK6wdW#~Yz( z-Sl(?R?l7h2b+B3zqGrCs@h~qfqSQ1LaiNlJ^!VuHPluhDymjz1fnb9wqN}iP;b9R zGZ3HTy8M82E6amH(pgh6FysMKRioBWxIEB{xgak2z?TJ?VO+<^h3WYtA|t|y*W z`FnTkb#Ph9$XS5YJ~{r~RNs(5n(Xr|rjVtq9Y~F}@n70qqt3O}&&?VF9*FhBy}IC3 zUUw_%OV+IpS1IAuj{$Y|>*GZr$V#xyy46~P_wx0%>>@Z;Y6xV`>5E|d)zI)$iZ`F@ zbh-(!kbiaG8g=r~2k!PftgEOj5PxcP11aT>&yrH6w1d4U*VT`}wCjTW>h3v8{WeCC z`uJSmz`HN%WZ6fdVEI~KE4;jX&#AKKMTxgbI#1WPef24=uB=Q;MFF+`)W`3Jz}vOs z^bmOMnl{baoo#)1PEfn6{dazS28Y+ODk5m=>#eO>7oWZRD*hFydw4;)oNsG&uFJz` zZ`9R3$5pmI7HV^*1FN3ZyPae_6_g4HC@H#9-0X=c*Xh!h^5hmJ8{X`3JZ*X++4E*s z!khg%BRA;gq`#W{wWu~qCA`%Vxb?cBT#vfiB`wb`RO(XCz^i(cOSx8N)y*P@mvl}0 z}`=jnPcp_O=vgj*Mj;D8ZCooIiwf)-hP^gC`0d-p5aFwo$jlfr9)X&qPA|F=# zK=hztu5sLGO&=&%5uA#BTaocRH8 z?z3KZ>TgA|ARl9X#+&eMqdZ^UsWKFjy=XQlf!FD~gpga0h6Kx>y0{Qn8ByD#P+jWv z#J3IgHF#>uM{s?pRld#!YYNuaCnfyb^wf|Yx3Wn=L-@uCrrib&G}ptlx}t$q;8WX= zsd_X;puXk*$@sV`DxzuIyuDk0%HMEf%ANhs+Xb~dF0if)wBzN}XR^iTl6-IYw&0#y zKVKyRx1zd)Owqke;B|8Rm#WdIOAYEi;MG>DOI>P^+jWf%DaGG$>(3+qBh6bKo~cS! zluB|vDP7u@|CMjKkZZf+%Bcym*5C?Lxwh(t+zTa-_oifqt2#JBooZMY}2(}^CBW|hkRQ-yo6NJn}xbr4kp|K-+sC@ zYVw0g&4k*sX74qr^TD^4IvX++3b-fvw-|7pwWz;^V2!rZZKalGJst30Jg^GfwSANo zvjVg3i`tfgVcWNj`s42d5of*xwcoJDd&&4zdC}S?*q42sY~saJbZ6$c%aS zis#gfhZ?+>kfTn^Kf$uYkq!ZMUS?y7aRV|sH{Up<+W7;tvxVYWqnxilBK@R8cn{%k3#ijgF~I` zlN^>&|FekPg5=r1_uN#2(#UAYev#LuR8QSTjWlGWzZ_MTJeA}wRp)uPq^gdf!_VYW zYdN)BENpo?r`R?b0d<@=cwu=(=Qyk>HLa<8y89}3D@b@Qvft?{`!(OVWD^!2Q`zJ7DuZ3M%j3q?z+%zG-y z&ki+t!SmD|e|DQOjpsC0q(nzT7aI*tTnx1Fad7a368QzFI&}&aS-HqMRfxC3W09%j zVORwMY`~dnuX<`M)oigosl!~bK0XDnJ@w~ozOE^uY3~+TJ^SF_MoIYmsgDAImQ~c3 z74vSvw{8D5`$nmJi&iag$GvwVwMnX&qK*?8CouJsFL31W5ivzQb)6`yFXgW3_D=T3 z>8$zwN~6(=TXnBT30EC1J*|qdOFza!lWU;1pZ+@H`czW-v zX9-^fXWU6v0vO%$BvvfI%SXVtLa35|uFBL@8JJsKo7W*P<@uhYlolvbTO^e1xwqGa zXPT#-hC_W#-Zg{at-B5Wj`xKuRb?fC`Ql$YFni|?$|K#^_){*Sgf}ZbEx2X!H}cZn zOX~G{Xm!eS8jV&1dzBqFn@vn{$EEAtwpxN|!KbYNDsrj%`(;T0231lwi@cQbkrwbA zI!H|le$OaVtJTWgpo}*s1F<1ehaw)hTd2Q1Wop~?NC}|g`ueC`K5(n`_M?^oe-@Pn z&gu8tAHMCYs9mPgkiOQNd_%g&Xuv7@DU6F;TFeDg;VO;Vdo|Oljj{fT&1l zk`hssdlDr(_u^-*0l%A~U8`4BR9D|V;^n20)lj!1s6(u@8mTDvF>o`=C6rRHo^UnB zjJ-&ho5(Sn&G>!RADBO9zL;_*I9*pZ7iw$MbZFJF9=nzg42Q5YqzVHKesa78K~@>*SEtoNr_&36)u2>XD+J->bxF3Ia4=dK@keH^z^bu*!ed>_e=$ZiFCTz=+1x!&92n{nLKPwkji&ac@;nMXiQFq+!dG=S` z2*`qIm8}9LH zE1*hhl!N7)WQUK^VA*s>oz-C0?emgSC%hXBs^8+@I>QWHW4GY?Fg?OrHHV!CQ;!|z z7fw+@gmn%6attkgP$B;*3NA3V3x~ zPP%TQD^aFov+;dVCG1o0hn{Oc^Zmg821?!f$yJJfDB=65@AE%ma%Veqdp^Z=&;JF3 z-r#pkNcdCZePgYvLZ9b<5bO8^H@vX6dMwn<{~Ag!-b!}?e5qtYl- z(*$o#mD>xXE0p|@<7vyEel?P!T4%f(7Bk!iu#&PYpZB#Vfpm&mWuMyVOQTCm!uBL$e@rlcl8hd;$z`qm}z9eSHKmS6| zz^+~V%VHh?l)g0NUrzGVgY@O3UdOI&wdOkBYOK%b<2K#P^XodFb8z4I7vcOG%+DWM zjq3|;?xThYzxSF=nCm!&IVKMnyUMucpuD}Va=cCaOL#sXb+w7xbg&O^!WX7YpTFnu zc|(Yx%U6m@e%1IVe35H&^(QY=2egvSGYdPyOl-ruWjhL9$uP5g6>YDcY+}Jbi7fFY0Wo^gyPjrbH8m4ga%OBD`%YICFA=;u6~ z!u6S7Z$&{&G&D_{Ab;%!JQ0(KtdP(k_mPq{*Vj%p)Yb_P4@aMVebKpFXQVYvLv*w& z=v!D&h~vkPW5?F**tB6YER~hj#pPCbtCec@MWd8PcCT_d9`Py5=Ef?SX@U^(*!ZTk9VPAS6kKOpo=XmPK=dFAW z<}YUFZmnU^bZpZp(ETdFgR12z6Y%zXm!gdjU$f)n5BK7sUy7j>f>6FYL!!#&K*EIo z;e)H=ptr8V)QKNpd6iKpU&SVl17Dn39qb(U>}rJQ`6SfUalUql5GZ{?UUn5r)gkfP z4;AaOvAPDx>KgytYUtUdnCHoDOEr47kK=Dp)c*FnZg4BpzIP4kiP!lb%r#!~a;}4A zq8M0}+~XfQctG*$&ws#`ao)e_;ArxG!F6~4f)KHPB&2OAMfjj2ZmWZ9vkx3>|MHa| zE?gG)heF=JR+|-lpW{JKhY<9U-*WhadLx?vpHWq|)BH*T401 z2XAfkif5-O;nk?`+5+~afpGVOzw>OUhlZ#kA2-z{*c(Zkr@5E>mm}*lWqju<5qJsT zxWaNlM%HeAZL_j@Pm9Sj@Z2Er+H2l`AMSq}OUuH9;A)l4iSR4l!vDr6LC+N&nOODA z1Nio6gpkisn#XYD*LrU{9J!u9xE&sdOdNDAM)z%tw#^a|7af66dkG4T?Z)OsKVk0L zQ!q2%tYN(|=C*!pzRBjEYo_9>^;VerO~S+r+n{-B9HLF3(AmmRc;X=Tt^OT+86C%5ic1D|M(akA2z6ATG%o|0%ctSfg(qRx>4adX1fLUCvH3>x$=|d7 zbge`cVEt^jR-ibW)w5cd4B?80U ze(-DaY0+(v$+N72ae#$l&0&%+@3o?!! z$3u@jgtD@7JpSNgsHmtAHI(tyH(!gfB8gjM^>O^=%i;S^_!8Xa`V~2c&l80&zj;iL z{o^qoZ~4-%u7oeq?GE7!Z-D<2or~*>Y2gcVHfQ6vOp?CLK2xwcCu@=_agBQ}e0SI# zIy45QkS#3 zuB$Qb>E+CKSG)N^?cjAXJ&T3=Uny+!RR_Y)zFZw5*3WdT7MLr|!e8DtzCxBG=y6p) zBB+UVdlUA;ltrZXVxF?ps7WZl_jQ=bVSQ<@657c7gA3o5`h;8If2*phU}kw{b~5Do zf>iOJ@!Qz8RV>TX`8K!3#s=#%ZkzX~kZv^#``X6JxW<(RtDn0D*IY3UT&?5(T=2g_z5ES- zsTTGkifmI5cfIxzkW6?bFx6NRL+U)W3aUrE?|cO^yPV^UsD{$AQtC?<$Z zwU#L9pDyv|i)X%)j$e^eE1$jazi!AC>WSP`bt62}C1Ees`O>zs)SPqO$AL?f%FF9< zoo_tTB;kLgR~J&Ig!JB*4o9x1Bpju5C6S|)c6xBiT~ZYG5|FsRt!=L^!d?`O)dM$E zuE{A62v%e4g^?qZ_&4FgyvQ8V6BmyXgQ_BYk}6&2T-QjOaQ)=hM=pGuUz2&Bq{FYb z{MyN{?>ygio#)q5>3Ul;2b7xZer&kj^K`D;ZuBJ>-m4{&4Q3P`O*jrwf&jAITY#vxq zj&idV4uc7C&AX%jbx+~Bd%7Zq^_eh7k#sq%??;Saeb48e%zv|?2gZyNgQ_CD@A((d zHTzJ$?Jj(N6%i8!Ls%$6jG@BH)!KUOrIvW=C=36?k^a_3rmR|OaE-A5P%;-3VJ`vp zjW6+k?olZLBK2pDc&@N=rJqUM{9|o&6RsfOYP_W9!j@H_ENp=Up(to(rGwh zFM;^$1NZ*%f|5-)VK4O(C7ZRyvzU@jwJ4&b*pT7_C3R6e+bM~XZhgf4+Nmr`VP?aNZ7w5`xSD?7K82k3@MMQ)Nef#%u<@peW zBqlXBITNp+D2OYQC5UnhOXE_FEKe?ce#d`Fp1h?O zn&~Qq*qkFxIlsByD#T`ZP}&mS4P%~sAD{m?9p6rU0rwA!MF^X3@;I9%?jk(!)myl? z6PtrEeW@L1M7p%=>~^m9d*WvBMpS5fV&X>+;gUE9PVGyFt!;ZGYYMRb)!Xsm4x{k% zC|58u1o3?@!_^m^hpsIW5Y0NN>{JGhE&3cU|DJ(SZ~ubZTgxV+0#0aGzlbZ}+l3)7 z|609@^s1+qqxtP$bmN^-5C7xFEuMxpIP+TO=YX#0GO|AqumPV)t*K;m4hJcEV7;M}=DW z7?tzobl0~9x8C3y*r7(FYq_~1HKo6t{*~~T-$KCCCJ$!Xd7|`l!U0cV#zS3UfE7F6 zzYediajodEvpz{0bQLBDeG*^YCmZMdh}jzog+Afs`#kr9(4U$;U*nZ!X}I*NerTVV zg(Z*v2gmOC9rtu(gMZJLcyUovTz>69w2!Vp&cxDq-)j)%CrQrrG;7UR#=q(UZz-}^mY7{U{(U}M+3kmAMo^!_ifwlWmD z;Vfzhk_TUgD@XQZ*Y;SLP>JFb`>8=FzUW-w7#zm4~uY68wl!UAh6gLcQtWJ0b52iTc!1 z|MOTdsO^PnzZ#$c^*8C=LIb$lAT|Zqxm`|!)l&x=&KLidgI>2vgTbv=*_y99CX|<8 z9Q({279I{17P$gYJeVr}EK3zTi{=_V3w;ug%f>DE_=Ar`?ZDv8 z|4OWP()E+@)eL1HL1_xQ{pW4mSNR4W z|MVa#0=PY7L%>HhU$O4T;a6cEf!6gLh9R-cg8dj)eEKOC9A~>i6N}!%dm##Z z1%v}tj!>kGe;IE-G8RKSH9`yuk(p6|%E;E}(J6`9W5?IAbR0fq6?P;XCCEzOi~al4 zu|JzHVdkqNTqB(S6aAljS>=sT!23>v1tb0L=I=veKp1JqU4 z<#XXt<+{>YUEJU5y7)UMv+FP-Wx`8%>w$~WzfCyI2R38l{&Ix19f(W*^Cq4i9R=8` zV2&P%J0H6W!#X5Do3#ahuiJ`!SvHtjpNF2!LtTBwS0eD$%zWj$S5fYBE}r|T8nzuC zVZ#5We29TV2Pu}7*C8Qndx$|%YgN$is`kGz2=FN|a#2C86Z{U`Omv(*IrXYydQ(ifwsg0E_o?bjA8?}4Fc z9mziA7~|1&@NM|dtFPmw2Zo`2jF$NniNvm#aA?QY7yPG8(M{x;KZJ-*zh;oPeox!?l=^eJo-Ft z?#TUSd*5y4E<4%t?q*G|?n6;VUwRRyd^Z(SzV$AbUU?bQb>DZM`^P=M#{TIRseNii zUdGdg^xj?M?elE+nG(5wp1vvNk(_NJjX9Sn9aEknmSvl&#Y-9C+AZ6b-pkY7#9O^` zzr2W3CQ9%BS;G56>Jw?(m$=HQ>IT10*K{V68+5)Z#T#eK}WL`l!P$m@{R#p`C* zMS-%v@#p#qIAVt2!a?D%I9p)U#cdGg%)wuO?n9MJiuXdhW5tLTVKWVx3z|e|Dwd>di$v7_4dJQ&@05^Hs|ps8l6U%ccd>|gw0tF z=)=NbH0ohV-;I4{S6iG`i}0j;WBF2iarYg#`_{Yg{K7KfRzzBx1(m5|aO-o=;rUnI z!bktT9^HA2(rH1~o&zXmCaN^;(0?FbDe8nJ+lr?5PQll|%*GEhKE})U4M+R1N;p`a zbe*peo*-WTC4~L!`YWF$K+AiORV(Sr`hN7FFjyVl`riF~#Xk&atIUZ1dk(_04vo7A6jQR;U^3824~+OH%^ zWdji>^Tl0)XIn0{Y_cv0stA2rBQH5n>NjETYZ?lp}{U8@5z1kwHhrO-o zXLX{~UP+YneT(FiyC}6yk`9(igoGO45bihXL{fS>5)T|eJo_8BZy#d!?iK#V@86FE zmY#U#eeR97|mKX$Fna# zhrz>!Ah}U842DpIhK3>~H3dUQ48sery@-@1DR|_*hlDapb#hUv!zJ7qWq7+d!$*$5 z+&|_D+wD7czya2uoU~?51RX)or!ZxyE>R@g96m?$hTyx1XPYG9^FW6o0m(6JhU6^4 zkqxV`YSkL-ILRYzl_)7;bH$KEBtS@P4Am< zTl)?8amF5$@F2aQFW>()GG^=;AsFmSF2DM6#jm7LXu}5(KZTI5QS($8N z9Ewe!Jd3xN7DHIGYTJh2pL-0?Kl2=3`1M5fJ-;OQYRS;!%dwEJKGZ~^+o0|+*{e{Q zFc3rAGyj}r*s=6)6o~g%kWYO1YJ`ny>tY4@Dp*VW2;6*W3XHJg@U(kz_oFZ3<;U*9 zqd(`su8BmeaaW;Fs1>&O6r_c+a`ydL@Yc(CfpibvTv;IuRPH_wrVE7;p=q3=NVi}x2p*$#?JR|$6a?nfQRmQ4L@ZY1lI&YPGaqI*W>yd9>vrH zJVA@drZLEwavN@*a0gym#Xg)cZEd?A=w#%9RiTs%w#{!}Aisry#&H1ipS~AQKK&RT z`sR@E0mc#A16|ViHy+IA&nIExT~Feb_rJiGUw)4FUz&<#Ma+h-5&HCRB?NBSJHeKO ztL?&un=G2Y5WoKPi%Us?xxdZ9oZtSa*%hJ$6;I=Wu$OA7ozL6D^Q3($2~P{2NOje& z|LhX>QcHa01ykgH{1b$|_>@5PYm4WYtvzl%>u6XJ0x4d%*-N!g;$Qw!oi3AlP`}du z6nHPF6s7(bL_wV+ZgUm4JXOM8REqm}R=!qoI-5;Z8=Ujjc9rC#pu{)3@&Qo zfMf5k_-#+kuY~wm<$=s8oq9hnJ4w%=zjz{l)1dreQ0A5&H1cyeCn4ve}qEs`} zttsoe-rj>J`NI^xt-0QIZY`c_ zvV6X}l+8CS7TByjxEtb{qp-P(GolIF_2_|~JvyLOJe!Ew3z0taS-dnmpUp|MaD=5I zErHJwGzjf51Vhs*k#jT;l@X26Zp5AV&l6+NlFbc-&}ePpNNc#dybE_j*seyFsY>dIlBoue+Q<3XBDIq|eOD!Q zK*@|GHONvu9j&KG`@|)RDz!xJjv7g`RzhPi0F@TkF)sWE3Up$h?nfywHxof^)4Qs{GcFwo2T)-tH+OHrkQ<3aaFCDC5l(X zyp+bNICk`yupJW{gQB7$VLOUHNQ>*pY$=P>9`|$!zJ2RWw<=04MFrt~pto$of(6;A z&?IBvBOl|7&tAgyZAx+Q`{(fPib{l7H)6rUEam(^**<4;D?UHWS@JU$6%NGY?|#PA z8zOLI)=yYgbsny{r6US{n1-bV8sQgE!fiOk+eb=$AjyJx3xtPzdE<}Se=zx7sVS<3 z`Rj_Vsuez2wk8k3MV;BC)@jAQ#ed?s!;X`GF2!~pyE1gc(7_3C@_QyUZ;N)q>H#bE z&i@@p9bt$F4~Hqlf|FV02o2FAg#TPjoRIQY{*$OsHi--kaXqP44u1eZh%Os{El-Dq zUFkx4UV!t$s*up{e6;7kv@iG*E4DeIwQu?t%ZSP6Y+OqlXJewq0tK78{@ zOuoDwBC9Ob`&6kvl|+8vD&=#LklwFd{TJ4+S&M6LxQ_K7o8(vP3US?Pk&xG+B)lyD zC&9P)zIQZp?~~aJ=40Lh_daQzBAhmB$Xbepe^~|te zSAgRu_%{@c%FXy|bqVyLRmePA;0lE6Ob9pfkf(=j!AqH+$Jby(rVe4D29)I&z{=G% zCWM9AVYeF)(fejR{rL?1_~X}@_SF~o>a+VXGM1OuB9xt!gqt6G)Zyy4+I`7o>4HU= z{qwJao}78V&%t~qLO-eyxn9k_Qz3$1;@@hK`zYqiblHC)kN<5bk?Qo?9=X3t`CNSE zc@uv&-9+po-d~c))BTCOoTsSKk$m+KU;mW!C27ciO7+zir8de%L0{fa(zXv_yGMmE zkG)9kF%Vpl*TepcB5ljl+{M+t+V<3lROdfecprI7m-k=&M7a)kQR;tnQsb*XarvWU z!?y_kgfAIg*DwCwaBv=e+wFiOWh@@Ow=dYV8(ZeAK$h&+C|pmN?=)+Tw*1n}=4%Jm zY($wM45rXZ>|DDW#oVO68Cte*eZZ06*V~OJ^ay9y+tNJQ^)}otSam=CMX7Eh7s+@2 z>k~UFoC)YS@kP9GeJey%WnlBiFJjtm)($qOY*>!VNdz zh8yn21MmKcL$+A78TU9I8kGP&n>{#rP?l*6RBpic4?lnxo_YumetS$<44`Xu8Ai2d zANCxMYJUWugukqe)LspnLxvh*4CO0j%TaWE2iDJg4X=K-6D4deDLi07)D`;vkaFLj zaO->iuuk_n1>xmO*`!hzg8aUhpy0d<;V?xZv^1}}AEohFT}BYk{`Dm_^-AJyD==|8 zxAXTth1Jj7A@heNs}SY7B~h-+w+888HSLu8&pq8G%6(Hi z-6bq508#cwt$3FADL($Wjv?jt_!4Hec0FE1$^Sq_$!=|t_iv3T`QMPDVXN*y^sG*`2epyJO(`?_{wPIKH!Vueoy zx`hp4yBe;>boDw7EE`r}T{%lL_QHVkyI}C(#=-=;aMe=ma2SNicCE79o|aAJ+|qf5 znCGd_|42#BN*r4;6SIE)8NbZ>g~=>IX3t)M{pJuDt2SWz)34&AAD7^-ZRt2(7KWq_ zeKBmp!+7qI!AJo+p$PQ^BBD>;L_&N&VDKRH8`vK|PXAtb5R+8iQzZDNFRA{PR)p_M z!G_Q?Ji*l`zx+H4zs~xV$uENZHhU%ZRn!EqIhv9plxlRs3KO0#_#tJRD`3^SR;vgn zUQT5x%iXDK$ljhh-#$}N|)WZpL(<|}7T|>|<*?{ux zKVj;-Z{YRUzr)H?u5s#QkY8*rAbsk<6~E)#FdV>H0F$%@-H7 zu8&@>w;bWBD%tQVuGO;qp_S#Pt{D6DOL$^*D#8mk;BC>?wawq<%T=(;Nx#$>f8g!N0=~w z5q^Fqs&bV2e!*5Y(RW*^BUT`Z_{R|vioEm3!j!QGk%zY-JnJAL4{kzU-^*Y#nyU5O z%2o?|Q6)-c>n?7`kQx;A4SL&HK}_ug-p9 zFCOhy?l-CLq;$ESOXN0uN&qO0GcEz471%+%b6D@3)6c@X60u>Jw zlB$urC%Nk>d@flPGaC0iGaduN|H98x7h(1CAMyG8EZ9QQFnB~qSffVbUN8AWMs{N3 z#V*8-_-!hUp=_?j=A9OY9?n+dardYKESS0jX?NU*%cK6nXD|PNUCpk;J=e7tZWGDR z%u0SOn!fleZWuDe1d8fo;b^w z@e1n~tAFOrw;@d6!z>%Ilux+1-RmB}*rqJQb^vQu9fy;3Jii}T^+{iVe{f(;Qm9M* z#CdsL+=lk}A*Ay!RWuQJp5QyXHMVS;&tl;l2^9PjndvtNSF3_sR5c@%@Y$g8VRV0}f{$#<3y; z(`Uxq%Cq-k9l``a2DdHb$q)jw9%(M0xQGcHUJdtKmL}MfbmgQrL7K24Cm$Prn1+R` zH(*`51)!u|iJ-^o^$^cviliaM zW1ldQ&S6S<$|BkEv@HtDs+UyTVsBK7>slp@F%dpr((PJ^`5A0Zv{euf~5~e+bvKH^RK<2fY69iF;6{AKY^jasZ+BWDq zydzRVD^X!(6KxxtY&PzLzRksXoIV5w=4*&%FShOE_gH6s#kD|cIG^{~5S5mSNUrC| zLrxBxt?_$_dG@K$HYOr`u4UrCx?&%o!o%5|H`D+fn{x_*Y7brE_62TzzXS(a-wJaf z5kZ~5i2cDg9qhwn?%?r=*}Dc2nR{W%PG^623xDIcFU85hO(vc?K@;{Oh-aE>oBQTXB;R-&+HB{gF#;-)cljzoA9R4@F_epOUZ_>`^y(r3F4V9ZgT;< zHd@Qdu|Fb0uqhY${&V+BIo-2u^7aNQcinosqR8v3t9XUAhqGwGe2lsD5@CDa-o0qs zzOAsGaXdrNac{qRsfKOeG3j@+DBrsQ+Yhh9rbAj7(weaXY%VC4 zQ<8-X7h%rtf4Jn2If^=U5v}JES#QGBnUBoL?!z%Go^O|7>swc1!qww&?KR^uVca#i z;)ai~tc>Mp;?ZYBPb3s9!gq7BU}jfEL#JEu(&tm~<{NM0z0W?v+gGPCa}Hs3X<=3_ zD*0Sk-wWg4`Y&F5=^5O49;=bnC)VuhV5TeF4nS|kz7^|{?O^ZY5)jKD9I$sGR_2Ba z%b@vBvbe6Rq^m41#}^8C*7X9b>n>Io|29Nk7hi#3DqM;g3r@n~)FFB7oA~IzFW~XV zp2myseu^)@coUCa(He2AZd>#a-1PnznEJn0@yc_LbUUge!vS7|!LEP-963OoA6QM6Xw$+GIdATU*x{JKc@;*3I;>!=Y-*WMnPjY6& z{dla|PfFs_cl~7|9_zJ)y?7N*9eFu#!d_~n$%VZri>L4MG|&8CiBjExijoZ}O`hkO zRtHh)&j5s@l&&oNpY&JZA9KiCo8{+7xyW_uBEpzwhS+8`d_EMeJB$@C-HH3|ydUrU z<#0Xpi5C#z+x%x%rlMu|Vc|(w*(Y_=fG6nspxd=V|d~7?=ba~_wd#SpJLkk*W>(1X4Gj#;mVcRgAf?> z71%xd4{YOe)A0Vd{>|6%*wb&|y_;LZ00$~I&cMQBV!K#TH<8U%IJ}>DpGv>J5#|P7 zbX|Ra1S?;@3-{mkfYA5)5CdzssKBTm8{$6!)^SmXHp6a;LPg^?OxjjURZIe+(>F35 z{_{CkeYJ$W1W&ksfeCM)B<$^3D@vVLTTi)-^%JFf)QN;_Gaz`mh7`3ZmK8UY+8vlE z^?BgJUYN)n6s0=roE2F+gYb`^_L*7V!d}iLk)OkYe;#AKRu56Sj;axNJ4${rQQCKb z2zyZy-iL>bMwFDYi*RTt>`rHO&{R=81FT%18FHG!g_W2`OeV#8YQ$SxYtlXYMc!9| z3NPQaTUWgC&f9`??%KK9u7C0LPyQa&ah#_ge-in5`K*^+@IK|~dw1_f$4(uD7}n~Q ztI%&ie<6R@&YgmeTs&>F|6cc@1TXjbS6?U!&y$eXBNrZf3(Yx*eSH398iMN{n}l&U z+>b{tO@z)_f&JSLz>w=b{{YM9@iHJ%MWAw{yf%bEp&PfLbm;gDa#Har@{e4 zVoS7X+ZN4Y_~e!Uq}(-I*N78Z{=q?5wR&t779VGha9HQ>Te$(J_y-22&pT5Hu*%Ew zl^ZNgsjj99b@3&~hLB3+{PY5z{A2CN6R!Czn)l&0b2u$)7ETTR~SoxyUBb47C_vaV!)F%tECcO~O)UHCGG>x=~eXV1V{3LL^eu;A^w2ce=6;3Wvs_xde(sB>J=fT9?J3tkKX8T9 zB=TM8`QDev%S+qA7SB4Ry5!quUf4^mG)dTtqWH@{8${lYh7$I2hD7Q5#Qw9qb5Gp- zmS?z$9M3e*{9vI}cc3WQkkaINo@sRu-jRYRXZw@t;_`&88JlN>zqg-l^TmItut7X?}e!Z52KM9sk zj*Zr}`1K{`$8#^@e}7pKsy%_7D>q|LW-%&_322lQ0bO|>GIy=UqR*ekD?eqy%FDBB zx+#AiK6v3X{Jj1s3c^~WTh~+=^LJvy&#&OsPq(qTBmb2R>l5w=6RH2*=8;UeEnyxb z>c~Z@@AV;U9^$mYZg=%Pf6wPO61?AeA4>hfSLTK19zkU0o?7-Li8!$j;W>w@%hVFD zC@K4sz~M7fPw|RUm8VVEi-$-*sjd;`zf6?voIB##o?weoooAyc`R*w$UH0Nr-0iwa z1EBi2Qj3z`0gLi^ioMiPlTj?4 zpZsMPV>0}UZ*Ci~$6kON#>L>!j9;+R9?Isf`B?w$_xRH|0k3@i7)F-Q!L*sXT=O+P z50&|ITBLW+o~qB!b~fN`HaQJ=cWZHOb{R`MpHBU9RV{=gQ}SLtdt;%2lVl zY#?><4-4E@71MS3@4B3EpScciJD*I~=y01gI>PD(W}62jr9M)Phq`C!XPMp6f=~+@{nXp$)?R3K8_H*abkUgFoDi|K++31GDS;&)x7%L%DET@YlF=8+q&a zZmM%@Jq`K7m#(J+^>Qje6#SMaDY*<+oH{p$%(QvvBn85Jnm0i4M^Z60K zj`Hia5WJIJyZ8qeehuZG~eM5lj3NF`I>94!>@z(vg@sMU6#YkOSa^}U=DAGYaZe%$BXjnxKDaE4-w`N zVi1()^J_o9uJgX(jTPpNd=4khO_Y4)ZL)FS*k7)1Fz^SRy7#AqOIMoz@}II9*j&in zF7Gp;@0m@WtmnaF?x&~|8y}CtQ>TP&-)NX_9Mdfj-`7yvvvULWexKzn^y8k8&4>B9 zx}(lUyqypkKC;dtYI}KZ_^r&j?0e7=JHz$opWic53;F5Iv$c&X&3HUL9Vs2#LsMJ= zM}^tzX;7LA@W0x~2sqZF+iHXC5k35VQo_$74so{;c-$qna6n>uh6aHMjOiX2{Fz>1I>F1vbfA7BIPT{XN z)adbfk1G!7nt$DyuK)czF=TJu_$l8)CY+u8fb#l zlWF!oFeX*LcOh@SEG**NSPG6Owm$K4;3soFXzLHJtPHR7$Q%$=+m93S_D3Y~y}IIB ze?Jy4+-(FVA3mOCpM7^;p;+~thU)Cua$n6)UfTAReNX=9GpknO_&3|>>RpuOL zNwXp<>al=D_WLO5>bA2IFtv6U3ENrkY~QhnhwnQGSTw@2p0miwRw7Tzii?SfNf;}m z<7s3TI8%a_q^9b^b;?SXhw(9Ck8skQO#qDi&3KZ(@i=dxRa*kch248D`Ii}C4-Xf6 zb9FvVscOn*7?-o#@!}c5AxM~OohKbT`<>oh_6R-g9xHSXOf)w%eX-w~qj{d7DJN0i zI7WA@^*qy?oQvTUSI=LeRXq9#`U#hI+X%UADkQYDx@?m=C%e%}U|ABoiOG4gF`C0# z%v{K`1B6CYR`wI_6MYj@4#XiTAi{l>ZylLI%prk7)ub)<`*J1T!!G33Zs7rX0F$k9 zcl=Vdlq|o8`;+nXk36pZHh@44uweLZ)IY zp}9}HlP}=PJ?=0o9y7)=SBtMZw|!+3c~G{pRi?$CJ}ZfMq<4#6x;q2le}!8LX0Lp1 zhgJ)=V?C+tySsW)G+}tfVs{rh?63+=Vq*k7aU*5vJ_D1M=Ci-D&P2*H2k+9G`X-;K za(GAc6FKz(!67_0ua1m!fMgPbJoWo(NJFlNg-hqlAx+d!=tMv zjG-ytAsw1;)qMS$-&T)a%u1kOKTR^fE^84on_s%pWEsodobl97bGRfiS$*PFpVQ{l z4_I-*yY}Ba`9@7Y3VnBn;@L&|;YPIzZLe)Kb`jUgWw&}IQ#sUu3TKzrt6 z9mrKQ!6w;J{%1A+hno%+pUSg(mZU_QX-bh`+c>Hbr76nVAhJbxiF+3T{;F_i4$)D~ zjX}6N02_lthL6 z+$ESQvMV=`RU(sk_c_Z0t9M@N@)eFhM@(uek+h5qH4hJ-PGvF!F1!TRAJ4rlP=KsP zi^4JBh3kjZ`nU?iWKY#alhQCIGd31-Asa1(b9o%_8=Nh}(4RpDQnYOek)T~KN6AtPa zSH$Ns%Rw%hF=w~TgAo8_pkM7cS@PbV7-h~dmxmuCsY~UdZ-OL){;|@JwN{BOd4>^M zsb75Z0+7_Oift}?=ZO5Oy@(tY9uQ(QtH9x%vnGAP+OV@DIpD+EHfI*19Iq1=FKh9^P}{aH#j z(*{Nk#xVMx=#91o7IqTnoSe(esJ~X~(Od**Onuw*Q9NX$B|eTFZ*@9Hm^nSyQsWzJ zL|iYW+5v;Wx6lUHg&Ud`(Ci?tNv~oxBlXvQ>lG(}j2l9?GbTA7>UN;r6B|@d%_R;* zy856`9Z$qLVq6sMG<7w`j|#s0egp*ezhultfu6rRxuKJHvju8Wa-VpGc753l1;D8z zlyMSVIs(p*z4`W9qJDDdlr!+&me>U1K5Q6ruR*lhs_t8yZ9B?>e+}6zdS5-^xpUg! zMjx}|r(ULP-e%CReet)hDdOlI5=6q{-8F7|8ba>%iBr2Nm4`fJ1rv9&;Vk@so@i*D zKHPc2HX0i~f%k>iALlb_-3~PEU}rk&9DGysxz}+V8-wDSMh~lP?P{kN)B0F!N!Pj& zh}i86n;C@x#_N{2$P@D zf^`l}i-!R^*wKL;7p>XkjYe@O4( zX@nlaD@C`zwIl30*D?V30}I?=P7?i2VZMuvE~lCB@+iFG*o4>zVQM% zAZshUdv+05-nCTD`esCbz9la1Apuf;`eom=vSN2cQ=PiscjVqT}c zX>1cEkdoC8I$&6iCS_=k$FMUfb6mg`{WAA;&{d3`I1L{1mT{7#JB)>FzaJHEu9Zxd5fLuWH^vjrFmhfK>`M{36`E0)fUqbMYvvdAWSu+9jM_L3$DonbGeQoH2dAIr$h6nn%-MmnM`$Q|G=hP z28vd-ZsVX+-qG`E$|?KR8+IG;j_BP(dj+UZ284@DnyMpO-tzsR-6i&>~5VPH}uo}fk3(BO=@s| zt4ziNDOl|D*c^5ghme!&8!Pu+-F+uS?l>&5uH#q@*`oQICztfCTI=#oNoId-%k0Ku z0EMV_{C6Dl+b+r_I@KZ$?4~J+e_Ce90-rnW_%sQH^G$MD^EOH~lGs$j3q zP!;+4e6tz&-0=i|s3dIX?!bT=N0O#h_0(cBMlC&B?Bo3notSwkVA@(N8m#Ruz%r=P zPm{PHYqk7FvP8Rl$3xcKebU3II_ZXRkun9tvG)V%*x?4(Zr*H{b|b(CGyu+8IT$NM z?_TEU4BKJ2lnI^*0(Xjq=vMh4o}X7TDjCKyDK7=0AzeWtGgf-f$W~tJ&;m%~DDv9u ztgcLaANC+H*k(Mm@4Al5_DaahhyPd}jJ^e_58}ojA?YD*94xhk6hXsm)IkV* zNk7qrnH><6WIc_{t*ibvI^nyWk}LX0(Y~)urDGXSbhQE5lNs+742hAPuaBj~)Bu7f z629llmayLe*TzP80FECumSp|o%xokcd!fzIys5m`@R_PKr?3c)&2=OO<99Sg<+fqt z*J2_3C;Mp2P(hfR9|DUfnH*>_)liiPI-wxKEimP%KW#;7e9s_X>$#3|e|zy(+Scx8 z>K*sYX+-;iyD7L4)q&+TK3cvyhUeS9#)}8Tw_*flRKce~W zE=0A&aQ-@jRUQ3TaH##QDT&&KST!3vSW$ki7CTDNEN__O zuPd2Dyw%c>f1ci0uL$h2ZqVG%H9SC)6)?BbgX|pUTDyDX`1S{+#;h9Zst2!K9|M-q z6f>jI{$T;VR{}A!M)Rv%oEIGw{BB5mkJtVp{4Tfy>7ZU1OAENsjn_RBT>_#?NVWG< z+4Cw7P#Y%xbs9(}-g$=TzxX8z$44+)N&8_&Bd!kT2LozeJl{(?NX=t}wZ}>K?@{-< z>@VvLm`HCPA8~z92NF8{==f0NDf|i)NkOVvQh{YzT5b36b!$P@SDrMV8{Y|_)t*f+ zIz2_W)NW~EANCVTN9Z`6by56QCk~xXb;|Dfp5g$Z$>Hb$;Z{s4)__ZcSu zh=l$Fk$?gu%cpzRUzkLQMC_f!$(%t?IIZvR=2V|_MSd9XIth2&pE~>?&&;bd|uPliy}QAHZkEfabz-I-}--`AFtm^x+*FBc%*&yI|)nle11l@TF~YV zRGqSv2<1i~awIoMjxxKwTw$Xw^c35`vUAh2E(@J01*a~tjQ1`nC_oi0=D3EGAMr62 zjco0oFKPd+<)54th#nVSv+6e!xvY@KsN)&kSgsm>`W#Y5cCf+$&4F}+m6pAleFi8~ zuOA{jx1u6AZ?b)yo&vJLq)+XXAqx3tof!5ku|(5|-o*&I1W!?k#WW=)wh@b+nek!b zqQM5#Bm&M&Ms-caNxVFk`49gvSWVYoP6%7l8uzazh&iHlopQDSTHf3VHU z&K<)bjWS|T?0p~qLB7+P8S(&|js%QPdDE+!BNEp&1OxEZpCmiVC;!U05lRc%XI9Mv zBKI_Tk&NEO?!j}HFs~b3-s5+oi}ODs3g`0`P|;Fq#%W-63{QHlq{Fs?9rSZ`U;c?v zJ-@>vS8CDy?W_Ie9sKu>JDBmWC)`d1^WSE2t>Hh;JH>$5Nag{%C# z(f?Bfjj`s#|D+3>3h)1)WOV~7Y)r;4Co+J;H}9WI&uZevHA=B}Bar9kK9}EeA&Ns` zLo$d)YlTemBLq1$Qk3uX2!{TfwN#M-qL=ZyyXl)>AckPIlqOl^fa)meNp}75HzQPL zIZ0NV=ZpMpKOEx#7;kZ0B3Ko#{^hZzox__%2){WzAnbEer%73kSo!H zZqFmybL(*ear#{&vA%}{)04H+w+CJ4w7z0Yu3KI<8%07kgqyBj zk@h(dc`Nh?pRB~Z9Iuzn{F5DsWuGb*Fxh$;93%?kPZy6-1;_qs(U%1b$E;ZH}I7HFzQ6*)4@iZIW)fY;`kdM`kZ^v zGX7sgL!19Y+%w$Rlqdi0ou|Z-rYvrB{fg9hp0lXGeh1P`SQsPTH)L~7hFrzIuz0a} zDDbX^zMV+vvFoh%wy*o=ye03P|ld%y+V+xJ( zF-fPl#^xQ4cWnDhT*uTvmVR`wmwn>1%Y_74=hX^mCQ};aJJm%%KZ>bZj#%r7VegS4 zjC*TyjA}PrYA7(MX(n}c^DzG2gDBu=q#ETmyuLpY2#4M9;Bntkj)kcl&(y9(EMrtI|aT`I(^L2Dt^h40LF@k_ABPGFcD?KWC zzg|3PzkU=8dnX~-$GuTWCmJQ_b|VHL^%Z1Kh|-SH#!$A)kj}{{9HOD zqd{g>)f6}{10JtdL;dXX0-~KW3qWEm3}a+uH2Mgy9}^_Nam-2ZR7)YJR9oNkQLA2T z;h<7<-gwu2i+0KN(!AO3u&~*4O69@<|D~msEqbfBNWUR3K>?D79IidzL_9%$bnC|h ztr+0EQHv=AWX>19?%U*kcVHW4+9380lJCcm>^SU>_PGDi+sJbk>rnRxHnH!!TN0XV z|JrMPj+{_ZEkk|(d97GtLFD!Y`jHNvs-l(n=oQSHEQA z_<`G$>DaaMLk-#)Z((FFqpJrzFt>8V(;&f7Bw8@^XK)Y@$B`olh!_>m38F(9LLQH~ zF{s__J!A4bkK;e>Cp}r4ZI?h^)0(I?*2R7;bp4ie{)k3O=kjgJPhYzgSG)n^w}yUN zVWvo+uH-p%Ma3$=d_{*K3wlu!er4Q~zEU}zn4&Q<@j!5rq+}Kl@Avvw8q_0iH|b0h z%G2P4I7uuDQ#Cna}u4G^uxC0-DQfjYDLvM*-yHgCiU-X)@ zc8ni9>A#b=_$jv!8p&&hihmDVP(59!=MC-}w zFoT;srjcB>e_UVd>fD(uZa12OErbvcWzQRT<;ybaT4%MwTOac3PHx03!njSeV&m;H z^0}N2Qqts#u56sAgY&WvXP>jOhG~&xk}eHKAtgQQo&4+HO1Zro0JxKqwTT!R&>h5@M z{=6~>t;pp)O{#!9)dF3pCer@>i6g_1_Faw36E>N+KaucLW`x^3B9DDGD`~8fy2T0P z_(Dpc-jNWKP#~~oq}j*`^*doe+{x2Y_yf{ptx-L=fb%!3 zZsfQbLb(}4yPwE1#pgmn9Gugk9=%@ru>x(z+)Cbi$^m|nU;O40v4b^_(j}`=qz)RH z1g>^IlA$}s&AivK+MHi)_B~uqBn*w9(4h9%1bANGq?|$>g^e79$eKiN(2t3yan>tD zAFrDzp!)qr8zIW;eX}cF2OLX567PM(3Qnk6*#$8a(Zd8>Rou}KC$;E=xgpzw#aw2m-MjbhoyWJVzl+RQimjR$7lzL5o^rWj2EPSlY^ zSG^Au+n~1@FrbgjSa6n-%ggUdnO!LFoul=x6Yy-utVsB}ib&+d_$n2V7Oo|(t=L|- zUP&PBP*f~$u$&$g1G2*AS{X>OkA(BzYq7XmWyiNK@|h~jzFugGkwTYOW?Kqbwt2nI z4Aembnw{g^!^RI_C&3~-txja&-_F*5FMd)S;9PP(H|n)u1;;W`t_2w0gN~+I1pKhI zE{68%!QH;9evoJ(+%~9v(|npB+qt&$0%FE8Q>=c&u#=F@LS?0k;Vi&Xh5|$>8eQ;4 zFq`2dX6+(vvYTf#c{BY#=XGQNIY#4*JdJEUnHR%n+nBeT(rjlgE(yA9460@hFsk@X zGsN#@CgVkZZ>F9Ym!g?jjL-oq(F-J*jPo6c!pypP0i9I>odKk&O;vrz&L``%Pky^^ zS3u-HL3^<)QY{6F^ZtjU9Q_^b+EzwaVhRV>(Q)x#Q|1`w^n%ZL_ytwQZtDLqV)@+fm5G_g2S(a9<809a*rQi(NtO$_3 zPJw;#3X!vc3sDHs=(sd(f%(C6u5Y5I;wIL|THHa+o!4Hyv`fy$j;>bAsP+p@(k3PG z1LR&mf0jv_R3{KE{lHjoKaR;B28R%uRW3zLADoX_&AAbc3lJX6ke&Xxoh{7=kb5oG zTuPJOJ}b%SEaA^bPWgdP_I&7I9Rj1p8T6*&`dNIxb2O(17#dfurDeV2A^~{|peO0m zrQDZGMojaChg+M$VMz*|+|}I`tJw+^5`ODg0erY_?G8AWnJmf8@kA$Zq=B7yqiid? z-&kSoA(N5i3*NhJh4rRtQPA|{R@JJQ+J$^f>Ik9H)Q$dlhyEw@G$(G^LINZ3z9=W! z%^x-}R&fdCMl{A2P`>8{kXR6Y4&VJcEI+7IF! zOYMs^z%7F6;)XEhQt@NW{YZB0wm9WnyJ}b*d&sYs;uh&+rOV5*U`KT)HI0GSMUIB# zX@gvTGwuRHhEa=@zVPuPPH^7D${W%h^P|dLYyvO{1I*+-0S;oU=^fNz$$Oa7)f%ri z&?8GgbhDNHIo78G+289?&SARUl1^Fc;%|&S`|RB9E-d6Pqo>bqIFl*XFtN_S+bzy462oyTnird7Zzy*d zMe=!dUu|oZoPTy^F zPy{s9jU6XDDD8_Yt*|2+n#PiBywHnz$apTPh1!Bn5{X$`e8XDm7&nd!F?EV@I=NGd z!~A2Z-TG-z@uMkbmtxUgH*F?HoIuDEkHJ0x1mL4jt>{vADD*4YApY=-Jlx5zoU{ZR z9l!2Q=h=!sH4kmtxC(!lTnJCE#oi8f!Eu!kOwmCQNnh__Nlgi|zi~~1yep?57q0AMtfTutG&To~} zoAS|D(i)7`7Djb48MOG8684&XMg58D3q8e_{YIYX7f=m1`KK_1qQc2>*xkueSzIz*640cFts2~nOjsVO zyDKxZM44KA+%G0Atg`Krsk|fe>-7W(#g|`ZdP^o8{yC0a_=y}62d72|pRxpacbF8K zABFU;7nsLtUjIkRdvF!+u8wR?64C`?{h^)B%m?;M+~y2+Cg0H*CJ@OW?QH1JXV|N8 z@b|GzCpD3FR&;k&DM?(Npz^9g$w9xkTFy6U*Wdcr`W!bBjtDO63l2&)n;U3qfCg$q z0=ehJfzwSR{XU;`YRq(8hJNAV67TPu>iKo!=drj zpPdx;>1c)?Ai25rIAyQnoP^lMb|_@0S~{_^wAuGLCDl|Wnve!EXt_mk5A(cp%gFlO z>OOz394jn{3;l6zgy-sjX)BOBDBN>cA$tLxNY8b$Li)YrkIigWe$R8alY0{k$19Z1 zsqX%KJ%5DFC<+5^K)I*h9)F|J;+H1LfsyRbkrx2yQ-doU#f+|BDI8Fb!NC_WWPGZ$~`)SumqAAME5uC zbkmjzPbLDLs!CfZ2;EZ=e^Ylf;8Q2KTe*?Uq4lVIr>7{!JP>t-lQL!L*^F==OCv6g zeb)_E%vXKAebXQK2v|XcKyr-yDvjC@jVn3&mp>76K5fp=ecA>~>En!a-bgIDY`v(L z$Bd}Uf1xdRZSw2u4wtz;7nTQtn9eELEverAOkup~X+exJu-z^1Bn-0S3qpw#`BWyP zB5)@b>u=3A?m6p;-KiH+?!#^}&PBu` zZB@QthnNe$2G;zHau_W_aq=gg;B7mksU;jBFau2x$75GbN1G6 zCdbDtCW)&I6BF5fm>waFlQ1eg%Xx)dC?s5A|)nJAwyhryVh6>6eSjMMUh z?i^P&-*R#Ju*7I{r9bJ2L}2)NGsnq79dIl9QF||0Lawy%m<)Y6p`S7lAX6)s*&&WS#!_v}ciLC51z=AcJ+ zFrx)l)&(=UYZO9iuhr_*x;i{1d3rvM><~kLo!~DZ)3J76!6G)n-;W4i3SRhp8i6bw zw0aAQY-K84!k<%?O#1=oEaK#rs-uH@;OzLS8HMhjS4eqAMUpIEzg_ znJJK#bwVNidHn?JWe3qyc*kXY`{fNe{o4YdM4zoHM58AHM5C0;*LnWI59W(kn$8KL3a&JMIb$LS>>?^?=BI@mjC zQOCouLz3%9{dV7hI(~;q&AKVjbA9qw8dNVfil`5ymi)l%MMThoQ#(Jk(oU}G;Q#aLh^T20kY}>hrykSqW+581wp)3#>xvhs}e>;fn|c&vJ!bk$}*eQ zboZz^z*-Zav%lgQZ-&%OrZpldkyi-jF^Uf?z&B=@OIx>KgCd!UzL5qANvb4JV7j<7 zGS`7s+Fm1?Yw!`N^^LgO7_*=UHK+huJI#~mWWSO>%X|WIrVasI-OMXBwZZ}o3?*7Z z^UW$rK6hTBuQhe?{hGn@F%_uY*DdQcsd?5wCW_QKH2S88BS~Cw9NcP~Xz*lqSKTUW ze~V&9jYFTi*1X9*cmGCI_S~C(;;U#c^#(1&OBilF-;Jc6hcTJE@0EbT15j}0p#fRk|AM=5 z5Jb)^^oJJ!PsmkMCf6h^-pik}9Fpx7V6@P)Bvk1z5t_6fO>J>#m=?LrtXU7muj#A= z$+R4%oH=JEzw>?4cWJwHLEX0eWPBK^pqF}h#yZxrHU#-@-MMKt>T?Q>jd$_^K>07y zk=4;D;y15X;!T_MLfSfxjH2H3Wb?fcOt}o6Hl@qr{=qg-Fla2~^a>S7>Ty_%m92_u zX+oo=Aq&bx(QCr%AIc}W14)LOC%=gL^>Xfg5FITZKFvv2!W{+H7JH^0IjXcYR3`4w zkw)uvCT9_4Ud&65B{e_9|J8(5LEg&T|dg=(?@-L4?w8O1-YlKUQ4L-cdi}hM5qXPH^W4%DP zeuEBZa}vI#3!^G}LcUP*{Mgx{Xz9xN+hlW9ZZ2C`c&z`j*KO24I?W};WUFjT>vWT5 z*Kw)~YhOpE!gijZvO#d}2hHNa=*{$>?hCeF@4G|{p8!u3<@Sab3=`qT>N}jDoCAq; z3GzI?VzhIJMk-S-$1}wjZRTZbjM43E-bB3N6p&O`k5P*W7NNl;i|#+1XVS9gGWGP} z_I$^)*21&C>%RAnXILlDBgYe4{MbEABlG>T%GeEM+_A|H`<;&FE^ixAlB zP-}Q}3c68E7nsq|RhQ)8`6xu6v63Dn%!$W*fe z-&U}`#y0FH`HVY>EH1nI!iP9KNwXCByRh->WtXU-e^7Nt(yu9DuH4IaBKJL&yNN#! z1yO5n4)(M(a@UGAA+os`e7sGuMv$-oHeMmIF~0uk>Zj+lR$fymQv7C>?ny6A^<$|1 zH2)`aPI0mdk9uDbZ>j&`bV(F?qvSNu0wB*tP4;P2cPA>Xv;5O_y=t%x?(^xgpUMyn zu)Ya)EEhzGiW5)B_zAHJJ2`h z$^$6ecp&8cVLN7a`zAEy(`#B&BpeIBC=0b>YWw|^ck{_76o=!yl3wNSb^(Pn>WT^~ zxM2hRs&6H-s}`HelwSmxo+^B0 zugVvjMvRP0OG@paOo;Xy2q1c-TNL1bMqExNK z?>A$pXf;v&><~(~8aTiTSB95jX|uMrC+m|1^0*Wj-%!V7=dKHRYuTgxi_1wgfE}KN zl(#2mohy~}=z(I8cdo|vl0aQnn(EC*dSE*jbY6FiB^${at>)%@_VlqwfpOVn+W(#t zB&astPSbpDU zwDI{iigW&~iigCH?d0!)8j)Pu<9TYtBx&H9VwDi5N`fs-j7;~d*$SyNg`N!jJVR*# zf{OUJc_PmFyqn)`m5me@o$II5Ya|~}nd30BMp>jGj7YKTe6;kMYxfdfL<#xe+oY&D z)ryy|N@EIaL&Z`sLak?gD@1k=)-{x@Dw2)?csM{lF~H4okI}6yMvrKY(8%q`L@g3C z4p>`!E~rL8{i5SV7+YYZ#vryfqikajNkP3Wv+G`l@yFXz4-M=KV|&%5It)8^Zf?@} z=nk-Is)fYV89wKGK=l`z8T-Zo3Ih!dkzNz)MqrQ>I0F8 zE=vyGUViuzx|#ttjt**g&+?T&K1j?D_w|`gvi$9#6!xlHlr)dj`(gp2D?7Q=YKE@6 zZp&xOlQXcf+$N=pt-ljSRkgcFPTtU3^X~DqY!0=z(JkLs4Zh<0dTEsL-Vu4p9N9}W zQVVLf4_Ij%fd$f#mASn52CDpa;)O#-yqr_yy#5rLmMr`h)a`(82u1Mzty)C|#t?&I zoYXQiK;y&$)_6JkEg6Dvhf_Mxi_fsKtnaaxK9!=oKP?r(p0QC@j;g5z&Q~qT4;_Mf z=47_xZnwDqrM)n2tkNF69`wrYcEj!7Bf(iEX_Iv@yx~Yo&qG_BRvoX4V3K6V&>4aK zun#n-1MvDTG3$(1MK+CzU3|7~HAVcOD_DE-PND%ci4#V#$qrsC63S#XTAt4rMr?R_ zMNu^ljhc*_3i`C(!#>V9$P_Bdiug;h)#Rs)pYZ}0TBie3Xx(eLALC#kIL>s8`WqSk z0wa&jGAUu~;VX#uxFxTUNmALEN8hsAO7}NoxN-SAet&}$sV1U7FhqdJt8!y2(L+gB zR{a<+)F2J}2__MDd4m2cRvB&@tYr1@Jkgjj7Fx1FOkzB7Plo!lvv2;z1TD?Zu2Ivb zi$wy5jGrr`C+5c5q`1N_S$j_n`gUg*Vzq3E9`(|L#X-C6{$1BPgmM|71}T`*zq7;q zUzM+n6@0%&VB{^L@jt3vxTeJq$w;Es%vJTl^RP#X$$ODdPC)he)>?6G$j}JiJ=BaW zST!;B`*FL`xov*B7Zi3XPPlSw7N>Y-@z=g`tm}~oKFE_e(lvjNL@eoLjGW6NR#ku!@J-D z5nOEIJ_QCm4QqSVnFir#pDQE2g>JHgx4R(mgc?!6!oJ1ro)k)sTNFG~&3-F8uU|>y z5jyXYB>qLI=on2QI_b^`cpH~@&~6$*)2+rV^i|YWL;!u&S- zvm@&VXCp}Js+w{=kuI$Q$**gGQm+Z6nB=mXZnGNF%=IuP=@`yKH(sz6rf*1> zDUjrkJkkhunuh_K%~X(j@$9-MI=O9LHE9sfrC@`U2=&6Do7KIkK}XnWoy3EDo`B<) zb=LlsIHWvVkC#=MPUKacS2lWY+Gt*%0gMN8&}we6u8UFUNgsPLZC%ibgehhfn^9BH ztzVKYhVjPiY@VQt$xiLU2Tn$Rwt(ZJszhe7RDhJ3x(QnTCzq*Lob+#CY6Cf)DAxBe zpqcMPe+de`s@q;?mylS+_fUKQeDQCl(rDV!Z~e8&T2Y`H?j%eg04Ora2McS8#96r+8hkBsZf-S9s-?qN*)pQCB zwk15?X~SfYc2APbvXcI<`-*?TF_OgB|0_=o%O9#$`(H_760pA$UNbd$Fok=TXMQb? zcW_ds5aJbJEYtnrAvj;=tu?iZpWtV&}8Muni zBi-5t6gv1A#CL?3zU+c-?Z;YtMASqqsj}gkEAyT!T{!CyUyIFv?x1sWi`ZNn z7JAFvQ%RpVcxoCQ&}_GiQ{R;L)^mqpHqL3y#c`EH+cn28kH8!Rynd;Cl|woxD7qF? z*GQ>lJA+(=^tF*f1BN6AGW#Nzy@@m;YL&vyQz_`3DSZ%55`*(XT-3`~q!2tdJtG><@syK}# z(|>R49^7kX;&i9p{_uR?CjvWR)xi9GldrnAVmek}=5fn06cHDBzT3WAOrt%M7$*qt z%{v#HgVS|sIG;$|?E)a|Duu^afaB$zt9kQnw4Q4UM1~hLlsJ&acJj0*h3&MBY8d(; zQL6pFf&{%4v$%>P1B<;nNYG6QNsyn92sFcQ&86r_o2HB{=u zs-;qkgU4W+cv0HKU(L;Z2KQpCfh_cf!87aKs8vEhn-R5Crw6y@;|2|w2S#~-)nXPt zBTG|WXq<3y9eSa;1|u7j_+W_T(>Aaqoa%B^P>}Cmn5+QlSiR$F$97!^J!XO7JxA4m zH|e#)JW59DVNX>3O9;>r4Nmjf%Dt#zy%1Q!Q|3Sm(VzM}7OJ^ccJO`7+0c|VcBn_R zl*<#E;c3?D_o4Q`0tq+Nva3R*C;HaiU#y3@7?c&Jk~X(}l&O5*+xoW5j_$wqV_15e z-+Y&KNoAm;qEZb|80mc!;L78U4e$*MHT}pzY*`+{hE=@G<2Xg-~+UGW20Esax%dBgzfSYUyST#H9Nf-`%V3yqvlR=IF_GsHlYlkED^I3$>NvXE6qWmB1y=7Ef z+qN|t+}+)SYj6!tf&_Q3;O=h0-6goYLvSyka0wbDxNG6gW1oA^-uv8p&ih__t-XKm zXH~VeR;@Ybm}89I`y9hVS|YY@G~RoTUZQ^~4-gvMITZFY8a%XSV-KSUl%2l25loYC zG4td=o#0Yi*y=-_@fYlGZla|AiCU!qvwlpl{Gj~3*?p811*mXP`iGDM2{oh?xnJQ_dFtDUe(0@uEQJl z>e%E*`}$1ZF9eG#8DTf8d!&)F#r*)u`n_(=#dnUTV2v4VBfA1N*QNvO z{_(Z1Cdb>qW?>KUz73UUy))HTtcDYdiMGjTK9N1dN+0Np zZ+@#Z`}FS2w;M%rvvwwJX`>#FbTAsvSxh?$qUpkTn@q=_=hPR?>VD^=q#ifdqZn57 zi(7Y)Kxnh1wj(hmiW(hn2Zo>b1Fo>ncReSR+Yh^QWRtJ6b%$owe^S`)qN#Cl=ige! zg$sl8#%qzLyS7=!)-w0N0cwE>QTR;D7~z4(Gl2H&0drma=pAXX6_F$h>wTg3SX2xN z;NA=SH>fDzAPIx9IkZ4d6=O=9?Q1{1b671^z96Qb=TbZdpur8|~jGEZCU1*otpYt4yhZT`iRO3bq8S^udMvo}ovk<{+%gQg)XA$22pU28ib{i0 zq0G>`&+&A>Ng=!(3HmDE5X{JF1cYd);>#}-i19;uq>-D?Mzx?6it1*9m@668c@Kw1 z61@vY`?6mwSJ4xBz<_<3uU>;f*wg=Ca3Y9=pLh{#-FCdkR!JiQ#0%=K%09UtCX%r^ z?S8qtt_h^$s6ZJqkLSIh@qYddU1-YG<2KU;MC%?KC_f?K^|dEL&D@S=%ew`tfHz#& z7q52O@6JQjh7?GH1x`D$*_U^XZPanZ_+x;8EzUcm3za*rD`UAXO6YD8u;4Bs$nc^? z^Xd7LAG`6kCecbb%jPyV-F1f{3dV%kIjMHF%BkVC30a z58qnsW`v@7!dH6(x*STQ2YLj^fSMrPj;BHFr+Rm!i1gUrDxR#+gaM~Il~TEZG&ikF zpomB11HoT$o4!4g+5RSfJ|G>7w&8MBBy;He7J{h>!1BH@jm_uN*Xu(PNn#^S-WrVE z>Q93jM^-hSJLXXXX;0_E-p;oUQLxoZF_w5Qnly9$e%A@gfL|UFi-np@1PCGK-$?j6gFOo$jQ@mjy~cbxc7P!CkfQ zC}89eBXhAREjjv@^J=BB`_x}Tg0iFy$Alnec4@){@&j7M85d?2T*~MO7g1xPaAfX% zaPn_S#54yTg3i_g!iX(aV^P{R#e$MlFW8uzX^gXAn5uNOQ;UOz1d(g8dxn-sJvmaJ zg@T6kq%rb2VLNXA;2)3nnMpBlJ0W7G6qS(e+XhzEX5ajv+7CVXc8O!S^W{(j!2B13 zt*u`Na?4O9sPmbMNn|EAkjoPs`JAip(h@kxKYQGQaw&y~caIfSO+)kCC6PzUt=Wnr zvQB3E)bSy$@=IeX2jUzrP~=tw$8?`7uvqo$XuyFr7M*Ul(~> zbmFIV1-zb2SS204Z~#%!T_xgS2C3u*!~rd7`zty|Aax+lgQDFMk$2`rs6H&D5!AhI zB*t3>0h@rudZQ1nC#!)ZMC!(;$3N;>p;fQ)%6&Ha;v3%dgVsGE-Mbr+8YM=N)gNGQ z!cF#wk^%ZCujh4#UVnKuIx63wxq{^^vdxUiQ_;x|5jGp6>SZ|8ALe}{_5}Te+Q|QHZ^Ih! zN-^9;ldT0U@L-AIdTm_ua#c4gXKf^hk$jIfZ6+13@QTgbi7qiT2I}c_XCeEpg)_W$ zBSq$<+!C{+@5!PaMMoiVrR4(3etLg??Nfz#F(rK<8}Sa|e=$~&E&sw;U79Bf6v)<- z2&6Dw6Yl?PJ`HlE;9IPxFL*y?D-si(QIGMniAsCe7LIb=zFvRsBQWF4zSf4hQ9HIv zvA{`ts$eg$q1JJZp>nhS!c1^#wLz59EMW41TW^Y;2~*raGwj;$HM$!{D?}qCBUjck zNTQbnY2FvijcYNunaI}-IApxtID8p5LOi}wF(Zcg&a0Fs>g=3k{~3Nht>iD@L5T=l zko`XxvUjNv;s5U#vPU|7B3TRkd1m>+LuDe*CXK1-pn~Nhs%cIuh~GY&l20wm*BCS3 zXHj6r6(}~2Cga&}aV=h?q*_{x z>Z3_Z48O{cSJvPwg_vrvOTi%0{Qj-7 zXu4EY`}<*6kOoZk17|va6rLFk%LHFAcje>=B_`OU5`vFC`Wz61TAOO5X_1T$eG!f2 zs)x36x%1XTN$0yjipG znd?J+c3<>D!CN4WQ7#R=@QSl%+=Qd~0;lsauGICVk`h2@6RoY!t=ZGdz1}{~Nh__| zANvYLXf*w9s1%|wEXS_Tg&o;{l|=a?M*dNzFAiWH27;p_Yae8=+nlXNgu`GIc6z1U z!I3HMFBQ1%qH()-uCb&yPnzCO@>JFrn%@bN)q_T1aq_D^$(_SIs`iU$TKK@YEYkAV zkR3K!?9Z}8N^7tT78iKX;rQ*A4C@8HpGl?Y=)j~P5Qt+tlI6_cN3l@pmKbUBcJpeJ zqO-$6c+PVUDi24n%vb!yi%2(g2|wj2otOg^D1@0TZ)_w`dK5~rsx?|TWG;;g`}nm`ZqpO6+0dM;A&B<#P1n6w zv+*4KUGagY=K0Qu*r_<*YHvF|F-L=`w2`fbvOw{u6@ZD0R7r?MNqN{rJRRQ-=XL44 z&+m<&(D~NSZ6(@AEqbcKO?%w;C)a>Sr;1{77hI^|jm_LrD!XwHFzsw7^&RmmKgyab z-rGT`weax_ZX>MJae?@aonC}^=aUT2N0-`<6rv5tAS{e;1LC!$;M?gSvKqLILe}nStmAL4(N=V5O!8Rg~sFTP^GNVDWzEg0d6Ak z{Hw+KFF)h+rjc%QE6bHN+xD8U2O(^3E{%4$;5Rj-yORlQ8B7i{&5bjXXKW`R0pEm z(;(*9s8ex1pMGpyJFim5O;sZN_wpBV&s8JybOZn6(U9C0J=wIG^!$7^>7>R8Hza3^ z&liTZAq7tzmrC=QzbvOA25c}TNQ4-?q76@Mk(@qrFbzj*=?&LfoLKX;hc>?{AJ5vp zMNYQOUKzc!(U@b=BhqzUlhE}UXKvYxXbON6B%!F?h6+KI471Qe03^T+sxpdUqgWUY z1K=br={MU|duGd#*=;KIZ)LL_(%B13F=1WXev#!wCD&(=3tY$Fm0NJQmPWYoDs@@l z3a4K%cfFKL`D&_#oy(Z^==;kXbi~}^(Zi|n#Z(60S*Q_st~}#rcbL*Wpx_5ooV7se zC8hKVj5^r+!8Wc|_hqG`_4?=G0iMq1Z6EuylVbvR9Pov~3c9o1Xb-4e_*hbnO5JPa zO&jfBH1aLivFKVC=8?VcU8h zFt5zN$LdzliFyj*WmAw|o>cb!G5Wn5-r&g{+M?Op|4ViY1Rm;Pr0{cy;aCtDKaLV* zQ}L?pz(TIe6#BvSC=8XvD|hXlBp%yXQb{Rz8C+eCrgn^E0L{7UBMSIuMM(I3+_s>g zAiRE?bK2!*QdbK8l6^EeIVE*_N@H5>|O%sshDj&?^^Fc1+cB+wzdJ8=& z{cDoTS#W|vm7$9%mFEFuziC=>puX{{=@CRfF%!BA@st_q$Ch8vvKb0=2lF^4Hab0+ zWuilMP;3!Owl*)K65C5gRx7YYdTn`yPH?K6pZp6rh>ExBA3K&J(Y3Api(Jf1E09LC zse6^O!^5JCi?Jfg8=L5kmPiJKyx53N3}al#ok2M=i`j5CE+3w+@`5JYBq-DCy21mE zCS8Nxdc74R=D57U06%QZN-glpbHAJfkkXqh^16|7=agzem4sJ)wALZjjq+FH?%`r* z>gwqX87h=zc6XfOtg$;Mms+WUQq94{1zzcwtG+*<)X3;e|R2r); zP^R8fbq%IM=4TOTvt^?e>nC1aj#4ON#3sXoH3F`H<#OcXQF!SRBo^QuSi^X%=e5C& z%utEghPda|O zg3uGb7m5bL=G2r?n@o9}@*{iKAr^G9m~M0pK>*R#>qj1Xd2iA)Y=MEm5MAd57bUrsvKh9Y|QX37>u5J#?^8{7*7 z^1tDLtQQ4`_+nkus(T1;9?;nuT~Xx%lO|76&#L#HpmJoeeD~mJ z@m!pyGFg~kps_}v5<3%Yy_k+7vWJX}ej;JsQPht8kt&{4(KY$&Ipi(tYxt{&$JV+0 zymUSSy;$i4);}gpOeTA{zV`DUbN64L;VC3KqiP_@|N0z+o{zy?iTTf;-+l=D*!ZuP zKu-6Szk?m8HNHJiId|a>jIlHjWv~W|f*IKCmSIgz4g_AinLOFtenp6UIqP369#DO} zcsED(=Pj`*f}M=!a)nGu;roAuU4{vhg-^}+8SX8SO)XcdlKL%o@nd3FC}XL~riw;u zDVLcc)!ykfn(C-gV8Jh>G_;0iI!}sCVlX8PH|L5WO9l8@Asz(c7zen2f$Np zCA4fEA(n^&*<9z}OJJWxXc6J&r7DR^Y6UD(ePA<;!^*^k+e);w>!r5_Qy5gE!Dot~ z2g0KC=yo^jj#K?LDhaSCgmmOwQtdC?cej-ylw^Kmz8eyi0DKu_FXOCNIhj;%l;*%1 zmA-gFl%7E#R(wmNz~BohQt&Kxf!pcFSH{!*E;{1yCz06(;KheXC0`r6sHJ;o$8ZX3 zy**%&icp#=OQ^DExVsB0W+)F6XfU@#Mmbwo+ctZyHuwdct6aIx7Ra(-t zxblilhqpbkte#(gEAO0W{fIf2HD%Y?GU4?IE(m&(KX3rU|dufX7#3@`bu@pm=%wIFo6em<70%X_ps1ll$5!Y_fUp`H#kn zBo#3p`McF;Q-04VYTDP+^OwB7)*WPPeC`{t-6|^Hnuiq4vH;SHR|PEL6968tO4diU zMi=%W;?~`f8UK;H5*Smu*vOVPF`#%#_Y2BOfmy;!O=t`6Qsn;GA9Y57Li7jRPmWr9 zdm#Sfyt!%t(>AN^`dIZ3%^#PPzYEn&^;DZ2`uFpMjI8vba3Y4m;5mTqA~#Z^+lp9y z*mWrVN&4nk+N7L^%s;Iljy1)Rrodf|^OPU<2$dA{;-bR-1_C6FbPQ6=v$q!B;fKZ| zA~;_=|00u2!Inm;P&dB=W2bnu0FDc3?w)gC?e*)?QQwz`u@e`#WBK2s-qfCbfXt}g zuI&dpvf%ePcJR!{l*e}8vh+b~Xvpbq$F_tt)mEGIWU>_BeT0+ORX5(%80Kv?rVK$N z2w@jt>Mw>ea!+&hs(`4Zbq9=}FTK?hB^`{-iPb}sXE#29m22cyBmvS?Wpa56xcc|b z*=TZkVd_e^8m>gCEfSXNa!UMEyNf!Ed?AH9rv@XSx?hEM3gL-7cHtko1bEaX(hJMo zSAO5rkuF$cI=&^No?|dYgo$&P0!jE}JiXvFqabbT6mnsh z5eyQLZK+!GcSxa8g{#YqWY;760$?|oysJMJAnev1?a|-gMSYv=!YJd1{aG_vGLu@WkY7f zEyQW0$&a!|#WsP;#c#5eEo(#0gu`a>42E)=k6Wg- z95Ld~i$9VCID&W3kX$^56u?iOBafj46QsbwvPAIFbNc?|K#3s0gS|wSG}Q0TGetQi z_MAZ&@agl2+W;LcAgDUcf9Fxm{$L5_sdWJBDDZ3T!8Anjo%P(0o+$@CB@Sa6-NBmP z>Mc?DNO>lERZ z*{@*Z++?xt2ySt|92Mi_?Er*1-z9V(B_#%73tZ$9sL$;u^05cCGY$vUeFf<5u>d+G zq{vA(2*4L2xoht%M+B5R#g&jW<~->H8`qV?zcK-X z!7-SkY6X8cInjY{z3^Owd>bN#*Njm`7%#j{^v^v*FxmEdG_7Qrw#_>4Jk-pfZ(uqZ z%T}yjKub=OV-MC5zI=kc!xr6}967A4{fv^J#^Ls~!r852gktYSx|mAkS%eX0TUU@; z*+tf#kGMb)zQc<}g;$y2t8QCDLhxE(Dl-cBQyAYjEz~u`^|6x{rXT%bYF-ZJGx(YW zz6z;ry66h#FJF_}?`o4Q)FXIAc`=DYrB?Y1+Q#0gH_69&3*&E`373dDg|$|N8gFcl zNc%^c7Y|mZ(Cx5nh{cQa_7W)AJ3GrQ6F=ODcGu}R2o!ZXnE;61+ zD({&-ll~bA6b>m@W9%nMZ`l^C-;ZM4o#;h~7PC57Zx}j1&J#5F0Kwl32r!QDO%b0S zFEEUaU;4_pE^AM0XyqL20vowR2dw(L)N>0zxg_D$=NBfsKaGl@>XmbT-Z56Ax!({H zRKzd%v^DY#x7TqHJ*RM?g`|<_$dtEWq31`76*uvgrGI0BN5eDTar$YX;lNnOkv_WsQkg4wv{>HK&^>|Po~+p#L?TFc zJ`5X4zmOh%0DhyH_%)I=*>|BBagp-8{?G}<Lr^RbKxr%aQ;{^@4C;k zM7rDRW36f9ePwjLD%Zldm=`Q5p2lSzqRvc|pI{65yW>91(JXf}n9TcDM5;4>l+CV| z#M-8Pz5RZa{pS_C@9i~6b`FxNVw<+vfwr2MpRYYPqY6d4@hAYowRr5F%GFzwT6Zb3 zb0OML#Eo+4leh!rw)~P_q8yms-?J2$e7gvtH$C5k1v{o4G97KuYgZ!jV)O(I*1WHD zKY+{8yEWplX6`6K8C-ZvZj05toXOV1=F4yADZ!LYHt=G`85ZYSJP1&2O_E3%xV0yX zP|pJM6N)^IRU%Ajo0a)2IWRkUF?=DO3ZQ)@(sSZ@=wIA!buoYX3YjjYYHDx(P|0Uv zHlw%GHqrNSrw)(K{rvA!Hz|GUcEo6|h?a@eRE(KHiO){#E|)H<#{x)JJT`r8_d-f9 zzML!~sw~72tOPvGti8qXK8qfe{ju5A`=@pGB8G%aW_RA5w(NKl&$oB35z_^WApSN zfB8ZD?K@-Vdgl_BZKt(if-BMw;xDlZAjR7pj3TSZjRuWAFW?tVVE{CL22tMEhqgSI zXQN(BGn(srCUUeyGRnghp$;!MG9x84k?uq;p??|acuw;va@KiYcq)tJfO9@x+`5J~ zW6O`E%%qq<3b{i1+|1B|Vx7O;qS)BgOo%H}Az$%DA=H@Wt)ji~i8N*775-p%d}KlH z;Da!j6bY4>J7rZC-TmZ_ z2@R8B23oRNsgj}SECL>WVwA3*1!wUff%7f968lOMkp@xjVM%nFC2!-mrWdZL@~Le7 z9mD_3c!F@%FH;+d%XM4^!?Y@D!3*Q(+UtochWJ;(z95jh_^kDtX@5{0bNO)`yPaNhsaK63`%juiGgrYP?85I;v|3 zeqc+HCyr)M6x{eBg5PW~q7ti^5gNf$QfqJMcjaauFqup}r~kJXmGM=oLK-pTzK~{e z;ppu=$-eWwgOAE+MhIY^1mEZ#8j~@?7{`wF_=_soYZFI(Desg_pM_qfjS86y6{&NW zkci4@g)pk;w%1{YS{*^^K2y{FL3)Vohxj_)bspIgB)cb$;Xa0zYQU%}Z9XJ^FCf_I z2IqD|>-Xnw{Sx17^EMPb-w6P+l#x*;r(D!O|C7p+qo|;_o@5fi%5{e)!F*R_Y35E#{0p7JLx&f5 zYXW6v8XGx}p$G~;YS`;(uR*C}5rS`FrSO?Zh&>xsiH26W{QiI026{rUW0vv@?Cn`+ zP~|5L=NdI8lT=UTB&!Y78ZK0sC3c7{Zv8$unAb4Rce?_Mz^+Mn>$Zeaer~K!4zuR- z#4c;Z1H^84EeiVKBEPlzY@T7G8kl;0kt5!kE%S8f$ze3OVXR2E^f|4`YDVbMoUu*SO$~UILrxxi8q|LdtOmfB% zE_lfA20;jZ^8N-z`fOx3xs%!nC5S2nfyqce(mqN ztB<4hj--J$0M6gV!qF_R{(ES~=$Gs4##~He^w|cWKOhW#FJz(;g~6IztNd;!yh6Wv z{Z;ma3qAIcVf;MC_lEaf=(=sL5Pr7c{s(dbYftBoG%O}RM+|@mM~*EW(@>GFFqEAA zXzlZ712oJAvgKA6%%)$=qy1i*N#V$lJk-Pl^@U=3pK0ylOJ(|(X>T#GfIE22LS7{i zN&bSWp)`JX8XuFG?ch6n{LNW%YJRuGjADrLdae^g(?tF?w{frdbA1fCVeJ8{ABd;N zjc{*7s$oNS9v$p~bee>wZyiKHrX1W8RkYq+pJA>})17c*?XZ7_dz{aU@h)J?6)h}HuYCIrou!zXF5~(5&$dlB1C&I~1 zPJ$$Akit|>SgkcLLJjtX_8K646SB{R9pSr~oTWvU)7fMyaQ?ztg5r+9YKtXOohtF7 zR^IlYmq@l)@)J5`5m7yBn_a9X^4h-`XvTnpB7abDu(X%cVWZz9e_ok(Qr|8h4iam$-B5>K}0@vT`iD*roc9yb5HRg+d&Dz|Dvfou4Itmw_kYBus}kmmB_t zVtiN-^=TR=jUwd?JJWsQ`vWdpxRb#)f{A@Dd*;)06i|!)2)a0VyJqmV00^1oMIX3 zt+UP$6{P)!&#-IxZlI;9*F6|R>zEX-L_Ol9Ch;LLE5d$K-vq*RH2jywuX0)$n7NHXr*j1`JuAClF~XP3X~>nGd0zM z{T1Qtd~=#PQ*$pMXQInQ2R)e(5H~3*S2yOlX4R+l#o0w9j7JWFewZ*8^fFEY-V5u} z^y1pnF>o3KI`1W@o1-+ts7!-?;t#y~#2j_A4z{?I12}hL3z4m*#(EaB=h>=^;eA-l z)`U9)hXBHwoqW)f7Z+-9e;Y7tqmkgbr=(R#8&f9GU%L;wm<&YTg885Ms7xfww zL=S7g*u(G<#>^{u?V0!?lH06BkG)bK1wBwcD>3)eM6p0!a=-irQ@QaVPr2%lPOpv7 zlHg!Z>oH-@q32t&%2W}B*b zHCmdRU-|Jko&cY12tgq-5d7&!YFz#a6yzRS|C@QTEu1lHjQ(w*Anh;|&ElUBdjX5J z!!yq`-w+v7L~r*8JloZdko~woO`bhGUAk#VJ?l{hU`o32RW&^Hm^JwRR7psH2`#qA zlonvpM;%i8?91GAEE3ZbXD=ZY)MK|`8D(Rzb6N0&43Zi71!R1JWRVD6ADYnPZOlcw zQp+ha;=}u*j=&tsIyhe-3LhIKp&3W!yH*cdU4{JpQqGhc-|F^(iu#yI;pDdO*n$z< zok#&Myc-y9Gg4Bh_XeJegh*_<@q8FcdW|ZGZORS^TM@xEeNqsfu+?AmxT6;9D?lf> zp+&X1=|{IR!51vH*BoXjm&y9hS%BF1PnHiA8^yR1>&xfP{zcbA$F|x2FWn zbO0?h2cj=DF=RKymsLESC&p6C%QeR zw=ED<=eMegY^{S-#8Js%|3?OoO}2HeL=jM(&GMLERmf|p`9lmCTT(j)BmvMe_ny6S z`d=ml+p{T2Ms45^WV{=5s}1$Z!B}*rVK*_rW!n1&!36RGoRmJ?d7V+p*}Dw%OTL6z znIj!(!yY45;2afLTzA(2$6=7D81l@{5P3g2(9& zDTJe_cPgis>!6UUZGq*#64&`hV75#v`{t);<0}4ZcG4#Y(yC)i>e6avZdT^qIDK1F zTFE2-8BYV})NV1V&sT0Saku6TMOmrTKopV#6PN2pJFrCeeAZeVIH$H)Y+mEGcY5jr zv>L*flez8496S7&2vKbR2Fh_azZkOxsTVIPkS{E+-!uOu9x%V zappME96o3gfGGHn2{Dy#=-$HeFCIBIv%dg~ye=HRod#IXS?#%kWEKz`rBrWTQ??z! z_H`3tGOGj~*A;Vl*K<9!yF3rgts3zFF|ep();O#o?yffcf5Q0q^DFmqHd=+l_qt;T z37`$KE(Or}GF3;!KWLAF@4zW7{g{fM5Hd5n|o zR$@(kQn0CUkt}GI7$q`QEYw4n2pLV!?lCP>B^&T~&o54z;?=DC3ejB>>qGeKAzR zf5yFJ{I)HlkaDRsDh7TJFa2$6s#M2JHI<-eqzNHm5{~LAtL@S$2TU-^npt!@tj4QV zHVV$HOVUtha+oJJwmnMItWSZ7T-kAY|Ma=@-=rk;Hs2YEPP-lXWQEX|Tpa`GCw>j5 zPv&n^{0Sg+nUAtT|=ud47)7m!k+@lv45R*8|VlKqgvl=mD{qCiUWs{}KbSpckMLI&^ z5CNPPx>{3W-}%G$aC@q+sh==Pmg2KFg(i-LLiUPUR6O3zHwQ>Ilscvk-SymHUQZ9Wv(*&w%NqIMvV*rZA<9I9bg^|p)^Tk^blNy#zi{p$V z;s1!X_eprb)GWX2ly(i-7`M60TU*%YGR0`hY>VbOz(ap?9z2^hMx%#Rac(*AW=?ar zV8-9%g8Pt|>8QXlrBy>KlVDsM!U9MA--k+ ziCBwRPY~m}sem?dxJFBxP~UekIcn2Q|CfaHVmXz=HKo+h61;(qNI+Q&WP!`-9OMK|1e+od3v zORxN=XLDWOJEf_7vi7elD_zhTlvhT3b(kWaZTT4qcvB z=j4!^m@+QnzxiSFu?~@K_}7rVRH-85D3C~;FIVyv$S{QuLX`k^mL%;hZg8dd+KfI; zEk}6_p~rYSglTU@gm(|*PON)qHh7}K&UYZKg|7izR9%~wBR<+>YHcRIxB2qWB6Ohx~tSl7Hf!Q{?5SPem4|O{`rD8hV z*mJ8drk!X1bGxQjrA8s-9E!ZA=#$YlA@@d&kvy)fP9u!tL5liGD8A9&0M{qY-}@-l zwj5d{g}?8FkEK5@_NMi4@>SKQ0ofzgQDT_|m{2fS*hH;#Awk(G^ekLXQYig+j8u)g zm-}bN6vz5`a1Zc_bS6-%nPPvn42OyNt@uhc&&n=;!SJKuon*ZqP#7s(^sQ4`d29&t zF}u$Jyz2RF-$X_q9_|1ltq9Hl?;b0C6#tu)AkJ>8&PB(WjE~euUWAf=6P7OZpN~No zTb$~%^i4<+cd3}bXSc%PerY4nN76OSaK}Vi#icrrZs9>a;4ZTsm|6JyC*-Nx5+dLIHGKYZy+ZV@k&3!!cR44{k_!g&S=UDC zKjb0XBJa2QEdQx$y4>IpZFKtdpFnkg(aZ2fp9gI3tnqr}^EcVfIbydz5?+$dT?EUN zhEEDvAy-zN&=dAF-LKbXCS8hX6{EqeH(sJ!}a-&ASV& z2nK}=42&*}XNK*rR~%No&Hm8nU2tR1{;Z^inRH`?TE0Eg*Ii2C`=u`GBE#t;fvX1g z5V|4_L9)G2WBShyyCb7d8KSk#)%kTD5=S}_#^1{+3Vq7P#p{+8lnQNDa|Nk`&@adm zjf~XPK@dpl3Zkjy9)466ntKWOt|1@}Jk1W_C=)Dcz!>!x_Y<_ilhlh@D#gmr@~X^w zuw0gELDiOa3tj1!uaF(NAR1dnWKYh9ygMZ00-H3Eye%Z^Gwgc|mH*5H(4l*Mg@r|S{5G>`=&gJr zPvsX{a(lWyv5MdEl=NVC5#ET(|7&hwdHtP)Nw}u(bexD}Ql%V}vx{B;iNNj_5tffF zVODrQ`bmkGk-i2!WJ-(i;y&NvJ`zTZ#83cSzIkO0Dr=(jIW*q_MGUjVk3AIF@-G03>nK`Qyl-!kpu zSo{5)t&MwEy6FCqNKvxvAST4%2*%_{&o&vA=4n6ynT=WO-kBPYT7HGG@w+ zrlK=^%<)RFI8vkgu|ia%8Rdr`fqExA@QA#zerk`Jm(t2h#Lw?p2PXy4Hu`fY;A0~s z7QbVi6Ka02`Jt;kf!fe|myH_Te?R2J>tRS`dQ~2ZSv!8@Vrd z`4y=d#(r7L7HPLhXa&KF5zUA{WFdU{Hsi1XxUxHO!dU%82V=!6C_Tqs(Yh8%?^c(n z{xPwbD-ZsyN4)@~!^?=8L@4oqos}B}|CapHfsA5#8DkSP9AFe5DQSDNUsr=`mc7`M z&a^#FO>A@{vhJGBubE{*D)8z2F@?+C#)Phv)WUMK5)Z zx~fR_e3PBY>yMTZ)9sY#Yz+f@b-I(Nqd=XV2GA{-Lq70F zd(f$q{Gs1OOmlpaZ9~ja@|j-SdY4Tt+zS!=Qrq0T*Yt-P4q^n(@;@_Est^bNiA7R* zr5JeuH~^!YepGT-E4sM&_At)(%^CJL9}uxv)x z<&quAZ1yT8#?8Yb6G4<)7z25>)+N3K%CTKPB!MZPuL?wV;{UN{JflF_%VA5Ym-_fj z*xb3(RMX;9(c)8Pu&9cQiM;8{y~+LZR5o1IZ&Q=h%|UXBf_LhDT0-9}w9V}+Iv2j& z9Q61-y^3k${OgLtnxlxUqmC0xls3H=Ok8*GvllCc)lZyNt2(C9&q~$70v+?bI%X`d=(+=D!sqtm94&jZxxL1SuUh?#dS0nIM5sKiprQ920|%_VXaR% z3EbRbq`Q;4B9}8wH-@&T``3iNiBvOq)%B%KmL;6t^_pGMuY6hCE1dHF!%&=JVk4S- zuz~I)C99s_c;=C7q92#{jGpf97q6$Vspf5aB&;AbdYLRz|1Eh5iKL=lC!DVSN+htN zTg#S@>3LJ_|G{acjx(PUR*ocMke*Q<`2dq%*jjU!=$M*6$KGj@LykZMDkC2ST(##Z zXUDpZhWch1_{;TlE>RQ{a?ki=X7YfDp&go6TI4RIk{+K;v16aUy;#}@LPl|nDmxizroCSKT&^d=jhl)D(E_z0++2cVhO@V?0|u&hrqXKRvC2g=8ymL_2)Ae)R5 zLE4d#^KS}P|3x;EmW)Sn=0wU*YzTUl5I*d@p*#@HXT1P>(Y_Vr(S0ZgG0Y&*NfTLW3#G&0;d}2HdF;0?_=q)k0&~9%np-Mq zX`Mq}NdFsQ=lQ5pAKe=ZtHt7%|2J9W*$T|gy+8KfFd~z=Bto9RWC!AWU^Rs}H;rCz zN~`R~Rs*t=Z+oUoeTiaem28BDhCMg{TK*9^#6?nwg+w{2RF^~wcE!WxIyXbuD1D*` zqMP@i?|C~=cW}Upv{`A@a!fmC;};~d3;_uGDl zLbLaTY|JK5H3y7TkTMec)0|&}m*1ZIKefzI|O%kcc*c8m*50< zm*DO$!QCAK1b26B;Op#tWFN_?ukNk;hpMiks%yd9Yt5d|9AomF?nzq1Z18j>)u@KJ z3cWC6Qsqj~d+#Ax>K?*B?e=4ys)-RziJsV?uP-EITzEd4Hd%%DYtr&ifX^z|J4p$j zFc8+mh(CiVDd!Y4l;u=sFb3tK0XU^yuNrI4+|SWS zpS8~G`N37BwSxy;N(5-aAIfrEQ6tt6bbwl~O3C91@1MbZ!yxL-BoA{5B{(S>9Oe5YWecz zi>YUk64+c=ZNB8}y=mv?HN7N9H{Leqr!7mh0<{sPtxOu2zEO$33>D;C zx50Vmy7b^e`Hzv&&D9zYZV?4}JP3Wo0MPS5Ouv(BXNG}Av}ZGaQhyaXATXGS%ZM{D zFa!2esbN4UMO?0H`jz@yUU6AheD(lcw3gf~E9r9!Wnp!lJnexpruLFYMD^R}59rkosnNA9BYw6Zza=`1r(kPF6W4LcHPsYlF-Me(2(I zVk<<`yWxQ zAz#CV&fD=f|y;! zPXq?)7)Gx%i5P+2v}i4i!_u1L6&ay3WKd>>3K%|2*bW^D>-#I}i|Dz}5=uI|wWMxt zv?q_sG*5z%98TET?WasaqTCX#J-9;%NSHS$EjL)9OU{P%N7)tEm=#*y!}(Ig_AgOZ z$hJS}?po8Iv_BLpb;nn5eU*h@+{JDt_VEit&k|56#53R7oq~xmG8=JI8>m5^j>?ZS zYa1N7j2)o7Dzs#O>w;lID$&X}SKFYEIJmeopM%@ICT)&YR#e84e>)!_Kbt9O`GIDF#C^$Ws>y*2*mh7-h^yEDA5jPiMHtZOB>_E+Uu80 z&$FRLt{(%4y6jA6%b7|>VD*fa!H4$zkuu$r(n5Gv7bDEPIs{s2x%Jm-VL0Gd!&^gz zR+=0YXNBWt+1YfZ(Hz{kNoLTpfojaf>P9yLDhk70wnk`VkH-aWfD(5%P3j%89xZCvvE-nXwM0IGpNT<6&8`>y2|}mopFN~sIp7DQ8L=hW<^n)!ENiBk1z{#tGzSFI&8zSyi$4O zHa`$5);K$BxCGns`%Rb1KE$4_z5q}8{@b8(jWn}R|JGt`74{wO=vrnk z?W<3d8NhuhBdh}^-X}iuHOZ-oCVY9^ewp36wabdGNt$}1If57c_0hM1!s89#^p12z zmPZ=+5Kyy6+AZ2-g>={Ie8)jW!VM#OugxBAo@{tO4b?XS<=P6@Q#KQX(Pj)r+A#Vk zR=3`gs*O8BJpib)Kp#Rl+wkqikj8>>9p&NlTdKhl=$4KPLvQqxGwd#(=gX|!-Sq7-ZM!gYD*5TVc^ zN-PB@kuDKTGbJC;B4V~te!5D^HKn*%dAC@f8Nkb^ER^WT&?hRk)R$@lm9Gd!lj?BL z;soaskf_Qt`Xl);84<+taIJ}v&A7%C?j#V!JfbBs8l}vX5V)t*Ofq}tBPIu@EW^W8 z_gm{&ADYY5eHR(A;>}XTGgtrPfOLQAX-WWYe?IL^5F|4}cdV`F4|C?G?;K|~V$Ay= zbIUF-=W7aPzn!$+sLVPXC*csrIbCrFFBaBqy0(s*$93`aQ3FH6?@V`x?!v z{%T)wpyj@!r%*_HIM2YZt_|%ttR$S#;v`pVa8cLSVt{Iq8)IbQ=#Uzs%-Nt{u=5f4 z*#Sbv#5;ySuw`~*4RF}=ye+uXp`WFjx6*CpV~hPtqG%w4_u%kQ;+-`f*g>Abiq&)o)`Dg z)(dph2J0GX?Yqz({T~u+tlJ&#RJcJc!wLgxky8!YgIVdFnd=CqyDc~~xE$YftQN_eGRYc{oq8)saCA>Hjx|{A0Fsv@{d`{Ve^A5E%%SLSK^|d$@;n5O3jX&*c z8Qhdo*B$4ok?Y)dxx4~qv72G9AeX!58(`YgI&g@j@wZ%becHBRbT1$`_|c;$9Wjg} z4h_wSs$UyueL@P2^pVepf6A>Vgt)oxg%}jvWTJ zvXtx1RFJjnR$~zz@$~Ynz3KfxT?tICWtd}ltF}4H__x|dO6X8?b`RkxT?My#Bp$lw zzArY|{Lhn%S^%nH2(@#e_gQE6ve}%%LD?(?EaUj&rr$=r-g|51Lw|OQ@72~Xzh=kz zZ_wAGnwnN2a+BU~>W&a3H&4T?mU3^*k3flS&=FtkPycizdbn|I*E5o#cC^drri+ z3v<99n#;gn^Qr1HRXf&jY-1159ahU9vhahmb&j31eBvK6q^KWyK0OfRjQ{Lub-6=A zyE(sls+*Q4P88bmovF4zA#7sy2+MW&Sw*~W)!#zk69Crkecz}+;>qEJ*!9B;5EkSx zbjYSZW^Ik!;z4m-dhksl3SzO|1;JCKkd|Xm&Sjm_^2O|JzqclS)_7Te(F;*E52K=% zE9}Aac9VebkX9!U@CwA0b!v{Fdlp?GZ6I1{=4#GjSW)p`e%WdGzS5quGI4zrM1Aj4 zkGJ7{$?BG|*WHv=qYC9kJS5p^-l;^r)&#NatsPO!J+i(T1fASMMEZ0;m_=lJVOZ9o zjZiT{n(GuK+ReP6oXF9ygn&7op9qAA^k@Ny>v3sBY6m%$8XHuRy!LCK0)Y;WW#IKj zEZ~8;6}(FF3o;}|QAIAdVMd1?_IX=5Ih(Fm{Q zQ~Q346cji54qq6)YDxX{#6^7??_QElYpO0K8(xd*-w76=jo-5MKEY_RX$qMBNvhv` zH=)*1g-&U^q2-Q9_6Y}#`oQ38RCJvM`Pku7fW}s; z`OFDnx1EvnhykNC>DQv(*S9M6n9I7JNhafYvMP$n<6Sxk&vCjcn;!Zgr~bWuJMI4tR$Qma!zl)hzF~o zYT!DbNUm!QhSO%cHwH!7o&M-E_;=2Fzx+UmvgMngWi1nCH+NNKiveG$#zH3E_Png zx0YWxH;Q-Qx$>LGBDq?~;iJ9l8ghtD0RE?467G26f95Gr6d2c$|27=ATKz9HI0)^g zB-kGL#J~qy{h7RL3Xcm4Q0fT(!6OVF>@cfu8*z_HdxF~O+*nk9Oz{XM*y-21q9!+5 zfz{SL*bb2{HRlyg8lWUvar$-EZa~3t_&#^g3h>TM=-L=fdJ;Nk@j&RB)KbTZiL-mS z2Kv0%mz|?(f2(|?<`uKx0C?O}jM}IQ)Qghfj|gbLL<@)+L#a2nL?4#CWRZ(Ybp0*X zco;cRTTJ9xh_0Q1J5{0bH5?}8@GvOaom@%gg2M39Kx$~>G5tX+ zXX4{!tqqcEMb5O%WScOIK~#b{N-qv4?~#apX|QmOw&_0#gYJ^DNHDarbU#|Twc=W7sGo+A3Esj1khv{$%mY^CkT7WxAbW(Qj%AsYMzsu;#NRA)^-Z|Hd_}0ttm_ zTE~m-cradK@Rgt9FDt2M_hjV@b-!cp;<5$jDyAA`7@}4B<0-PP^XZXy_sDDw+Q85^ zsi$=NV7owD?58>T%2W#=DtWp3kxNd0 zikMJ~BxqK41zk`yIzYmEey?l~*&{PL>qd03IaH$K!e~PC!$+&5lIR8^hGXA3f|uku ziKx<0C;CGl{YCa1G*zm=!HE|6#Y6omOLXQ;(75IUGRlLz)NfLgLT|F9ch#cCdYzxh z*l6JU$>vs>FI`;S&)`p2BDzLoAAsCCE4|fm+S+Y!@O(;%Av@@B4~jxV!_0iD3#g0L z0G@V+sl9@<6fxK-?-ul3rvyU1uT()1nf}}pH0$lwn2%|b+mR8l#za273S=KwmQTN) z1xIx@xuqduT;bTP<-h3rheYPDk)wFQJlG8TGm!8eIXLf84!Q|=Q!+mHC$ZZ+5JH=B z_N6ui2-`Dv(~|LNOnsPV(fxD?B!Wtl-ZuG3Zg~^E1Ob}3P>}nT(#r7-e83a^tk0L& zI=b!Z)rqUUmnBH}+VNZFyl7Vv+5d5$#Tz-ua}J8s=sFtZ4vh{UXFy>A-;Oqt^UIA^ z>NbW9A?@qp`|)KwLHTjKqo@hr^32P)fH*wUJAdBIgx9o~I%J&BIDOkLwKcpVzcQ85 zv(Q(o+JT*mH3R*k!|?D2A_}X>_FTyB@e(9+6^f{u{Ojy{l(4H+!1_x^a{Sk#ymZi- z_2h7wIjV2*GK4z3?S7qWD*9s^%lCtQDLz7yp!PGY{t*X0ZN_0&o;`c~v}A9oM+|6p zIs0cwZiuVzoStmI!SU6MKsgIwjzM;If-lqsOoj7m-*m}Mn?}A~C=WO;L z$@y6qk^EYZ^438FU^hu+Es-J-=2Z99}&lbFx$pTEPoUEYF^tr~eZ}KLT{+ zYDwgVn1rB8`N!*$*+41Pw{9!A#N1Fq-A67Z6Bc|O4|@BDCOV`ld>5AJn=dV;kFUtl zm@Z#3<#ENrux%lpP=;jRA-&$^N?p*t&1mt@)|L0mTV6)OPm?B57S+Iv=WA@MUgdNYkh=kx-@ z%>>qC4@#eTcG@xm#?PO~%95U&(>JV+G>02MqP{bj?|mFpKZKOz-mac?lfKx9iUDOe z`X4X6*5->;b$pW=1<-{62Hj!Dkurg!~_>3=SPjw>C!z4|152%^FD}0?34i^sl zngw?4Vz;G)5efD|b%l|E?2WjXSP{#?G~+C_J~aH7V8G91bn|2gv*{Dz&j^ybg$K#q zDhTQz2wqijiK1sEC?8>A(T4t3-XL81SGAy{7-=l7y1`UJ_vnGs8P`k3wmyY_dkdO| z1dT2#py&RdML4S3Ajia~0b87PyM6WH^~yio4J++DL{$e8yOJO?ii8Iw<7{oM;Fu~u z6kC?{VNWt|to{7{$>Li`)kgifV{u#D9X~rJ6@81MLxZ$jR~H!X4(jghsWtuc5Q_6q z%D~gwL_l5hj+dXFgot#o?ACbPG#pyzilN&35D9MMUDSa;)rr?T- z`?d9e4#RLx-%9g+P`*WWg42!E$4kSNTg*_Kj6H*qA008#V`WMTD{n}gP+VVcf9{DN z6=FBJi~5#09d+{6Qw4%PV3Yo{Ri6!q8WDW}tUTJhk-e@&`z-qqKR1iR@*bn->;>%N zLgj#70i3%S$h^uy)`DdF*d?oC1?$uH4%Zg1`Tu!&b!LZbFqZI>tRuh+9LG4=&i zdZ7DCRQ-Slc0doe_nTSXU+N0LLNPH*BIRS^{mU$c#T+U5jMZf772iByuXNlY*N|#Y zeOFlGO_NE_>f{__e*iL{4o9eS`t_MqMteQ)|F}pOta& z1w*~>(NyEv=GfMut2;zu|3~Y)@lm^h>5xzidM9$MBl-sZ>g5{9)F+OH$J}DxfIL_Z zsk?q)W>Cl~GuT!kbzg;r-K6-d3+&siq$T%3WRt{uTkeD#aGNqr9qhZt36LS`s`@CK zLZs~Hm#_21-;+^LIVb!pldvtf_o|GBl$u(lJC9jfH(c0R5CS80e*mx@)Hl<0WSlZd ze!ZqXp)$cb!cctHFP?B?SQutMp}vBPP#gB!z_{3@fvmzgi>S{%+_Is~B5P_f*d+An zWvL5vqVX@2ZQlM?bX&@0H6W2#XOS4LnI@X!(a3+^692{IJ!$U10yn+)8Gu-8IV5{6 zymL2g$ns%y_O2(|RiB z36MhC5hFsF-aFV&1pVh0IIU`IN2q6C69%XpC0_`$Rpss+*5mfTEL`QHRGeP(=ckSYAjWq`N2#IWS zCTgIizMssjwBvw`^4c%(a@L{sZhLr*x1<4lPAC;dw6TOB_UwcoIKwIC{R$->6-Bs_ zfKnrI!t=M?Pa0LardFf!w`JPFGH z>VrpS!3*~`e_U*HgrPlT5$41eR9;eFLifN{(##6HhAwlW9`Gi@bs8Wyusd5)4oLqV z#MUMO6vX{fmpP{E7!C}_4*xpD2puC==FE8O}H32QT0 z`{-^{A*frg68rFwKoT@a2O6D59ru#9eL_cYJ>?3ls)}3uya3h#REG0vswwKsl3I8t zIN&5a6p~aj^Zuqpjqv9yOiA!|W;^E$xmjg#jU~v|pWLu0(G&m46NOqCI6u)Im}!6@o8X$H;Ra`8i~V+ewVTk+^F1A zFBLIxy=YW7k3iJ&TF2GnjNj3c)>w6@vC8O_G9ccucfxxmP}q>z@9r5Y_)(#^!~N?W zW(7w*m#Q{@$Z@3#+mWtdiSsEq0i`?J_l zvE_xO4dUb~I{1@R85B4WC^i#B8~iA-wXNkzzJ#Zc&J(smtikGwuT5AuiDv^{aJhPB zMO;Xd%la0&%|ubk^K&&j3Up{!a*2=Bd9r$?0BX>!XpM01_571>zywM&D8Ct=>xh3v znuMI0Dxt9mu75E-L@@ei+S8|_m2&jQnafpdL^?d;5pPx7L zQLzTO?HFpovlSapk-zp^C$beLDSC}tb!1xur=7LX0Ha2dN%uPmpvmr0wOwR`F;>Q4 z5D^^~n>CJoRLp@dobyunpmB3z*5mA>zaTxC%*>T$LFQD=s?oiD3_4fOOmyo+g zMp*vIwzg05lY0@^2=r5vHwi`;5Umb1oXg_>NW;Ti9q+-q9GD#{%mH*Aax=M3Qc@(QNKx3_PvO-dJ@$T3!(w}v! zf)I(7_9x1j0F|hgdvaX-1E14t^vG!Z0yxkw3jMR!Y7XoA7RccnmKA}+zKIvO)#;7^ zYo`%VT9C`I(9Gb1*{N^F8Gb_L5*ss$S^x@a(8d+^0vp3dZ&JR?>UB-qRN?f*5wWMV zCJ&SPHe2|s&t%$IrR2Xp5~J)AQmYp^oT2_~{J^@}=KKG@!G%Hsp%k4l0V z0=b1)ZrB%|MP#&xV+9o<4o}04?#bbruW6xExgz=~9TB&kuQf;8$y-EuZB71+b3Ibc-$+hIezDaQ zrkEyj7Z+mD7~c$sMn6)0I;bpdN>Ia;K0MkgK9rBBJGr1TbR|-qwpf8*ig02|pR!oj zSYg4AwEJFS#Fdij@Rbhlwcnw~xpZykM7fMENhuo*XI_ zZDpo{s$TYp)JJOfj;?9v+0l>S5zXd@QQsWDvMd-Jv)bpmMyOTjCxHNoHY~#0Y3tQk z;5i;1dH7VG-Ju0m-5tqIsTsx-N4?qJ&Z2Xy-JOpjz1J&I#i`A}y>BE&8#UC~R!{XV zrzjhTbZ2A-Bkd=1E_BIX&l{;{eeXFzh*jP;zbkq9kj=&`)$@Hp=Y(Jyp42P)45=GA zIs9?I{78m`KY9?6=J0Ro4lo^&*w%YN8V7S6S;*ORvRXwc; zp4pbVqv@t-mcc82XLnUeJbSPUJ8(OuH~;M|minyd&N12~23B~@h|`Q*CZKKTalul0Z<^|T9EX#9 zV~PcjA{AK*f3>85NK;i zHU$NkR1z>9_!}sXeWF@MY`J+9SXgI?#nYZhe(}i?%rtzHVSe&Wv4PZZXEa+gJNA-^ zC!LIrpnNxE(@BG`P>Y7P%zcufAxMf>@rPuIK;`PfWa%pE<6KFx^M_C6Uu3MA7+PdS z7nwH>JptH`n7h6N(s6d&j~w?4p4D}@4Ywq;iMM_B;g64_tN%F5Xw+8pYfCkdf1cmA z>w(HAyzJX^Wwni^k0+L?xx#7v^uq-svkgAR7U-#4{~HR4ojC3dTWlbJ`f63sW%834 zc7F+t#!{oRfrz!=Nh#9KU_sdl@If(Md3MgE`U$R}A=fzbli{ecs?Xvy$*Gp)k+4}!;;EO45q^IvAtv(A?r z%FiF{Fb?~425v-C*82Kld0}J!YC>w3t5x{_msO>pbGTuRlay`+u(URZMg5f2*Oh*s z;7;9H@!eyciR0#-U{DK~s{WNM4E^o$&2E5L(k*hdrwr{jd=!By}~an&}ECcf8B08SUnk*!WX|puEL^69bvWZYS2{$m`^Y^ zHR-=AI1Q2ic{4KUoE(XSDfwi;CSv7IWpDF5A@Wxde#?9DsrGGfRBq-?Y%H8UuRfur&3ON}2Yy#UmU#LmknHJ@6;z&rppSfu z`LdKhUgWK#v`#xx=YJ=J#7Z8;e_i7rzxV6MA2Q<) ziM@#BKj*tlJgN4Joskp_{{{ac&Jwf8nMoePSSo?FcmKkvi`sB+DB@Iay76DBAeh*H zNxPsw{XXqt)B8)><=>c>&fV5cm*<(w))82_|ITa9a}(!arMv z$IL9ORw1gOY4ocjjsdKFr~V-V+fW-E752QWhYM9wo!Q=M6J{2s7}s)3@nM3xomXZ~ zOfr2om#0<)6?t^^Zsg9(6rpwH*zv)?PT40+G!-K!oGQH?6vyJ!JJy;|C)%(2Ca%;D zzI{hMYM#2%<~10&J5SFrZlXv>uty| zay2lAz$cWj7`jCk1FjUo$6zb{D+aFvShu0odV>LEOMcyNr>+6r2Ky0r1d?7plTlJytJaa}- zLVG|r83WKxN$`)052A1Ka(GwkS?m7s?}gGY)Z(88xtuSqLY6+mvRs?%EjEzP0T}m? z*amK|bj|bstM{Qr6Zm_N7%1t=3#tXMnI9)=?s{1HL47K^0*IIM%gh|0W-p>a!0H*O3 z-5O@Sp>_pr)XJmj)wN4s^v+1G4fj zq|z1wNG+IQF5sKMgwhar_RzKYQvt{Rw7Qv?Z%T@)meBD1i417uLw#SHE6j@kpl=wX z&IF|ai9P6mBl#C=?TQGrPXn3OTKtU27p6{4U|FEsv z$aggw$F$7|Fkk(U^`*-$xSmOJ!M*a7QA_&y(;HE-MfLQhv%nuSDw{r9Dd6~5UA>0BfE z+~6-BDqBwtB4!sK%XN)1!b0bwe(^(Xs(4I_A#=0_p!bh1J<33E zq`;-0z2*Ochk}j%&yuItm65-~Lz@3D;h{18{{au_#d@3Py~UTQ-7R|BCEsmo?R2`7 z0A7qETOYCNrRQHh*Mz!kK(!|an17~`qhT>s=T_5*?=IaJTai{d*%~V8S!#fXm0S>@ zIy%H8UE>l~K<2@x<^LL*pcf5M zh+`L+n zqCXH-WYd&cVA`^e$S7(WTNPsU=y%S`#M!y*xn4^iv^I^Ip+$1Z2mpT)-EWor{R0z; z@A)~a`*}7f2HpZs!jWfVH`6GY(Hru!F!rYA#7*)lk5jhlzH4?$3tQo213m!5;i0A~ zb3KBk$71W{)(wN#o5z`(xi6$1n|?~8Q<+|}S#tPWkE6}#nOVrbG!wn|u)=W*+Iw7I z2-;FLeCg8H2wZDgH4J%ZZV%PY6!ud!EO{ib0(jxA_yp1*$FC^(pQ#;?oez&F!K!zi z!cVm0Ir>42U3(;mv9lB1zKnhS2)P!hxXxooaLcoCHSp0ZA|X3uzLGJaSnU}@Go@zK zqpveQm@;)K62rkMv%nU7ENgCf;D{{&i1%yqC%Ww$miK>n&6AVkPvnou&JjjHcJ==u zA^#t#l6urXQl*|2AXp~v`YpFBEbYG5N4k@PeP%ur&X2vB@STC+bL}Vsmt=S@GWY^~ zV&FM+LPVQVZ@K}~5Q*1NNN)+)e0CtMwRnQTN<9&;hcKjRD5+p{_+AFHwEGWgkwT!@ zA+vazLn7D?NH^UZ*%Lkhl`5m zmXNFVV<<~_b%&wjH$z)O#4FZ^qr4kW!1cD94tyFSvr7^1bzAxDOEt*3^185nt8xB5 zzo&zu8r?;Tb8bGEGf_80nywJw*nA#Dh0yiE`c=;wJMS@E7o%RuJ6>d_IktJ?&W=~t zUK8r>;d}gdfnK7^LJ-JlZ+8-N?WLoFJ^mAu40^{KSTBZ!ewZ|sWCn`Qv@x_r>#uKTe%hg z_lL&2D@U1sIBotjI+F7IZ;ty;&%|yDy6FnQJf^O)*Vmc1=nzt_0WpcS0W-&Ni(c2n zetuwfo(qPHXmNb9yf=75D={7J;T^mZdZua`EN>_Bdp+3DFJRr_u=pPW+HvS!xgtY) zMfPWXeU!xfYE(YcONR*9D(&v|?}S9G!q}W~hm2qRQ2o|a43=nf`Q_l@A$i})x7Bo< z8k+SpY!B-iZ2wXi`_Gh4=Zf_B-xEgS2k&<(DegJNm}He4I0_gWm4CsV*RrQ%XApN! z%BN&L6_6AjJOK6u{=j!@_RGwtcaC=|Op3ifJ5Mnq_jl=Gp72%udoG{Zn)Feq;h5&} zaMOks*($j+JtF$*ynMc1$<$IiQDajN&=0QA{&Q8po&ge7?n>Rx@lr9HmtV9qW#9#C zEyJBAbmVwOpclhZCF70}Pwk*G-f-6XW?y0STyd}lMPn#huUfG-Z@)mHHPP&XSI~Zu zsF_O_(Ldxw^2&GkeM2TROyQ z4J)X^s3hKI20Kwx5qN2Hr1ZzPQPa7VG5QQAezXm?`&@%f$NaGne`JkuOt9IK^MRMP zRIzixF3{kQ$|7I{lo6q$vaeS%H@!Vuhb3d<;3%+y_j>xawpiX&o(f0Oi_=9NY(sCm z>#UWiVbELhovqFcwv3S`r-(ag5k(_Y^1ALQ$8JxC{)G(jZIXkA${gePbjg0BAueg% zJ@{)q9tKl4L>&4_*W7XYqLA$i`KP^U>J3`9`ksu0R@WVqi{C{=TsO3=#>g1V&izXf zTXHv}QsJ2%?|ce1oni*sJE?>zl#1kVkY7#%Ol)F^DSSgwCHXEW(JF=77-ceaujULH zk`@#ht&oPJgyT{b4;-lL^8$8rT(?6!P$I#OxaoV%kSbCm`?dD^#eBZUd)^Vlr@x{n z0=PB|GJ1pwlXSbMJ8?6LBaISz7@FBsgPaKx?2NJk@(gJlw8b>XKRI~XW6P+NohYgF zymcs8By@hVEaSN1cmh&WKa3JCxDECt;EOi(C`1|J(}kjqRVNiPv;GMo?TpW1v%7pY z8>TE$`@`wPZ<2jMEp;(uwZx>K9=8pT?i|!YNj$DGu33{%yuyBHuf}#`H~up{;EDP< zITnPMgd1906U7*3j$8bwERe^KRC`WtGNMH8avRJi6kOJqd$UzTx(V77(oi{)8mxP$ z2I@m>kHp%M{uV(R6#ZWa(g@8z5u~X<5TvcDe$a5&Fl< zf)S=(&3}5F&LEW922#EcoUhc8xwAG3b6P+XK2Ll)LYFD!O{xvcT5ct$kzg!thFxb)+~qHsbV1tl0A4`#@r73IMc(~IcYdRR-kNj7 zYOm=8vhn-+=W4FK&o5fQqxVY%#~@m3EzOpqa!Pc|v~iOfFsFG^h)f`3=) zGX5kwHUpoZJ3dr1X2;9bZWZ%(MLTYs^(c%#CGy1xp_z^vtYc zZ{L%={Hz!&Z?0Pd?U(ReLajs^QI3x#oy(SHj^C<5N^LB8ei)Iji+8ntd#oYbjh{kE zCE1d^Nkn{NFeV-_tfd;*%Z|V6B;_OpC2eP%8J8DlCN(s3IUFN)#Q}np4hCisze}mp z-;}<)qA&E;!L9Ch`5U7Y31XBg$9^530F-aA0ENrA<))9S@EPgalVkIcG)p&~Z#ZY- zzeaB6<9j;)3?8bgKhOM(q1$NnHcjPZ%Igb}M(OHaj}KV>|A>Vwe-jI(v_E8|MX&tN zSSU1oUb%ieEAOf4K^6JX(IE)%WK3@{`6hz;Ft4-ceX{Sq6jN0#k(58=@oQ?yUY7js4gyOQ)n=)%m-D~ zgH}7)XD_IdxUl-BQ!&0+8{O%MvT;dSf`ZEH(hAs4go4~kp~l=o1zKZaaGf)>AKQm+$FPkR1#Lvqn^Jo*Zd`Rup>*-VaK7 z0={9xQOOrDbzaR`s7>YE=x81E)FE~6IGV?L_jz#jX96tHBdjKklHUh0OZVUGAF1Qt z$-2UJJX9p3%HhC|sB||Iyv}JJ(9iy&KJ3FAP6S@I!EeER_?Q-Ul20e!mTN3$ubFxz z!<6$0EY)!MJqDyd^(Y-K7o+rl_vthZySsOoU?*0VwetTbqD1fsfh6JK4ppP8KwH+c z!+=F(-Ydkb{cVW!5b^DWUp{rR>5r&txwdovvzLJ)R(s4{xQ4CAxqtDA#mXlC*X1HG zW#geqFZWJsM6-D9AE>lEax$c^;%&phgFyRDw52YPVx7zPh>vIImEYQpN4eoD;(q)> zG7kD8;RFa>-02g_bIW)HB27jW6F8?Z>K~|LUdnm#PVkjoXa}0TlqHHS*u8s%o9dSU zbzk$|3{ff-(aYX&M&CoR;wSLsR=RghRU@=Cntf5Ao7ht71rUEinjn-V+;oCqu0INV zfq$NCwXk3=p%xx(j%yl@`Vy-^Z?=m0{fV>NY?6;{(S?6RL%gZRgI(pgIS}Chon|&O zETc|AlI^x6FuBJA@kLffuR|Tw@_}jNsD|T@s(}`|Kw~95 z<5PL1y2e}&;0Jr!NRxq+zueIldUeAwIrSD}C6JHf{iiycb-lvAOWxS;*1t7-bI$9Y zo8#x!_fYWtc~k#+QX9lN0H+^*p5|x8z?l+omsOFU42#)R)UXEU15veAjO%9oE<}{8 z8>>g=`^yW+;sdIHl-Jy{GZ`fp+G3<*k#M2qmYb>UN-!K89-CVhe1o%#wELv6n_$iC z5N}m5T{w&&{@WAztsZsD^zRr?XV0Y{#Dsq~V_fJeIpHG5Z;>X5tegQ`)_84NQEGma9PK`6OIv>;aDn6^kc((x=x>AbKHKL`r6@lR95>hA z)y0`CQcHRbMCh1}fDYy~SSKGK=skVvK#>j zL6&kO7d$wOkXy@PalbnX&U98O0@C7WqK*2>#vc-@4yp|)piv9$54auJ!sVGG9*|Zmst~(vt+4|CcU*JWv-ld#=^9r_+#R*q$$kN{JlHp`$9Oe_mxn(%F zq<_9Zg4F)ZX5B|dpBF~L18XLmyOSOOEvSu(Y+|+N6uwi@{0pqAU22!XG`RtUitY5p z^A-!GE)q$WmxdbjvVJLGNHf39yCz9vx!Kp~j6FS4G) zBr-1V?$>4QvG9el{3vUXy=(xEwh^kmzjT0}IswAQg&mL;l;pI3X~sXO5O|_~`7?-m z{L!|-75KL#3a{y}a(xt7_W{bkrcO{oXaB`ir%**4WO~v7&$Fe;FIx|q9uh+${R>*v zC-YEJ)kntk^)1$|V-LbJwxDQz7>a?G8%qOzlv(7ik`6?~QsnI~gd~k zSPT4Aeh_}LQAppR4LygK;7JYR^dKmf8HXZzV@xxZr$G^c-u9rb`WkkU^XajtJ842e z$!;k;*+;Evw)$PpKR~SR>gbfP`)!Miq%zUyYIplf~zezO~X^Rp(~%enw}Lh-N8eZ zi@%oSk9-p;51mv#k4lIxa=Jb*x1(T)nR`Nu5L?Qx`{guA)d4&mPfP^nX( zzf?!BbGG`wc~u9ia;nPLT~DbHy4n-a0hvy6HBNX)_Rn*BN-VyfYp%v_ACW zrTD(}PhYQN!^l$dm>FE#IQY)ldm9?P@-Jg~CoyIU6pMN1 zinL{iYF%6RZbDYrL0IlA^p=|Ohg0EKQ0aiDhf04Cb%j9>gYAqjYO97JgMZQI$1g2U z`cosJ_ZyG<#A3)ZnIIw(s;>k8TedbQWJ@@u_f?saakIAn+=4@y{^t@LeY2>Pzq{J` z!Umdp#XLwE6*(8@y#2p@xw==ak7Ds5&L5kyxikQ6ibs?IV)~sIi3Mc*DN0uvA=x>m zPYXLcm=sHo;tGy?|JgvK(w#gzlu7XySM~I)Vfn)c$CqFuBpk|jtzr&ixjnQ>H1*zg z+_VK7LIkMrS!u<5!rwXPG!ghX;rj!eIrq=ZJd)q`66Die~VOvg@v%?~*j+P4}G zuhXK0{D_SYcFO|RSuvNnD8P|VT20R#wwiAmGj2hRm#XN}6VWSSVpc3SF7*o(NiiIDsL9b^~$e8Tafgn%erP^`= z%u2Q>J?{Ge;3<@F#|>Mk)cN$ewcbbFs1B|G9a>5@>Cp*eXVq#cd2ezA`>zNwzA&i4x-d=M zlUvKB8;DwFS(7m}pe`+{E-Ec6yQd1Hwpr1Kq)IgPJaXP+Y$0KOK-sk&S%4 zS^83|-PUrIc3bus8>b}g0QJ~r4TGUw6-n2pwaR;}le|O57^E)(?Sz)Wvpvtpm&`s- zUPm5g;pmoJUoxu?2K)`IFl?WGWE$?epItiyL_HDJ&?h3e*1{(=n{Yib()GTpvFj=z zv|{f~q)apZe~C|UK~hd+ttnICnFU~5_hLcL$3GZx{?eI8nC7-kDd#m;K7=|i`7hv^ zUK=13?~gLNY0jKtfw0~vJ_zd_ek;nzbPqH*)8&Rf!NfH@#T4}AAD`904>vO+Qcgr@ zhUtfgpe&t$P(SCV4RZ@VLmjSjh%9nXOB^D_%T@b_6z`t3pyBDSoOs)eg8$aV@VevWRxI43Lwag%>_y5Gog#Pk8z?%&(22K27^k8qq%q=Mw$ zJJqlW9qqr-`K)j(O-kXrVZVDt3Kb~&;X#<4Yy|29|3}>~l3+A+H2+I=o3);8NAB_g zSfRAY@u|$;zt^C*z)$nWzXS>WLcbfGX>LWBX&O~ktjF&LD)P62Ds^5&Qv`yel&Lmn zA34TX>k$Vo-~MyT@IWCAhYbwfCbwvau1K(zRD(Z*V|+)iDgMrri{|l`(Cn7>Q0f@L z>yxbjN-lOVR%}aZFn}Pfb7MR2jI5DWigHb7ejS%ew=T^KBc28;q1PI-&cCVi*;m%4 z4BA(L_i6Fo!@jpl+1k6#=ssxf%7Irr9?`b$0DPQ+WD)FEP-nrZ{%L)w1`e#lpPh{2{RtZVgw-FH_Z4q zl0D3vCebJ`jMb%xi)8$5r)4d0BpAkUmJHRZK?y}Sr2!6(YPW&Z3RIF^bBbL8V(HRs zds0}wV{~!v2Z*oT?JJ1>;`1Apu3gL2Q?9C1WDpHw@2Q9_XtVrI1V^WgJo~{A?R$uo zwZ!0IF1tksx&uR&nvb3`UBRe`{Q(|npjSac9vW}%j+sUi7Vi+BU-_9`V4hVTq%Z94 z*Vsq}7US}$UY<1Uv!7Rix6vYZ#uujS-dkC|9J=Q5n|{2QJrxgheq>gHgI4fkd)h=;HJqj#0PY+?)bCr7rlQ-}T zb-eUWua$cNQEc#_r8^_5CH`~Ro&=pV@gNMO%ayej@jt>i>NU9gsky9q8eM4R(mrIT zNEtj0TMa#{`DZzr8t6yf&=I;PjwbC*SqDBejG_@=Qy(+hQUFE2PATurSmvuf_amEc z_b`|JiX41PRju`TXM5T$8WN)cy<4f%eyxpRr%U&DogTZ<>yJs6dIsg=9--qNEF5=D zb+$ZRwMLQyskHTh?L3_U(M*W?MbgWmjT1OTd5@{nbcgku_gaCD3p@}?FTI*v_MGwE zo+?FV(^D@lo?k5yXgs)mg3VswFk2hy_#E46oD||P?{eV_%9eYpuWZ`i$#hAOh9=zg zqkwOV>Nn4tX{`>G=YR3-MYX}RNH7ns6C(YpCyvjF!>h1hLa`=zUnCPj4G|V`2F2}S z2pl(t&Xg$qJ3b{41s45Z1*jJ?TNH&-xBk*<7nJnhTa9gb6_0) zF@#&{Z*RKIm~fY9WFe-~1m6+8#?5{(gFkHyQ~ul9f9tXxPQH0CDiI1MDy70@lUqfA z4Kr&uwQY4@O&3Y)R(N*+d(2(&VC#XtiI`TTwuR0G)-wYg-=)PMB8BufJD(wvKG~H2 zko0Xy$Jk$i{~q0KqU- z)~UV8DGNW>aoEbR_4tUE|L0e>z6Z+QgHg(oG5kt(knp|oB8luN5 zvDd1w>kmV~a|FrTmk>!2oDXZN510rt>K~tU)1R`go?44Ltqqo3;uc5j7m;l|+pP`V zN35Uf?`Bb}War=GIks}G`#4+{#$3W1kIEK<>=wyRTv<+Zr9E&!XN?Yc)6D&6lu+UE z=w~j3!J2u-qbr z%H>tcDOz?(o53H!oJJ?3xRWP1qlMqM*aH3|&c)o9d3`m0eyITg!&bj{nZMqca~C4` zY6TH8eKngXAg8sIX-e|KskF)Ns~`OuRypP+$6x!#rG#o7{EH6qwU zI1KB>wI{N9tHOdEKwtl8=Ykl2uo%ph_5Oqn6+0oq`%|AHc61tSbUTSNwqsEpt;QEY zx!i1{sFVHde~hi|&y6gM$kKmomJ4=tAU1823yp18|N#zI;nDA6P5=cV@r7vsDkXe`EGD|2Jko znD$54SHDO;ldoXT@@>%mYKBW{w(e*q8?Ty2c-v3)=*fn{nfsqk0IpXl;S`$@JF8VB z6J&z)Xfbb@XRSWWw9(yG zYm_R8kzxXSf?yLrPLHloI$Sn#-95ImHOb=w;+zUPgAf2GRB13Tgg5tM0}M+yu(jv< ztdu)n1A*~n2eX)?t;F zpnT_i|I-b9krRrvVwnA|I63gG?qX(;EaIHY)zEYEr+ZZ$jJ8^JgRzv&D00P4TySTBRZp20o&}aD;t~#Ww17SAR%yg2Ul8D@I zz5I(LuCiBPw(j9QDW9bA%W{%hx%n*SYMDkJM^jXX zxe4c}twhS3*|Y>QH39-gkm>SY{O{0lXB)pRb&KQto}ARHVL%>$#X}~P0zL_ zR52p~PUa3e8Esz~@&6S3vH!OPf5`tc!C&NmPw=Pt*Mh$rC+o9+lKZj!b+6Ra5@P|j z^F8^$OD7^loem^-Syosy{oTa_ysUpSpc2R;#un$)|DRoe_2Tj|t$?UO)F2(|-u0&i z7Q0nD_?$-B4u{jU3LgU|m9>ugs}FLYxal<}x-kHx=CckyeZuGPsEnz-&j_oA?GLhN zdV=u!@hxz+uIj}CX%QC#@jNws)*+N5NS7kln~k5hDMRCobZ8+zdFw&CsW)GIL zd*74`UMR<`R5#=Z0WzDNLVAXgCJ9%ASXj3PDa^;x-(CmZOSWR05bGfhW_6Ce<{i2_ z38}(K{&O#trPq?wKa;Ay2vcw$Zb%LYzBt72ry4$7T7VHcUlCm{%+|D+)C*1V@s;$T zDTG2x@qqZC`CBl5;Oh>e3HSL*kO{|V zzkTiKasT}vpz7kExzzs{RK5K^BbvwZlch=(f9py(*GqM7<7Z;m8)VOh^^ljSjI0GQ z1eVe!f<sbu<|Cpm(en+ls9OW%R}P;<2!`B0GLtOQ5-N z5@iTcmA}r4?U_E@AC@2`$#-|aH>BIu7pKNOB$1XSg+)r6#V zJh@P+c&c1f&`)inlDf+&5ltsoi~h98Vb|l{HPwA)s)_L5Lq*gVlveCEXCv$2%G8Ip ze$XE_`dPCXkG2r3JZ5CPf?2erSw0o@FF=)U>_34j)jvR$-G2pDvi}lPea?vfzYwNm z_9MMY(^f{W64Mtdh?MM#9b~2Z&NTS~pUw*bUP{F_KuB4AV|Mi!4QngN2fH=0-!>!T zFLISi7XN#^Y6sSSAO%`_`9r;W%%O$PNax=pmW(F<5wQfu*ci0pgR~Z|s%&{W{9of$ z4`Rv!QwP;>d_KwkYCnp&%WXK9dg&$Bv>yby&inqlZAvWguiB=Dyh+SRl11@a6T;Bs zgg~+P!oM6$+m*ONaa!&gq%Kx`olx5R>%A2rToUg7RN-I2D|GPR;Z@{+175w1u6VbJ z*|S@{@jXG|2yeb!Aq;0Ql(kaCY@EZC#)=Y1Li0R*D0CMCQnrZ7MsoDzSA3%mov3h) z(lCq=h@;+a(LBlTSf-~BeR#S|KB}i@tE^*Vi` zys2rASt>o5dzVJ(JgIMr_m$Z#dJRd`;m_Ma*M#NX;s_+LlZLC9%rTarog6^>pa9-$ z(rkYmn9-BN37?JdwbqC4hUm=CSK-4G%CVm`_yFC` zslkHPMyAmR4pr^GUhmEi%cbl|%=-hyC;8FqoT+KMy+_325uUFHilWE znu4Q57Bfag)vazXg35NqhxD)bY)lJa8d)pqM^vntcaR6eh65Um=9&N%n_u3@{b~+} zEgK>ft&E97K<=__IB+_{Ferm#>u&JR1&Ar?!K_ovyngU+gbNWn!!dyLrBsxQ}}liH3xu-g( z_re?U6CjZgD~{kVQMHf&nHmwbXsZ1NiyV;Tv{{1e`0TBWJ@d8Ue#Nh>AYc| z8Ct=&O{e&A6Yc4_xx(_;3hA!=la=P@U&&PhzQ&*Y8k$-9jWJz3>5R7!%LI}uvekp~ z`TKja#FoFC@{B4L$5wuDB#Gg1m8f)E!ln_x?VCals`j3Z?@UL2YnP_LbdyA5vOhL^KP|<0n@W3roVElI!$%L zJGZN%T%2rFX%~@lbY^530T~=GdyGD}Wv_A6hESfSJRx&ALo`2!SX!60k%T;r>6n7cgi&DR3Qi*UGay^v%?)t9m-K;>ByoT+tJ4w z9#^qm5V6=^B$E!P<0`!uHlNdtCyp;xWN$|kPw)#|u5~?ZBf2zmmbZ|jC};jj6&j`> z=v%M<_}*$B6O1n#W>L-ky4F=1ZO+7x?_c>rb}K%F1K!c+(R{gl;m#>AU#9z+Cr>FO z#2-n(Z?}`VB>G|X>`;Bo^{75W%E{H*X>SIrCU3@IJiCnLYdWTQq<&y^3s=^Su7Q^W ziKlNu5KDld#OYfj2&F={{y5T`Zjq~8rEin{C7%w;hq; z#)Zi8*5EJdt%xS&ULYd*s8DWGyBZ)UD|41f(#u-=z=(hIAPA?`g1YT zTb1z|YUo1sGhrjXKIkL~f>PF(1);7D8gZEGNpJ(-EaWR}KpNt>MKb&d0dps86+^Vb z#OM+Q`PHu&DuurJ)k2i+4!JQ9elRk)=^x0_NFYVJ|QHkPuN@1?%+-0jUS?1qs0LQYnF{0bGxWicLe6n(bxiwI}-tn}8Fa^3Ox9Qf6O zw%SZ5;|z2#v3=*Fn{FzleveLNgCw^!54Y0}L1 zbmp)kASG4sXGzgON^0S|z$tZcH8I19E7r`0qBE+YBEbq`vL>TQ$pIcZx~8F^b5mCU zVMd+9B}+%7ft%MQg0P54R>$=tQ#$`%#^5BvdrzF;Q-8&T)^6Bdc|bQLY`%PG@LPj* zGMTT649wQR&-ZR#*radUxt(v2eefQsaJQZ`MC|c}azPh4ZO*HxzMw z$duzez7ar44mTTb!aD&tK_2(4Vpl=!uca}fvpx*b5@CIJXdwb3XnGke1URg!vfk@S`KovL!cTJKdMrU{T}d>K z>j%83KYjTP&dDf5Dn^;jRo*W>8{_*#2;y0`Nf@uL!GJE-kHFopTXV_6=@L~SUnT+HfOP0kVVhFQn@&DH%_ z_!tmc)@qJPk9O!8F*f+bX8dJpI}Q$Zr1 zJ#CYBgc#3KIyP`<&>Li^pXZUDYz+{XG&$iq)q+3llpxU_6n68PnAj6WEd(*jBVHZq z5cxKEhe%r9v~y9zwZMWm)eN8gIA#`pCr-Q2;)BJ09A8amwaR$UPWMvJbi=33!F(C7 zT_ryYyHC&Ub>L(ZgljKrC5|ZZHEHlF5!LFpTZc83(QXC!_>gFDb>jnd=)>hQUlLj_ zKSQc#`kpR;xCL(l=H=I#WcKw5MLDmhTLv3z-8j{4fcr^UDeMGL#k2WbeHCaq&@E$%t2&vR~jb>ov?;ei2gN>8zb)G?C9gSe1 ze4790`lMTM2%LZq*2xK}TT&}jxM|!XlkBBY&_aWO>|I!i-FzKth_^t6K^&V}%21S= zfOcUhWU0QCdB)Avxjoh=`x}yJ^t=whzA-Xk)5g2N4cve9(c1_;e0MZm-FZR0c-}d*zPF)(Q8Q7HEniS^ z${DM*wn=ttH};$m{jew$~v-T{B4&w&a| z8}!R90Y-YL2*&AXN%m6;v!KTmGK=MPx}evD`zxEo&>qHUX4=!>s>sU%vZ&cUW9ogS ztd@O?ei4?A6iR|7-MzYTNmHRRQHWRlF-}IS6+h}uCh)Xtm)o%@2R|*PyXSm)Y{8za zW`!D|_g?#aG2E|w6utFkZi+Y{RylbMG^pTJyPe5-y6I5@q7*hEgoj@YlC3Yyk2yzq z_gqF{^yXvBR@HXbLIJ0@f09d~#JW6{!GJ06(JxkXZDD0W;U7ucR#CF^wE5rHIfea8 zdxtK>yEvd=mW6{aVwF{Ve~9R((-vZfw!yoCv5+a~ln%Y3zssan)0);h))LV0;r4wV z=8Y>q#*>Pd(0sQ3#cwZtlqgOXwLz2_z&#u`8og0T;?XQ;(W*oVlpJfZ=|XqM>;*0R zXUMq6NR5y)5_p!3%1rvm4BwN1pmw*X`1PW1Ww(FQ)rRBugnd5xWtqZxE4zQZ6l2wI z#b@qoZPZWLg6mU)v^wnRNS1l^QK+_12aQUoUJ57W+IG`3Q@hrNrM<$D$_c*#2Y1Ny zLOp|zdHh`YD&Y)MZts}o%eaY^J_lYk22!SmjI?DBR&LIHi|MCnoMDB#QmQ3+3XbBW z4*tW4^ub@JEQKqGN7@IG?eBjvo6LOtC3PgtqY@8Bk2Xwm-Y}st7A@T9k*3booGOL=R*Ex5n1OF5!5o;nO7_z99)Rh%Oc%2AqXu9N zgQIv_YNcJ_(g&G+b!_{xYjQ@^$nV(hxMDp2ZPCn@N8ub9s-WSF+;`+r$1RA>lh z(s#X*lT+R-EAbP|ARlgB6if0*r#4O!9zK#lP$9n66lj6cj4xA8G<@nxF)LjwOjJ}f z8{^utIF~FoZR^DIU4a&t=#tb^CX!5d`o~d=M-hB@P~(?^!0F0gyIZ=mVQJk0OnC?A z;hzHQ1gG{fbtpBCJTWglc#<_h5-%(5tk&QHah3Rtxd=CFi4MZLnhNn`&-JYv=d$<= zHRs6C&>uLjv>V&fO%wBEmNT;NtT>4FeX4WRMZsyOv0)#vFD z1dFy@2PtNmkc#TGpgqd*Z34Ng+;LB0EJIM7iN@+`Q2ZWI)~}hQYktKE+9Jwmaih84 zfIHP4Q8(l|B2Iio?Du&U23lYo#7-+^;bUkrdM63F&J=Wpx|Hhm;i)S>#Y*hQuF{kH zr$J6@++-b^=r1@zbgrTb6H0o(F=`8B1r4uK#<&4Sn*D-lMV<6xG=sRi;_jvDv&EOQ zR-!O}%JA+-*7uvuZ|DItxR|IrpH3<1m>)X1!R5UQn(CNEM(~vNvI7fZUDZJQX$`#T zqTDk@3svj+cO2&$d#G7`Rm#aqdpz!Vh5GtYG_S7okk^8T6Oge1*Xl&D+kw#0;!=?S zOrSnpE>>tOuVzO9nTn(TBhy22As=}V~4JTF20#d((@rZUc%TncjV`Ew>w-L;|l zVV+UHtBaU$RfhVI{M0d(^ z31>MuK7v=0@boz7Fqn>d#+4?{ca|`Xy)dNVIyqhT*2e6M4{p@Oaagrr+BXBxu;OSb zE3Im2*aJXK`~|7(Y;XM1%{LZu1mB4qwcCGk-aF>)ED*tDH(S72PKu36i{nnztM|f% zsThv8ChQQhnyfb8SDC+!bYVDCe9=C;;ttC0QB~FvTvxX{G2|YoItj6nwa8TJ4$n7K ze+$j}(ZQHpS&-GLa6Q6&li4htE%&;F#Fo>LoiMs*?4)F@F9W|p{#S^QD&WiMgLfUF z>4(ed^WizK+z-5R_)$F``bX#PR}cD547ssyrv(AQU}xA0mqmsxdR-jj!6e~*;dz-@ z!3};F8vz)jXVQeFT9!hUa*-nR%{2{ntX6B}NZ)A23jMgiu%*WEoZqMeTt{>PF_xlt zB2e4Xxm8X1iW3fOz9du|`na)i`-6G!qQr^Vo+IPFN#%p22&bU zKd;w+3Z7~q1hlUnQtM6cMZmw;?#xhQn6lKshE{Qb@9}&*Yh{>KYui;KuzNlhIv6E>dJk!B*gj7Qxfrugb*Xl z`zLo_(UuqDh4)UA*L=AOCIirKnRJ3Ho+QjxOXvixRp5C5fo+|y9>1!{&TqRd`&71# zs!nZIcGeZ$pr{wk>QC_%i%z+n`gcIxgWj8{TV2LEUQRrgnMUe-{BxBkdb5~eU3Yk< z@0r>a){UGFRxq1AoZ9n-X~_?p2jO_i2}2lxQI4$D3W5#A`7@=kf?n(JT(^U{u3Y%x z8Iq?neXbUWKFDF0Ze{1#tTASxFx@2Whr4NO-W((4i9D$fnxYM=3+c5+baV5;8{k)& zF5z44^VKLbFMl8aM{IZG25z+(@A5|1_jx%iZejvg^zyBKrs6z;*}^P_YKBuox8H&@OnRw|$^uzkYY(o76yqi6>+T_71@5c1&NANZF`r2T^-E z(z&FBEe8LwXux+4e+%HwDvFeExD}GO&(ipK!R?dC0#Em}*?RD4XDR05G9F!?iKy#E znMtv)1%JVHKOD0CUdviAj(#48cMIFZev}|XJU7$hqQ6dLb&sPnE!q`IgltO*ej#GT zfg6Wf2+v3BKmD3iZ|hC*$TY15-DN9E;+3WNmp$6t=|A3H`>5Vm&yFk`)M4Abq0);L z4RG05$niAwTFAz-yT(E}eSH5?L0l#li~-oZ#KE^S8*Xu9+e8v0%(|u0Tz{Rk`Q>{r zW!U=lg1q9CMafN-o zDeLWrI5QU}`bk?zJ)8Wo38EiYE!JS!S&}#ZpzJO4Gfk z8tK7|9@RRJ>nmh2+gLPb#(OM7(|c}lKEq`b1&=caQ24qT)$C`{ zV!-Jw>3e*&Om9k-Sxa1YxM$toASEhF6a50Ch8)ZCH2Z-_z=J&uEzp|0u*(xTx=7A; zP-PWJl<~M8t=9^=L;hW`7dlTj+ior78zb0^M!xI5j=rlrxhR!J*lk(e_AM-;{KB_r zowXAl#egJ%$BL7Gbt63GWMLm-F%cED<*@1h71*9<7kWB2=n=V$DIZvg)c&+ZJ+vvG{hh^X2S%A)w?nIf!xFuytjiX2i^j zzYbvYOxrv&e(2yD7~y*qZ*gT*ed7MD5C^(d9sP z!bLvIV|6Mx`$9|{kFhn*szn~uBDtk4r2X8v1+=k;8sYoD+e@E!_mpbOIrT4+c#uyK zt1dXVHHpeB(r$!%7lMjOn1&zHYR?n2(lr64&WrAZIaN00NMU;dIJRG=6?)Lwa6n>% zWCg%z)uJjV9<4U_RLmoi7m#}vEToroa_#ptA?}HoxUC;5Lab=?5szY~ir`n~vM3X; zDgc|OlbE^$Rb46)SD7rGM~ubFyY1KWo|d-V9p{hcuJrWrql=OuN3pzLhJIY%bq=Dw zZGo*YIwra*N~WsmSe){|JXuRXtf(V(+r^vu{M@*6E76mAXSPT7)U=5B;`efPen)JH z3+|GDo}M|bDlR%TY%FIye(b!`7!eO_Mo@kMpJ8G7T|539;?#{X-8Rh?(*1^>6II)?JB3RFW=*s; zShe6fLgof-wzr%uRpU~Ti2G8-qu;tlO5ExRj62qF=_SnegnIdF`|5S%3})2bl0~)l zO6$W6C&hOg*}6hC$5m7r=6J1$KNy;^S)f@62z-nEDPgn~h<(P!-c=60InE2dFDpmi zEw^#f_jD%Z(-XToTuJ z*}hK=z|UIJW(FPW?Me9|0qu`Ig_cSu~LDP8?PckB*eE~*X!2~at| z92f%l!#%A{Xc({wLRX!p_tz*p^w3ShouOrc;h9ZH9g@YbLQt;gC=qK69Ur5=m`{FG zTBJ@2a|R~)-Za8>b~g+`5Mgwb#Q1R2u;Kar`$<~&ho*eJEQ-jMB9=!}d8jo0AhfiU zJNtc@z^=4x;t-U13#4`eTS%IaRY)) z)>2J>(rd^3LJ7z{tCdf254lij3W36Y5d&QKLE}c?+WUl!FMo#Qpxy(IZ2EM&>_B8r2`0?mQZJ-ZV&y&+gvJ64d z{YR8UL<4E3LIGEt%?d1_wR5g@8v198cBpgB$wJWh$H$pg!T^q|472RqLR0t~M>Yq{#sy51C3K1fFwdifLqmf++O*&@{)egMEXH4Qzi03k zi>fEnPl_-|qCHa<^!;T)Yz?`Y{cBD+4!F>hjE=pYOMQG+li^QLp~!1^0NW|w?)=5n z$$2F>*?_BJxSo#J<%n9?)N9=VD;Jhd@KbgBV8v1akucY2$JtvEX6r$LxhM^P{A0u< z^4~(vc|i$h_E7vdk>am{-Yj>?i5k_zsI!@`?9{PwffSqeftNRn3MV3w;3bx2#o|Nu zH$`&6i9Sr4>L{ z0Nc7!66)h<(ovzYG?|Yr#m{k=ew3Axz`q19(nH?`kUwIvnn#E%=W5oUx$6R~^mZs# z*$7}offf`q8Xcm`(v*}IH%dXd?1EO+Jio7k1B=~h6l+t=i1`{QGe(c< z8ET3_Z7-1V=G}|AeR-g+9?m0M**Dab3$!GoeqSii-N0jruUpJ|PK{*PINHTp1m`SD zl#@AAh;|RB{?Y_1a;;{jEM=mh241=j?lcY~@9-rFLZ<_77AT78Rx9;pA7q#uci`NR z#~bapA>2t{5oQG=weM*@o)!OoZxR0+o04u+OFPaw?{5&D+|K>^vs2hXqv#$FI-gD8 z=Jq!6-DXSyt~7X?h~a5MudZ|jOjqWNTazzf!4r4I6h#M=?e2@7tXS9LLDFb=mt~oJ z3P&*_yO7d))i-NMyQCAGVY7u^oOsC)j7k_JDa~nk3O`x@iIKz_LMudC5b_nDn4b8D z;Mv|^Wa)-9akIN~@4#${+}gH&n$A~z6Xr0pEjnpG$kMEf5?D+CjyBpg?A$jk4|8pU zAi|mEf`%j~0e0#Addb|<0U|_v)@^cUnG)$m-dt?!0}J+)VZ>jH?5IW(px(moM0ZvT ztdW+P8>q5BW;4|!YY=4*5=2vTM)ajpOLV2pyqd^PAKI;VG>`xSn8~)Z+ib`wh{=ZI z#jnx4xfZ9!Kge7m7edBWA3RA^G$2m9C10B2?yQ5KLNx}*LLcm)s2A7Jn%+alDWo?@u%e)*=Qi`$M+7~{crhO!oPp}@{89K zhU#5Yx#J@GQYepzmb|pw#QBnsp&+!VV0TK9;qo*gnj%%Q%OgpMtXP$q9?~sp!{#2{ zWdpUvz%C^t@e+z_+M7TkWT7@v*SnCOF#0FncT3)AUKjKusoeh75jyMFnamb**v!Ok z*xQ)yk5DKiD4)KKGS-bah=gX)63nOu?@qBmF5x;ez5G#5I|gif*L>BbGA&!<`m#j7 zdWCkTlERgZ;UOa?7Us6t$xhKeA(8szp_e6e$QusF)Z00yg!BVb%iShM6^__s2&Uh- z`f3KRV({TT_?ulC!*20a;}oIr`!u5zex_p?Sj+yYLEFeZYSyKKE*>Vz_)mzh@c$^{ zi=+6jh;OZ;PaX@~dD@Vet+Fmk{8%_X5BkBG5d>Ur>CEPMTqDGSg7hNV(^b5*5m;K! z4_WZVM)s2RyF46>>*ux{9vERAZUo9(y)WNeJVn3j@=};I;z`B_1g_1lg1TpbY`QIK zWF>}mX8~-vC99}FAZl^)MkIg{M~q6*S7w!k!ro@$;3>8pML*IksQ4}7I=cfr=h2P7 z{b737llAmR7HldB=d1eTamNV+tEK(7 zOt$hzE6gw`fW7Z5%2!1FTVY;y?|lVkNhLr>I}z1i5gsum-r0~Fb<;8nWB|fEVO_{S z6eA>YsFKi)ld6|AP~XzRtq8%V5rVs(zYlA)cwq|v@RGym(-}d(ROiXkIdh71;i0uD z*UMv?Bmq@-u3cZc=t#5E)>r-bAsuK)My+ZPE*El4W`zx%H$<&ELsKcbsl@mtHAzrh z(d7HbrhG|@MZ^chscW1oI>CPA^$91z^yz?a7J#8&h&(QW$x`zd!iq3AA5Q3~Vqd*Q%3sLDM_9cM zd<@GDR#i?g^+tC}yc7_h_Vn|XPGNEThS@W;5U3pH4Ig}P!9u2VR4KRgD?9SD>e3#y zy4N`N_+1|&UsGC!)FBSFO}Q9D$U!LFO|b73J(oG}oZ4|n4xL1LUj{rLyD{pE)Ro-K zH>nb!(0(s$`R3Wrm{@m=%(Ctn3s&uz>p3XWwkxAy(NmEn?=V57f(l=Ue@*gA{u9Z| zK>PoJMJJwd!7MEIdd!&FP4}_69EhLUHV!%Tz zstl0R{O?y#Z_!P|~2&AoP&6M>(aBaIhGl%w()MJjO57eIkrb|uD9oVPpu$A@}Mf7`8kV7wMT zUr_4pTSapm9*Ur?30_wNwaTJQ|>)ek@6*LL|TT{y}aysla|_fa}-wxw9)(tA_^nL&{88(%8P=L=OJ z4Fe$gcfKAECZn~hU$Q6jh+b1DTkpvFOi`CyBE~kk7?}S?mwAg$Vs1COueOERVv0U6 zbP(5YmDKbJHiYxEBgNr(ECckOV0MPoZk4p@WxFeGa$?uoPZAN9Z1lzc&f-L9N%DYf zRslzr0H+-&fkH;HL)eA!{|Vo1cu>;{RP%--W)hPPVWd?jG~)Oy0$$?D70X2{-1JGl zG~s^R?tC!ZXeZih{t7Mbi}!q|r9*pYVdzYsK~XEI*%(@PLsd$EnB_5zB`8%=$e^A6 zkp%JC04veOLK#7T3RUF#u!v>*6-3Fay+n}`QyF+BooCNj{NczFH{w1Z>>_oLw;GZ3 zr~_X|E|#WJGG2Zwg^l~M_U&}Cci=lU*A-zNOHHoKzUxE-36pg9fUJ*F2u^!r&_wEg~rP7 zt`9<=d3_1ur$``bShd+E{pvW={lg({3>~WMgL5gRfrm6a)?Sqzy&iuqhf94%hiZS!gn@-M*#``{8cj42rX2nZ34?tsdh+ zwpmjxZYg1Dy2fgb#*414*F-+a!Hlgx=W=6)-~BSvKrQyngV(!Y9!?2?#({j5`i}>` z8u<5YO&wVhqnppzLrPjnY=WIRm$MnQhe&xSvSaDy+D?{goIY9`Th7}Tx_Dij`Zd;Y zuobeixjD+lt85F^Z4&pk?US2WKM2%i?e7Ei+v_9NsV5r%4IBkwQrL8v;kwXReF=TB zfim}fPQ&B7f#4DHNuBn$GZP33TVPQFmh1|~fI5h}@DygX3uarhGB7IMT z7H0^lU$^;Xj+cp0P#DhdukP=DFCjl2)S%w#9pAxd>@Z@TZ7lH~FZn5-D2PUx< zR!TGxoe66@)e&(v2Y=6&cMlWas*;pdK?Zq)er$i`kaKR%Ayf%CJ_hRsI8Gn`$An2~ zyQ5Pe3$Up;?i)Ku8kT_^OUR;5hE=Z!;|3LydBsIzR=R+Xz{}vl$TWLFUn&{B2BKAY zKqOA=Wzy@#Em3bAkHMIi{FCJ>EGACFm-`KKyqOlpaIx2U`FQ-`NFIf}TQ~X_17aVo zN+e|uoc5sURf7@0?WSuS&g6s^)W|bCY|%T`jQUIAC({MAs?s>1kk&ePy$j<^>JX^r zCyr6YTyB(ZyUN^cWd8VXnv0ihNE!jo%TwaSMHu99I%aEU(jeuS;kW)_lroON}O64SSwJMZcGNW zA3Ri8tKOYxy=Xr!8XhoHU6~^g)NLJA*BoD(`;;ybHMp~*m z^E#Xzlf1N`K{obRh)-XlYZ-tUTH40-+Up}PnFmyA;rll|LZvH-I!~4b%ipr&fT*QP z$5{@PfhROpDMr_v{8_6x;_$a=hOuDLLAPXKixrK84jv6ih6@tsw21@nB1jYGlOVtb z6EI(ufP`3F+ZKab=@U_GEYmosYca0F-9X#+{xCtI5g_jWAL%WFb4E(HK>DTH7TUYk zy})$?`PGEi&YQhsj4|gjCN~1_mTqQc=F+C%7aQ{?28CzD%J$DRsSTVRDoRnCal}$v z7mn2Ecc%`m9`11zVzR4fgTxi7(yg1ru-P1*oE$*l zT$`rP>2-=C3dZxW1c~GR%_#)NYzLop$CG!Of$DN4o*HxcYK4 zN%YG!VC!J+WpB;|LXtOqMU$naEU4Yg(aXl4L6OmQfFDcz`XO%WK29Gu%BPQ9gQ@Ks zA*~v6eCPVnYUp2EJyLv<@TKn4i?zP!F$wPwN2fR6>VuQ)sE`@cb+qm3%{;FbcYu6vEvsz!ivr0RdaI zs2(3#Y+fsvjJ=Gp&S?jR=o@WE)GhE0sMkTI`rj|_*1)PivLR@KyWN{(%u+_C94&e0 zZWNvOd-&qMl4#uNwc_`C@JoDRdZeM z`dc)AI%~GD^N7clvwV=-QOQUx0xY$nYM$hlhkp=ge5XapomWmMF7B|Nd}Egi$+`x! zl^av62H+roGJz3eX}yJtYK{1`OWeaR>`^>GO(Wh6SGz`*(>tLEm&93hZLq%LneQ)> zDP*aLnBpfzn0aQx6W%pu7G3sjP2jt&lbE8 zCt%R;bG}<%_kXhJJyl_kd^0|X*4kk}=Uuy4+QuraeRUgX=AP`iz+%(0==V0Fq8#c* z>hbC91(Dxl1k}}P+~KjAKe#tGtn6`Af+*%6i9|5W2!U^j4^Nsvd8_-JQL@6A4nPi*Ja%YRI-0 zQ?&^p(5a0_G*|`Q%^mSA2<$pPdTSgMu260bb9)$j}sQ4R4AS!6nLM-jCYpXvP}Bb8gXAa$O7*O&ya*HmjI5Y z>Ve1m{Uj1+Q<;+2Th?vMML8-}c5K{kcL3}q?`AwqJ;Nx09YP7k6`T#P^H!HO5HFmk zJ8Zez`P;Js07qPow$)MdKUzl1-}JSYubeesX}e(|uurlcd+;@kHs82LI4e zm0Gaq%}pnTkpd|D7Af#fW#2_Hx=IoC2Nr(uVq&LB->po*`&3zD8Rc+w*}K(>w_V6^ zo0Y<54rmAnm%JZ2NhapszyBatTnkKq5GnC5W8-Y(eoStoe%p=Ycjy|h_9DugX7xDs zXtT**Gxb1W#I`+HbfP-Wg-cKhcun~IDd5+(r!9rf49e&61ZcPYOoV~N8y&1NXGZK5xL@3H`(TcR*r0cpCwjHLcnfFx1j%nb` zr7v@&1*FG!y)h_z;)ylqo#h_~-FGswUK)x#pV)CR#z~WS#^Q%)+YxXaTt0EBKh{DW zB0Z#ch*HI%|7~;Y-A5V?ugZ+Uu4`6fQ<^H>H}JCOika4dh52`05spJDNLoz(h33TmD-%;gF=qF}q=uEr4XbAjJe zNTl~$cbiztp4(YttZ6}xdBH#qKq*8kkbi-`yAc4itpImrF$C=Mp!F9Mj{&TrgVd;{ zNpgu2WLZ*~x(9EQR-F`zx|Nj5Mx|s6@h&!;mJyI&GsE*qXDh zW8c+@NNWx`Mr`M-B`xBhUKjP~txYvdS>8bC-E*c0kVs!E`i$HC*fHFh%)tDLcgC(a zSTM?8&zz6P_VEHiLGj?zo0+fpyKZF)?w@Wf6Rcy4@sxI}gt)CY=>ul1;#rFlR_mO!!K7HaF2s1@7_#ECLkzJI>q0uz6fRu#0hfq6U)L~I-u`AO1l1t zO_bbKQN%g6HHKGn3Piun5RHA-E&EL=JLxqEQ`bBrh7qXek1S!}_gd{hJK^(tV zVr7xETa62t#U5SvrUq;FEvx?rU3K6q*3#_SzE>-|7BkQYT{jEFXH1WhIM_98_Y{6L z)x=%Hbph+|0Iprbk0Fr2)y{_w`sC;ry!lw^8ev*en%FlcC^5TrQX|kh^!G9l1U(oTF9%2-) ziE+Nn>DDm4sK~TpQvr zSF)j68D5DUo@kRSY~)k|Bv$r|RXIfY-V~;~%l76ZOLdW&Uy7Hk(>U#w{3gqmi+mPh zD2l+*t_A35X3>UpCcV)){%y%hMAjgOa2WWOps1QdJNpcuocuo4Ci1sw*yjuwFV$e2 zpUi=m_+cK3*TdsV)b~DqdH~b!4km(rXW$%FR0Hc{St&v6;ZzurKC&p5f22%L;e!5u zQA;IJ{!cS0lwUu-_{T)b*S{oEj$7v>rZ0Y1OJSzE^!;Zdh3;;Uct~$BTKK4M2I;YG z66>j_=h4Cm5xdg2)pc)|EAj>3z<LvGLdgaVVgW(Z8 zoYy5MR-`+YsKR7)um5(18PI~3wmMJ}i-z+e>uTP|iYz)A0x_YWnd`~d!c%>!5B-w9 z%UER#utM_WulthW3bc7A`G|TDmtpG^J-*n}hgP>{;AHq(j-!?{fXa+>A~SaTTns;S za^|a@wVMUL*qW#vw>|`|mMc1xqG&kh;J7_6ttJp!=&I1G`$!>An4`e$o<7K^XcJbo zhNNvmT7_8Rq-RG{;4VnaYczdHR!0;Sp^}zfIFy%uC^dT;cg~ux_|DbfW?(Y=(WQ9} zBC3gVI899q*wTzicd2D~lBq!(7`(mMb1773KCkPgxA|d<$(EKbz6@8(_n=J(Bt4?n zAX(cwaHxnEhNPJ5Zq;rrvrS zO%o6r-c;Usi&I!_?z!?@eat#PEa0QX6|by1?tjmGFx)O*PlsNWo`MYLDhW7iO#;K6 z%X)x%*h7T8dCp3Oq$8Ip;w%ZYU}saWmmhB`^fM)>%mamqC~I=!i5+1wfn;SDA#CT7hI^sR)`&#u)6e#As{W9?y=QTFPp5 z!$eSsapZHnnyjoo3PG_KHfgMp2cY5&(Ywg=7Pf}gM`)H2)J-o*3s*@dIvJWHP9 zCxyzrKI$7FLfwu~*kglAU*0a5vd}AcSKjKw3iLSYb5OvARZB9T&THEJ=B`%y?rqv1 z$%`G0Pl%mGmK9R5G{-F=0vV8}neT^5gw;kMjq>LDCM7yHTuN1;Ecr!T?bo)!ci0B> zT3rcxo0ilgWjg<=LwfZOIwZuu(IG{$;V(PdAJWg&r(q0=x`r8tzbn^S)UEi(IgxG010Dz&OqdGXX1`@Bx*pnMi5m}SZ9V<3=6$%$(V$vmHEBXfh za8+FfHK+R_($ALMXb@w#F8E4{#T4ii?O1>v^eH_JQ4UVDKow=?Uxbh zKm@YV7H-inxjcohcE;!lZgUVQiR04BNtnMiwNGA==u_3h1lBnCD!G+?fiTW{^kQ-v zGje$(#w)VJeG6}r1&k7u5E1N1V!^d~zvUOW(|AKADxD@O!y|b>V(+(WVvk=R1p8EB zQX?zo)viTn_^`==eBS)nO}{^HHU_J9J}?L?Ok78wHOm3>NvX|b6yH_bHYto7AWSA0 z??}TBQ!;+K(MotA_njQ>JIIP$Euk|YCU*Y5a^4SMqhA`Wnlqdjeo;cOlc}{@yS31y zF$Uv#yr5NYUhk#r7N>jd(0nYG<(#kH`eMfR(C_)iB|7C|vFh&W3+#<4{PR7kEV=c> z{6`(&+VBy4IN`hS%Q!9xBcq`!#JC}C|AkWKWX`wz-nnKYC92a`Mu7+=)NzL$uWGL= z%>>y4%oboJM@nIMz#a&xgn3`a>Wzc?$?xEYG!aD;ChotLh)F3FssI%CsM7g* zgofrIbeqmx+?IG&WbfykdTNV1@AoLl|4Bad=5OUgZvQMF(xtkGy%x_3a)YWq;J^)b z*YbaH!FIgr1H&GF1S5=k3lxj*Z7x!ws-RjdFGG|o1uUZgqUj-sGDHW~KX~Vq9N?j; zpq5LY2LBeis%pkjbp01Xhw#8520X0~_b&hlY>-_N&{nW0?IhJDP2uL@av zW!9Z2k&bnIY#g2Gp&83u-6H)i?v$aX4ziV%aOH91X`yj|YHi%_Y-ac8nLn4s#6H~b ze5}VzYV+CC5-<6L?a5eJ$aY!3JF& zCxC(#oV4FEOxVu$#-YLp)K@y?AQ=EF9!yit@H{JvT-sbx>9j{dTN7>gj8+S;$NB6J z%p!BGmN-2pL>X>%rB~Imq-(A(pP@YNT8##EEn=0umX8S^Y5D4u%pH+-bPMfz-K1SXBO zD5Ut3j%2OP9KZ4I?U2~C00h>{RQpPJb>vGvH%FhefCi-SBWqSCSuCKvVp-EMU-=m|c58?Q>TWGV#FuR3YeeS-@nYNGZ53C8Ifj zWyu?OG{b}NKiBezcQxxB9W{8;VBoEO|DZXNBS<|~_5}ll-v1TF8fU7o_?TjXGlimZ zrzP{!tD!Bb)u;ovZ}J*@v4PxS+WL|m@Q>_vsu5Z@6n*!Nf2dwEQFaz^(pV{BG~W4k z-adSOw5zd6h#Zm@3JgDX3PxnF*4J*rVe`uj9l4DvK6*4_8)}ssS&*N)#Qk(~Dwm5A zM`^$;VJ9fHNUaE>NojqvQ*Q6PxM+a7<;(sf^l?)lTDcL z+{d6qoTz17bgnO%X$ z9{bt=PYoP!Rbnz6dQa-Lb zih=ZwIpu}##f~{h6MgUHzCWroxYsVkl3r+5+03bHK(rudJ=;`D#S~7YB8j~UL}(t` z@V6;$Bj>FKl2}g;q1^b-Ebj@j2qlG9EiP2uK1pe>Ak>#wAUx8$=w7QT)+Gy_zdIj8R2+|%B# z!!WzU$GUL=0SoTjAV9OKsW(JP9gRD>*jAXqxh{n*bj5$2z zbT1*?6NoL5^T`Ba@dO6j#CKssdY;n_&ixob&O;J~2b=5_$H4Bwb48(+N9vPiWoUXp zZQxOi3X!Hi&Au}0F|)-n^CJ7Q-s!O1EQq+|F_4=hD5u#eg=XD>EI?-R?r>~OBm28L zO@Gou0Z#2D8+~(i3Jq~7k1mH)7o!%<_(O*0&mTom(}h^gPRmSfGnT*YQ3J@E~g_{b<_P0Ycb`P$w>Jy7_y>H`^nuAG z60NA#rWcBTH4zT+95PZY?5EnDi*wC zyqaJ0`)$I(H^zvVo|z~oXI2P;;)M5upI#X#KfNf%S3>tfo&cv{s>i{2*_s~%X-71 z2+P#+lGfUw_P+;wqF^p_A4||zx_{$-X;2|~fLQpPxU_|sZoOjDLl_SIm=>*n7K{P| z)V>Meo&@xxZg_|WSgP-RWuiFBb4sZ(L!SVvk`V$eA!bUU)*3Np=ygIWv5W<{MQ7QEMK2$3dAnM83s(5)vgkj!in5KQ)SD}zoL;3D9E30oq?fuoDi`94s=C^?ZkTsS$4^v!65X$`l2}Y~F z2Y(DWalFT`P(YJ~0Z;{{BVHw=Ekgljf*qd+0;`u-7WcbGMSNrc*3{Hd%gk1tT6{Dy zNoTIaLy$j#mqH5(uyBJHQ^?)ximzPNd_cq$hwU0~CV&d8FRl6gf700_RDqU48HhU~ z!(U>iMRJpYD@GLVAXS=}c3je7aT)zH%HzIBw`}E&c<_`V^uw^BkaUH@bJa)^&S7%l zmm@;mXw4}8UAXI0M%Ve3lJj3O@Mm+0DV!E`w8#{M@Gl<&>$!r5!H~DiN4NKTL3r{m zR+Lb|;`g#MGD*-N0f=+Ns9ZGWEDELs@ReI~uj^TQp`@BxjRWmYxIpf{;W!bv8knG$ zA^cAA(2Sh#(@w665;`xvshaT55I*Z^vVU9@oEz(L0~xcO-XN~@w6tEIKFiuZ{-(7; zrrZE1C8?Fah2aW%M;t~k98b+d&325`HCV}1p6RL@$kZ>P%BFSa`WfyNa)J@E%{S1Z zVp}c|zEZd|$?ZLaJ9Wl+UXjDZB|hz*+8cZla6u8#@DmnWKYwQB8<<@80~Uh?>L<#5 zup+Gqow2Tsj%8ZP!e3g|y{7;~gPeE)Ixb_<>lDm~(FyvM;b&xalE4&<)*2C-!=#4< znh_^^3<**clS%=_r+puU{BV(M_)4Rrp7KBZAPeQqkY5m2{W#@t#_4pzlFK!Wrja3xn0rx=@* zhq{W2E5Ii@txrZjfKq!Xb8izp1$kC@iprhW`A9fH6*De!vOZjcx@rh@gSWtJEVe4A z8aWUJJZv0%N42={?t^P7nb!%Nm9d1QsRLJ zuMdhxw)9fteSQn0@5ZF=+6i{`s?`-Fxr(x%Mn?;qP5RVk#X=usrZ>B0DhDdQRTAdL ztZk^`m|`ZqRv&BqwE`~SuRpL5T4VwAR-xCeh7Cv9jz=VXt-Xr^w6N~6O+-%RMCz}A z>z7%pLAiJ8gQg&#kQH9wOT}YJ=yf;0cH#^!T_Y0RI093dh3kBo3p0DK6Ip@J+n~fi zy8f*kVjYhDdWzZul9RlqS=og4ye4Mr1~_A)Vd=2!u`}r(n#$Zt~deVJO2_`f0MvawN=?*4|4<7N@CXiL~mV)*T z+c22RFvj{lyGi1&1+b)RjUtSYI-S5Hj>j0q^V5ASAppR z8;bfHl()2a;fT}Lu^9t}@(T_Hu2x|u`IRSTr$J#WZEr5t-^Nlv;fz#m!zaXvJ7vdt zmaj!l(*u*XF70~eMR2f3E6pjYh+;&GVn72DiAvpBcN&56M#^An&S(oWY2*a%BB|Zt zd~NVo;h4+D4_K|)140VF1@vTOO>I={ea1k=`+gp(nIuE5ky$EWa^&R8pL-905r2bp zOlH%gbV73ucHVcL6gZQv6nCX#i*@zn@C0{sJH6t$Gf0dzY59n)RLwl~#Ud^#(Ji=J za3Drz14MH5%Q=x}GLeX1*Pf&aUGWPQ4iKKcbhracK}wu0W-W5$yPDi01118F{NccH zlkc%Ip?`Ej6TL$oTx&o%Bp_Je#1j@!xlrFleTc0*6rTZ)iglf5!HIK>$++jgUY`%L z5pJpy7K!2KT(?8c>S~Rlua|9kC1JvJn{y}WB&4Z|jLEBRvXGm5RTpa~Xv79ik7U}) zcCB*Loq!Cf_QpiwhectTlg#YAZ1w5tK*dTYrKH%r{4{Zwe#)@vPZbJbX^{D|)}dr4 zyr!09_Cw7U3dTg3Z|u2DY^0J}T}sf30r;!pBWi1oYah>h^1IzbZHRBm0uEm{;(S$ZOLpO_Gc`H@Q&Ij}kJwD5gh z>-$5tW7UM?yfx();5~7c@TC4VRWmhl7q!<94$3q|Tee#`HA-}}B{N@@&P8+gvw(Cpq3=#(4^N(dn?CgKJA_6r#$P9ju1|K{0BMykZ6-8O|~>k0>vv)EUo>P@bTD*0yF7z5?lQ z(|6{X7gW*DH~hF$9Vhw|JSc}_)cA2-FANlBY_FXRL1CZeL~7=u@GV~oSHBh4>|q({u6}G7UcB{$?z2v zDR!gNNdNjbI~^2f+!6UhukK5D*bO!Opzjy231rm z#h{Z!jtmP;@`n9o3+!XtP0nJvV2-JUrG6qZ5js^M@w82*P=TxCy-V8yPmky=W0k0~ zt(~pcZlC4pOqXHZp&wINe1#>A8s>65PxJjk&rNav$g`o-aNh>P?Y%YS#mcnLuZs+x~Gnl4H(uc7N6ntUYU1g#EzN1B)z50_fdn)A+t z!C8_U>Qi+yNG?!etB&RL+z$KQf&L?IpJOiQ<}I4?RCrJwKm4}y$#n2Gik8#H4*hD! zH=l_~STJskXC%ViYzNpJP7w(hwzzUXL19nKfgo~gOx-1a>VZw;LIxQZd7l)yV6yJV z)4}4ms0Fd8J9K^r5qUo4nfj{3=*LQle!G>?#mX3N?|1bPbb~a-LlPm%1ss*bK3VO2 zsH(8f!@7flTHgFaCfysC5IMnsRs1*)#q+=qcgFR3zbG$5dY_w-WW5pJ?1(6h?~b!r zE*bQNIb-ClG_*NEka*-UnMv&-Yt2#<7dHRyO{P!}>eg4f-n)EL25xe${3I9?iqAJt z8@lsE{!534p_(QJdTstNx|i}7Dk`rv{U%OG-%yWD~?^lM3~l#Q#3d_^|R#SiwJf1H89* zvlIR{;iO{XIZOdlw2?L@kk6YEcIj{w(17;vVM9s9@e?oEINP{0#7V4BPwm&32#|}| z)?%;$D44NeFFw}dskc62zQc_XsP%COR5Oc`?AaVXn#&jT3&Bx9mV9&Rslpw3)OPOzlWAByi{?GZhQKbQg#a3Qr7M(N1zYLP@?n(FXcbHpMSe{N+R z?)XnD-|>CrK~9QWLvk_{hmJ3d2*)5a7ew{iXC!-7LT0+ID=3G0%M@yLgh3+Y78EEh zUrilfq6YK>HtduPYY&(6mN?6q8#ygH2g{SDHs&x0xPlzH?0?XCUO%=K(;A1Xg^}J% ztb(QG8LiiBP3x7Ap<+whwq`aWpbPYLQZ#VH*F+=#(Y_gQ|E1Q>dCi0o!DhV2Ay zV&EFH(?7H$n-s6&Qs~a{Z=|@O*zn^%r@ea}%pSxW{ASn?UkMr3>I8No@4^IsmbXQx zMCmh2>naBd$Qimx%TRtFXJTE-@hYI1XRzMd$u%Y$a+-7G*7bXSQCwxT-qSE;zei*p z4egOEs?EyeQK#FiE*ciwoCXZdT8NG#6uf%seA5X#5?EnFeR2XpYZ_m1yEcA89 zS$Q=iAfz7BopAhwlL$j^IUq$`EHI+}@^X=31FVM|G8m5_9JB#o3?>vU?$5c!b2v!& zKs9kfqX>G-rTuePnM3#e-FOur&EXJ6i7nFF+B=`Z9Zkbbs+z+wDhPKrZX}fHW7Y-!umKhX zIG;a1P;XeSLl6IZquSUR0w(4K4zzQ^ne@)DIX2zhQi##+W#{lP)NU^Os(-NXh}fRX0z0hYM!otQSwG_wlK(DG7AK;8^*0u` z9*&38Ui0LSfyzS>lh4X;)9Igu?X~YTm{`*2A-yfdGr%5#vJCKE$GGv#`?;Lchz??aNxd9B_ou`($YdojuCO-0l@d}MppQJAhOziwqiU_D+KY+dHH zL~zEvqbwJt1dRXkr}1lWqCVyyB#G0ORywEa3N63BL22U=to$Hj7-GwgdBO3|ZfkDx z*K1$`uJXcRv+&d?rH`v0P#m3HbT9oFEwA|eLlT%0>l`($cbfj7uVC!Y|ECZb4S@5f zTkJWJQn7evZB%*k_gbDjf&{%n#8d(+&Ckt^_9`uUT<-B4_?r0Q9ZJUO^sn)z!e@wg z3fP)rmTGWFOy1CNCzwSp*WBo(Xk7PJ|9B=zu|N^-+ui4c`t$(;wbREsQHj0jWg#Nx zHYD;H9ldUkxTTLKGfW)!&wtxqLe7Wnw8eze;w(aS8ADg*TX_`MXV4E*OM%pk4tMY+ zwH0LeASGlW2N2+{a)w%qMEuo+p zcUhI~S|pjxoHn_<--Mo|xuz6yMcT?^M1-*f$1}jP_;tKVcne(Pz%^mkPIFsJ&JZ?t z+Y3E?vyOtzi4`f4l*vFd5{s3(<98z^>UHr;Qw?a1h?>C_3N^4J4+LqM{cUz>c?9G%T|6blv-le40{Z09J z6)tBYppDI9j49Byj!*$L)#G7lcuF@%If+cL!=7$sM)2&CKwu!wLzMZ4VpJu8RHBJq zIFKHAOSt;GvFut5-&M|cb}Xfr7d64%?hWA?>^oz0KII4oma4l2ZAxGnDqbTMWs<|> z_UTrn{DnOd^!H+FtU%djYch?QgJ*1)ACsFM0R~|yoh8aIsKbFwc06}0l*uU6pLZ~r zmSeB27UL~fJ@cJLX3>RGZgOfKR}!PEQ|cg;;V8pH%p0TxJzSEz*L03%U6-r38E)Y@ z^%@1dBGqhe39;NH4-NLH?fm?3aso$U8Xk#9A0bAVh=Joo!s7j+Qh3!eeuZJaFg^Yr zD<_>^<-6m^_IdU?s~hv|iaE3``O4@H^jabcvV}WrZ+7ApZbwPN-LMd^!GH@f`*Jli z!QYM}=xN&ruLXr|uPR}zlhyq#8eP3EybW{)9}pbTFLSkuAfd2sRIJ(r6=5clrOk4B z3H$dpmvT&72BF6|bSbUK&6)=6c7|ThrDikLt12s?h?RsE=DejAq-AY-T|;b4gKm%# zv-;D+@?!_`WX*(l{Om1!MUV%-xvg*`7T!E@nLO;AXNQ_93$y6iV1^m70vo=axYZ3G z&YCoMg7erO)9;}3^4M*<_BDQ&yPcUD=YxBW(S%?jpHY&{bQ;C#Dov>lvR~G|4BypJscHZUZSIHX|CHc+GrkSK58lbsUe{fz}ERi{@UjbO>l(5y;YPH(jX!o444wPMPc*hFI(sNmR{Rustr zv&cSnf_u}JF$mu|KRENog!PW4p~r3NdmkA{qH!pfvk)XG8k2vB=p9x~h>~ zcdP@69slO-(CUSiVT}~iI|SR##H{NPpdl=JAlUAWxp|70g>o8y%Mel7lpf(J$qX#TxKi>j$H#${D(R(gUQnC z+)DF_aAHKo6q)>h!sCRi<(Att?BlK>De`_gVUw#68ov|4?A*o5;@$PJ?xV$#d$LNO z&+6JtgNh3i*USgCkZ#@9nJEm(a&5}FOT*!&Wu{ew~=MB#)5`HD}+pUxJ66&SHq74YS0DmUqGIh>Q0wf&I+ot$^o`Z&2~G+qcrt)0od2YGTtrAP%gQKKgjMnAT%-&@{bKejh~&w%^d z0ITkn8sx$2H(5qlk7tjQHGTovPuf{f*`O6AOXg2YvMC zA8Ehs>xc7{Jo+)H(pIGI`@oW;lCNugK`pPG>mR~oQOGin*=~z@zyP1#eBDohalIK)F}QD50PI1+eC&DX&(8`5M{sZiO`{Q=@7 zx9NgaH|2Fzg8MxL45s3|3XdQ>7by7)v(*>droowVG4S~sWh6a=1fQF3A*GY>Vg1HF z;uyc)hkU-{+^;P4`{%Gq-y?b0f+meUpQ2&utUoGs0{zhNJGdEarc51qM(tP1rG!NX zv>K4j5pJ^1wtcZVh7oYdkB9Vdj9($`{>MjY!|?rq5*@y4f&csy)Pvk9V*bYn*8d4K z^s|2;_y1eC?Eh(FYp_aowfQqvZ(4ShkXSbq3q<0eko??GCL_)xMw+O`v$x%Ibku|j zq{cSE`+mzwrHZO)zaG<8bC&K$7Q8~Y83c%dqE|cQzafQv90yTkn+;jF28!fDCvIbY zH@rmcb5e`d!5#hY$X8_vGcF#=fxmjII3-?4mBJ8}VQiXh&!~F=hZbhwjp^irjctKC zd*4m5s@frE=nnHQ7%>Op@X4q52Oj0*5j1gS$g6*`6;vAb!SWrBxmFT30S|=&s&M6| zP@vrPm-Z%Bj%rGS{gbE`l?D3i)fj7|go)Wbw>`FBA+ccJ6r2|w{1^Ad4nwU3Y?6aR z?C^tT=gdu0izbFLH=9v)_i9~WYj$|t^HC<+U(Y-9ExN%+7AW|!{ZKKwr1IHW2sqSo zG*g8m;j9>N8DI9Z{1f)W1J6{Q5o3$70>gt|+f8!$E)Rg3^}z41ILQK9skKq}=xg=R z*D9;~p)B6}UlezYrV$Mp)}P#IauPdq>@(MObdcOK7P#+ew4fixNzt+&B8z-$Y@vv# zfW9e3VB&qiM;z`$KGsJvmi7Bb_d{?v#Qm^MCmvzq@!%=Z<16SJg0_qC@89z5Q{&lf zIcAo4c|5P|RC3sz2L32vM(W5c@ee)>GpwjoI`l06mH z4GMto2u(Pvr0aTn(yPmptC}?ItNaZ)qyTW)?)DChd|)1?GNc#w(n z-_%x1P^ZJ$V&{0Mh`(18`(miQFjZsE`rK9JmDcTO@xDq?5xmLuNIuUmFj{toIh-a!uNQgYVd*XE3)|as<_* z@aHul#_*TTY<(-CG?gyt%oNks>*XU4!ES{!aYLi-eDkBY*8db+GXE1>st`8(Rrl*; z>!3%JUBg^wE>Okn#ROEilNc+~Ou#!KwcP@z`WRsizQj#jPfS{_?pp8m zlw}G2a~e(0ks3YG4i*!xiJd}8_qT<^zvlkv3rUhEID1Fk0ZP5NU0^#+~c`MSqak<(L!iD@NR))uY@hyEN!m{Vfj{HYV=b0-%$!u77Qc)xEDFRNuH0i zrk-5=&vmB2FS%pg6WLMgEask`VDf2sBXg)p_T-k>jAWz}-ZVkv9w(R&jTl@Zkt_Ed z65REGAZj)uk^yW#5L?-+w{PE4L44>1W|=~cQR9%!{l7{-5eV~pwe*GSPEb3+gKXGw z6Y&Rx|K9{dejzAUQ_RrI7`RzFujN;6I#^_jj+lL}Qv7UW%0OL65;e-lIWyUX26S)RP1)b_w9^m}vm&}`YFO9TI) zZ}Q9+F?3x)KzYQXSXd*8W)3gjkzf%GK9ROEtV2l^I*gBT9D}GCCq|hNcq=gI@Gh6j z%g9*=ta}Qh8b%5Y`l0PujCOWhM{TdwGPF1U-ia;kG?D5{0;?HFPBW%IT;x3A*Q7nF zG4{pxPXnv~<_SZ^wR^m)-3H_a>!jx}W?)6Z^?~VcwmXDZ*SDR#RC}NS{aS`K`T$p5 z5*DNB|E#(LcKC>UyjO(P&cZ3Jj>uGw1~mohfy~=wa=FF3-5wU6l|WFNKPmx{=o8!hQ3+`9oa6jW zaghZl@9#sTaX?V0Ctvg<3CZ|Sco3fud;0gJtdZ@^=SuE)i)izWcEK;gQn)huztz;g zWNCbrrql&QF-3U5y710xFQ_$Il7+aE#aRuiycF(d6FS8;ILex(RCuUoEJUYAoFC`4 zV(9}-MOA5!giskIS;*!bp8#FoOR4p^JXPT-rY9pD z@LZ5k`grC)67Jp4$a9AlM}+C4S!Q3(RfB-XcGy!uWp*swoLTR3T)FmDZT*6}r$5dy zr%HIDxR3h1qEXIsH=VJ3RX9K#{gU|E=h$N+{Fx>M2XKv`KPR%GhPr>7oBt7m&NWWw z5tT;qK;(K2J$umkk6JjhpZdd0DRE#VbT~kfz0y~L8b>n$hoDQrTS5t|204!qL+!Wv zOVd>UV4wvJb7k?e_Kq;ftAr~9Xd+rYWP_^i{toXpRchKX^Zj02&T0--3A?Fyx>kw+ z92dmxX{F`Wn1Gtv`1N1hr`)7Veqbz^*A14><5oAwk@CUe3T_jBfD=w~YKOlzloUCV z4x2ila&up-4qSKG$-`EWOQlemGV^!ru49;*yi^2S6d!&6RdQid_+j_Ilod&yzJq!J zh_kxTeLiNcg6zrFZ27qGbPt{~_~yb{gWG}h;#h+^`=X!b?`p(=t_6sx$Fg%U@V*XV z&opD;a=aD13_NAX-+|iGY(;TcZ7N*;o5CxIkUBy2*4K21#4Yr*tUiMBR+33%?72M# z^|lY3XS#SljHdWZzn7@0Zl6G^OIU~LF!CnI{G(JPE=>p99cGixFH3UQeQP3{j7qUM zfXW_2=9ON@$N-$iTqUD({0Cs_Y%r{PGyF)V`g*6 z+O#Sm^!-|$m~0Mn;p%*t5YE&P-oKYf5hG$S%kP6cvD&{nS#e&GJ^i`w&1CEU!e+5h z@cG~BI>2avBs7&2OeUd-T8?~@Zh_y^voo+0&&)$z^JsE5M(Bt%*n0rZC7c+;Y8>0fk8l7QtFbpJFmv(G0EtBSXZ~yaEjKf zx@SW88fG(Dl@12&Ly%PRY9kfE^Z|^XM;lVpC(3VI6HEQ#x-?!c_Wy9f)K@ISQ>F3PSm z`KZ=B-0&z?l6tL}y#cpabe$>>B-Mxd3mg1!+;`oqa_f5f1<8X+y%o`ErW8tfFr2pg zeyPpQZ54VFt}0ac%Q3TXD}A{?-CsymA72EZNY1Af{TWsMp@0r6Hs*s7CD7YJ#VlV&U_r)BINI zM2qgY63)U**2zM#-1ZfsuF7;%MfXd33Tq{zjFV=2=$%TCm@c_yb;qQKs&oaKAfTNM ztEr`lN~2&ppT2^}5q~S>ts4%p(v&`GNgTBxC;1ft^~RH>W)IQzOG_V5$Ym z;_iRX5}A3Dq-ME55Rz`^CQGdM;!G3klRFIm5B^5)u-Cto0^iY=OYb)h*E#hkzRewp zw>he~@l*^w{fX-Ti1ry~lwt@+A<<6eV&sD);epR}Kkf`ro^_p>5-=YR>C%!&Npn{9Lt1p23W?Y0eV?KB~= zb(-rY*(lrYQLOAw;b_<6`{rN)-h6_XubW={#Y3YnB`9||cw^Nd^LpLs>Hx&-Sn8}W zE>!hr&&w7D5E6&#{-|IJH#8PtSdj9xT4psBEMraIe>#tr2+mmO$CYegh25`q* zp~|u*Z+^4f&VXxnhPo_if_n8g<#Ce`a$#4r{~Uc<74wJlm^M0WT??F1oVdIS&U{ z1U-;~%tCW0&g;m5IieK}7RsZMlBc?brw$dmi3o)ah zp;)Gks*)vl=)`|URwkx6LqXQVE#`{&wwf|n{42Wk%o}qE4~9@EkZ6AY#829F1{R`# z`;WW9yOzl9JHF}+hG=9;AfoL9aJg{D;?z9?yya>j*UnQ3CbA_idL=iRUNzudy^-)p>@{hktao=zSq&XxjCOb)8aHD*!2U<@W?`TRY>FRF2Eg5cp8zC2~^0vpeY56Yk680ody z4d<=LlczY)MRDh^0$UAn*>dQsEG=9#z(uD`2P;U4uCgDbAHHzI^3vr| zZ1ujbt*7rl33uDtooJTwB;#3<(e!Pa361xa>-sH}AhF*~;g&)V1niE&To6=A9hpDS z4BJQnn5-Lz@3Ho-oB`W;Ll-aCU8Zsz6&~D=9);?8gpAwPkwtv(^SF_!jYO!CZ#-7q zNOOn|+cMm)c3@rtbDJLa9bXbQ$T6XI&pj`ygN39mk*ZPdO?vBVcJ1Wr8?5c>K1qaQB9gxiTwfmI>h!|hy zs}2u{=W#5bJFCLQY*OOPfRBXjiLj>9bGO4 zIO&l{pdE;&4ZfQS-I`Q{%czXNXauh87eFo~o3+62ji#{hWm;O=a3*qt-OiUQve+af z+5Z)T67JQx`oEZK;Te?JUeM(rT)8Dztjn-1p_Zds6IBLcNSs?RiSq({TBrXE`{qJ$ zTFSXk+fg6rV5@kggvF2)7n3OUYJ^rv6uuhtj9^2Qqwf6`qDRFbd@&`lP1g;K$E-E8~G?^SDS`SYzF~M$i6;Ir z5PQGUIOprh{j)Lt1f2?tJ6bac3Lc+GG!)-Pz$!inx@F(3lD{047{KI6fj3qD(ackV ziLG~xSVYn-9bM<-hri{Fmuj;eXc%sakATAMdd2pr$NRc2C5{qrk6WvpM4UR2Uvs!^kf4XjY9)IcAu;)GXbM`SN;A)hNm2vaw7xVS8ZK{XxNI}!IHh-ib-OAR+uJF!Fx1musNJtz`W7Xc_xvcN<0p(OXcnF zm7`%e5l3Fzw<$a#OsazbtF>=?bjEDy$0 zjbuvdga1vVv#Meaq5Nf7a*-p?%gqz_# z35=?P>C)8}DUwWtfNNI+OYDl2s}h$Uf>_C?R=+t4>dUJpHRu$MZoL85Uv0)a=RQWz zso`>d6mu?{`8(Kl4eHF94E~L2s6?xg#C(lEgvpuyZaG+jBZ`TVQQJxs?|GNcB2*VcYWj{6fq|9MbOvLGLVoCuzJ5+>Xp(|E~l}Gd?0^9voCFW=JOQ`Jfju zV8r#K(329Atmw8utki!=E7*-Y>8v1EIlx)IbMUY!sEJXpU_^A@fXGs+^_uZR^{XY# zP=LH*vSHf8uZsM5AXA^(F$LOlm9BKZ=h}_L59rd7@gGvQ)@&@$3i~ewO`aLg^o~GF z9>m6;H)x4FnkR@tmri1li-qKh6DU{ztRy{`W-cdP{peo}qA>TLah9*IN7cFNcV?RNmy`=qGnnX5ae-b-jn2Nlcu%KAV|nb6`h)+BQzV ziaLp56?m%@t=Id8C$dmq7Rv!y4kt7kX3Am+Rpe}=; zyR$uk$)N}K3=7BbwacY;6eYbFA8bjg|IXlW>!7>|^W_Hj_;F9j0fLz%w*Tj{P^ zB8XI9ldd@x&+;Cuz?wj8)jH@Rv)t2*nU)?sfxk7>S))We}H@3B0^x;^8 z2kh-8OgK-5SJ12~o>mO!xt|QN1HJ^+B#Y(GILRNS%wvnN>T?($858)|90hihmAcN; zj(uSBb;9(2iFJZHIuLFx)gvmf(IWVK{*nguu>MJjaKh1oXZNT>wWvQ0Dv0I-2JxtL z?$(F3l;6Kewn7)#EbnI=c6@<@cm`fSsse&hCIHkN8_AV_3j;#lVp`^QMkbR<=`O+< zm?}mw=brZ|3imu0-@(kHE#V0oAqYE86*U;`>i>zWI+N=Vj9*A_>ieeQ3{E#zmVFzJi_p~$99|zvCCI% zy~^@qUjtn{c7Z6dnuOL)?bbbY+ zKy6%cEDa{1+)JEUS~bh;HO8M_$6hc;GtT>(&0s2@jHDcnWEs5HV&qd!5|e2$u%nj( z{Ixacvp=7L_RC+Y?pG7J&q6LYIUr-uX#w&I&&jO#q|OVF!_AUak2Mq%YA(zWm0#Bc z_ET?r3e*v7en;2y?9opHx-Tx30vci1_{mO3BW0d9JgIR-j%J016*7Y^D;tJ_Ac%Y$ zOCtgU{u|eumkSuZ2kaDvFev{c4JK+Hy-tj#|NCp6?M^`vCWjp-t0q-2kc5h^gXL9j z#XorVmLrczvVVF4*NLVMG|$%P=cIpjg=|-eRex6wi{w(e?2W_%c)Baq8@zc+4m{~9Y!`p?3l3sHmrD-M%V_Iv-zZ&D(XX4J_4 z;W#X{l`V~SBa0GLZMxb@@5keDvAuzW%bq|$z*6$V&S&kf)W*20e^+)X!Ip}y`yFB2 zm#(3fay(;btCew4lw9?l+r;QI<5h1DAuNR)HM&jJDkrvZ9*5!BE|VcHaMAQ?N5}8S zutADRW61%^Q(SFU-7*}LEn{Z-LcH<$Lh&FyY|#s7vE+Vy#gcPdoHXtjhV7MG%7(Qw zqzink3)jE|t+%0O+z~9NraK?}hT&RmEZ)qkDlznBEvG##(q+31COA_?I2Ig?I$ien zd(0|F=qkh!)2?80bWoOeEOn#peE&6O%LNgF0859v*653h*>GyJ_L;L9EJC!LL3uCqJKbZOJ0W*9Tt}KFE;q^kL`L3 z53_C3cY|W{)8%D!5G~t}eLMsXLJyFlX-u|sHBLz%<~;G zp-qaG@%UendlB4+m$iV-jKGZ+UDRFci>`|)FHthpJSg|Tb;HTS?L;@qncs=8U)ROf&Wm*)W%y4N5LmLochz%$~Jn79Nv7JiUd z2X5}Uqp!9O0tp){bUe4fwt#$3CmT^wEb`X%<4JL*{$itYV}~u6S4X_-x_s;UBvsU^^bska274mIt{ig1@E$+fANuOgO zINr=&MIlYec~hSF$@3-SYdiRKsft1kRq*fl-=om5ty&>mJLd!Kt z$>)PoN^-LPZjvX&4E@O4lSylxgse0zrfI1tVVmu%2vm`e(H()_8B|wHqp+psGgYFV zCoLgS5kuN62~H?TG0s_9W6)~OTe}}+28R0w24vrdS}~XbU?7ir`}1n&d@5KZ9^O6t z;d20VKXHpYpHxq1Kd+_0RALYV7n_Fy{=Dvvoya7MKq6P-dC5J_&}m#MxPO zg-9JnFIv&Wn>0>+hCS&wgv{K4nwP#4>hcIno?DUmgaQ4lN11S^opZGL(i4rjej1js z!Y8oAnyGv4>~2fb@6Ul2?O(~o4mF$3m6ali-~=D}4IM3_vcR>XKCmolC?Tj;aVM~Y z`N<}%{(}^dtGXMn$o<@<@N%0Q&_usie;8f9fr&hc%k@{yfIZtm-Gv8~X*WC2mgv34 zbY;ReWuEcu9170i5okmF;0g^9)4998U0s6f;U>)n`k$fVlHjlru$Krf*CjO=@X3?KQKDCMS?8?Y=j@xX>TIMg{_fX7G=wDx;&zX|IKZ7k4C7!K`a zuJE>S481)bruQKP;k^e`x`uE=jUocLTW5HW@Xo~V9i%6_UH{Ve(~ZR$i;y-)g3t$= zc6sjZsxo)m3M*%JHb;IOcPL_b{B8?6PwxlhpU6D>M|(JzIh}43cBBF^HD-c0_!-?U9wg ziL@D?{<*`P1$DZ$f?WPvTJdv((V$_+Gi8PTtq{?Vc)W@HS;uG2dYP!s8i-G4Za#}q z@B=PiD}CJZR$7B_pzWE>&vkeTs148aUS=<+*gl$0 z7h}OH?S0mthovd|*l7D0v7gwSSAi(yc8_4yd(>?t zt||Tz?D^uef_-?1(`*q)xO!(DRTW2ugv2j6?D`c2ulqA^8=A&gYM0TYOKnV$zW2b^ zoHAJcN_#}f=RQ-u0hn~Wlx)}3^8GDEmGi4RR0Q}=?3l!b+CwIvybucyq{r7jRa5M#tD9%>~-;KR@0-N90T zVc*D0PmY-BS_W(tmG$SM8ZZdFnQao8=3pFG;3VhFeEyhA%(vyna1(uE@kl! zdbakaT0oG_knLzWG3hm~JT>mbq+)WmkK0e{v;Ks2Vde|%5bNc|Y$GbNYy4Bcpv_gE z_|N^AN)~eh>5EEo(i*^J)$wvI(w5WYI-E?HWcgzyd67Yxk%0d1xvC9U@Po7R32J2a zB}$*`zHpu}Iz6L9hfZN6aka+UQ9A?9n@}e3$A8apFl zW*wVgnJF^UH-wn&UB|b9)A0J{zJ3fy8}O3D`+e&T=64tXx@14T=JWBUQtS*D>L6k7 z2tWCudq4aNmq>Zt(=gw7~EUwkpvU6{1qkRt(q-Qw$? z6tAk7r+fG9Ph#Vbo~WrjE>OC+J;ijOy85;}k~cYb}~m=@8Re0E5! zR;OM_>*;e;H3r=dr^U~kLeyWk#zw4oi6jbZ_I(VeEf%rP15~tv8?Jj4rE>g^tTbr~ zB!4c~_EGfv;eF;-1)4Ty`#Jq<-AOb>xaWF{!za-j)1YE-KbgWySu6dk>53}Wd`|9; z7o?6-9f*!Ih#c*AW8ml^H_-~*h6b5_6^qeakBTDIcDK6fGl$^NOPA5*;it5D#WXE8 zf}>IFoTJ)2Rn93Kt*^s&cMJkAvd~kaG;yqV4S7 z;3_}h#*5V+ue`vfC0;)~8AtWCcGFGhZv#J98bUuVBn%Bm#I$?JfvQ=`P%pKjX|Rn2 zOQ6vx?sMX#)?4|motN^bhVWKI)pBF2rV6wOnhf7dsjbc)TW-bO7RRUAEw9|r^ie|v zXJ0W>YjMJj1|g|(8gsrjrfO9HqSb;p@g*vE`xh`D%sgA!>ruNkgY`v(z376e`Z%Nt zy4;LpxmLl_U_N8*$5;5n3l->9ol%zF)AEdf-Ft6Do`utk6_Mqq$2-tWtPMd;8sJ>? z!v!h*YNqzPiK{9r2kzX|@XT(2&ez7#^n`de+&3cE5@mXcg&gnDRG&l~1Eg`nik_@H zy7*69KIfSDzDjrj_P?~yH_pkY(GY2=tN#S?>WCJ8o|e9+CRj#Kzx-oicN-cClX{cE|8m&J5@_z_$hZ-g<#BrumbVAqn@4BpBNm57mZdeWxu zl8NE+Mn-H*R=z%?cCmi?7Z93r=ik=Wc3AK%A|g`cHK_PQ>~Bo7HksA=+`o8eA>=53 z_faDLb3xya3{v^1&SKbvOlzU-A_eyNDv_=I;n{IH&o9_HCm~p3xr&yh%ft-^ae0zA zM86;Z@7Ay|cCD?HJeAE@9UiEDF(h+aZrE8a53o?7O>5%i3VAHxE(m=E@Ck)KGK0w2 z-b4p>cT{U{tVxUPCK;Uh*O2@t+TuSBOOTPg_Nn2nJfkR|az>jFPZs7G1%GZ)$&Fq( z3b)#Mdk!NSl{i-^HV;-;r?=+8%tY9aNKlT~BmCTxe$`(!VgltA=|z2NIfpF7?2s8k z_vj4@qX31k&FQCuLDQ9ABV)In3&&$16!)E6qxJT~^R)pLF8+QH8peIT*B^*<&fAv@ z9qssATp9d65?H7(L`afQO#a8*7pLf)t28kHp<1}daiQn0r)SfSE&_B`C8 zW=?Tfjc^VWQN82Z8OS*+f7|eOelAZXWPu;_N4zk6l3zo6+O8P90Nlr(v%n}`Nue_K zVBQMf;=V1kUqtKh%O1a*{Qmy)C2mfow4NixE2G3#B%|pL?K6i*wJF)IDlTBY-QMQ; zGcVHkH8YH}e!U1`-)(9*iinf~>YxW-N*mHu89bon?Jh)G{)7%Vz1|+Nu@TSNbd!O#HCQT)tRU*C;1 ziWe%S==}QoJ~Fu1HfH@5kTistDKYu+__SxwXk_6r`Q%^Nu>5TCj3BSo{1rsti@(U_ zf5%(aN)dEA!4$NHge|809D&o7p8{MUM65e+&jAlw?iXxEDxIc~)hs0x7hj0{=cs3b zOU6c5$y^yPzAn~gXOadL)hbST@XM7<)nv#g7NNxKQF3o9JY|BP58MhHq(NF$*Dj!f zw~u<7xvKXg*Zm92GJ>)a=JNP`-%VWZ&p%#|Pz&2Vt2E2f zQ$i=ryG^e#iVd*WHx5J!uhbphIL`5A<}<8R^aey`JuzL?S7e1Z+mKEK-s;yrW!y;4 z725pQRclq7;Ro}{(~$T)I6@jXwtCxtyA8Ph$8BJI{!;Ug;wOHrfQquD%m^eQ=fp}F zJ*}J>dh^W=E7*X1r2>f7fLcG4Ah(8|m9jO1yIx@867|US zmnnVN*8wD_N&2sFZK*FFDn(GZkGDcFHH_nQ8u;-?3%TKU>@MnL8)F2XNY64+OAj+U zx#IkqVJvP4$5BDwAzVr|Ahi0V_|KpV)|s41Xi3}g)#6i_Mb0wnY_{&oAiW0$sQ%a) zC%k)CSG8{xUUPDC6#0yh9of7tMZ6>MSy9~aS@_?qy?rv* z3%l0z_tH~ru-Gv8?meQ-x)(bYRe~7b&v(H|$d4t;Ff(}wHaxd#UC}_|e{YyVnP*NV zjdXQaKk$f-0je{VH~O0l#+#PcRGDl+6N1Q!CJPHk#G+bS+Iy2eZqPeq1m#H-bS=;W zOY}?j6F5KM{>fN+FTx%GKoBiDuIut@ z{!eZWQX7x9wsvvmWrO()<%~?SwoVl#6$ps}N>Wl%r)meebuZaXZMI3;Li;Bp2_cEP zzjm%wyb3U7BS-MgGk!u~5Rl5Pptp9gMLel?7Ub^x8IN`5!CwD(t2UM%rQV1F|%m0Lg!jNU8 zCa&1Rt2)@e*l9%F?qogC{>8d)>DU07q!#-IH4tp|-b?>g zP2$ue+9!jVo1=9ljo2vR!E~s_=?`**UU!Dq2S-q2XMPugb$8ve;wxYc+2E}s)#5(R zwwkTM2FX`R9|-JLen%1+jlh*ZirQfA_``XF5|dIRBaubx&ZQ*pTU%dJUQ#j~`ke_8 zR~p0_MQKntPPpyr34bMklJLD#X>T53t8&1vSBBMMcN=@ zuQa0&tuKr794bn{`*ad6xc%r`SgwEvGACASwGs)~u*MTE!;i(}(p%j1WXHz#Eet_< zpU*(Yx=#yX&3L9gN?#$NHsPS)?6qzvg>iPX5_6mKE*=@3Qcx2JIARo5R#y*{K5q1% z1S4X5AqP8`_o!%Dmx5Yc+v#B?sh=#4%+9r z;Cimr!?-1Y5>Pb+mBwE}Ac)3ZI50$;up96weY?U00Q{<)SDY~?bM!NoeCp(?)c7Q? z7W_QB4qm8H1)du8)z%&t4g#jN#8c#1CLUmPj=4RMHwN8a+U*ES?QtXe`b0R@z-lp{ z@n%n;1XTm7E&bqi`xlMQt;|663M6sd3?;(=3D55z|5{Xctd19DH zcF}Zg7qqb%lJ{6$lw7u4B6#G^S0NaURnc?Tg#?gv+Y*z1@GedL05S6%S3d44)X89R z*lO{fmV`{75D$G~;J3@EXJi!HnN`I$3l3p$0*SK^gA?#OqyjGsrYw=`7Movq<$}7g>3M3$dFWy8&2q4r>JnUl%v>Cgu?Jf*TTRjU>~yQN6{^s;061K znq|%PNqb1;OewjO5t0{|V|Bpc0}*u+Gd2m?aCg)bIwrF<{Bm(Ivf#JxLu!2OJ^}VJf_GhMRoJ%kdVz_amLqS;UB&=ESF!rzz z3}YLc(p1k4>aw6&ON%X$c_d-(v1KJyUcWp%cmAg;o@IcHNqHxm zWgz(cWV~QEQv3N$ark_hJ%d{ao3WKUWGcfDeM_)fxsOQ|)PC>*Y2U4llQl$}{ZWoyj}wJzT%#t`@%;6}$p_jq1y-l2vzh*~G~;#N)W6UCr&5|DSM zoarY%-@2=Iz`ooTz2$;2E_-Zz^tWahaT1Dfr?q>1WoHuMij0UVo#Focj@YdDEmG63 zn&d}TT(cNM!884QgIid8Cy^KS2*f`Jfo@rDKr@^<@OfQ6h;YOX`+S$?P%PK~j`S^& z@-9A>yR$9W3=pN(!ydoJ2ld9GqkDbNG4ICJrp+`lq$!N>l(ok|!+fb8K|xSJWHaFc zd&-AM(w^o&!zHt69JTW!a1;kPN=;!;OJPrNUUl?Gdo&c*8DMCTG|={H;#_SvXnL(r zZ@73$e0v1wSB;!SmQoGEhPm(Aa&tq+^xx^vsGwYQ9I>o(7LrI#`cs98v3s#&7u_s% zR((H2o@#|U9*#xyytF1YIY#2o2ytiAM{?gn--K!MR`j}qjlxF|a~aUSAGr~$Ed|0& zjaL8*)+XLirY>Le@2GeA>2j)t=C)y9> zz+O@uHYMzxk_8QFrYt)MfWJYu{i*|*VT`6M&Z~c@T?iPI|IAZ^$D4|< z>9Vi)HH8bRWDGku)iDW6{_iVK}C%-Es zAxl1wosZ@CD}g@WY=kqzMDMFU1oRr;3Vwc&#+)vpqZDNf}DJSG!iYdk|b zyyAeFO$1SC)~lR0il#I@-1sfYL3XwcyZpgB(#!&Zn3)kdTTEta^azQ7WxLV+2wztn zDW4H4g3}|MW@)c`K@z&C2y1ZYY(4WD77VZE+?KKfE6P($2+2#IZ+2KGyu$%ISBij)?&{W0}9U`gD&rtv~sk@OZ%$xWO*N-S;YJ(O`Dkb#A%|kv^fO@aVU# z1Ib7rCtN*qwC@ewxw8$eo55U(IRla}`;Yx&z?CLlV~`~cvI7PW#$tN35kfI4SnQUF zyBGUpDVgdm8+7M5CELo5)Q?0-f_0716+rWRr3p+zA~fNf01%I~+~BbM$a)4PEPJkZ zyu^$IpGP`AG4oS0%f;^-U-X7rg#cI);{iC)N6Dl?*0`zSbYs@Nd% zanTtG4=Y$De09=_nD`RD_DIcWo1|?10~}TUh9tkPuYeelBgS`8u*CEx{5i5Y@wFA>nyPxsrN&1;T?&4HrYxpf7GgrW5s0wRy^>E0dVb@xyw!-YB+j z#oybBF7Jok;+fD1R7ZCFll!$d1J;Ofzc~b-ood~?j+$OO6#}(!GkM7EMzT2%r*L2h z_Stz|-ZBsws7b1+k9^XfiKuVfV?x%kZUO9#|GH|l1w+G^=2x`EP*vbg(S549<%S2G zu9K=dpZb_H^2~EE3#u#91GH%tPzACF+L{Ywp$Deof=Nc)&FEM&xqrk@xUV8$&OL*{ z(`E)B3ANJOo}#C@XK>B+pm?v3cbo6uhqBZxf(?!@dK1PgATe5_EoCEM}d zv16ej`mN^4#~L(Ug(t6T11U}$zYP)!QOZyC?0i>Q9Kl>xOv?VkBgW3c8?Ey^)^)hW zT3K!k|1;8}G3KfjhSRv}=8KTJxm#OL+RZb$#0%msL*DlMO48vh9KM2AFIEvv5cNWs zR*6{WD0zuBs^cp8dJ>Dxd+<9gXvB`k&cSbmykV5r3#xB2-BF3hfTDfd`wRl~t2vAp z2NR?Dv1_{g}Szny3t_k zEo0tSx{EUjBWw^MOw_%ws~rhaiR-Kiot_U7%6QDxlf5%q@6qESfIuo0o7al)#@Mt1 z`8QGhC4wsYqh%C4u9obYpK4#1Y>C{Jvc^3aAq!iYSy7dg;nim*TFpNG(0s&Ev8Odr zR8T&Qw{MJ-M~XZ=+Kf6>UR7Bt12vLbt$cpeBa*LpjSQzSuB@v_d6CpqV_~jSnSHr* z?IAy+?Qq7&z#kYdd=V>D<)1^6ciu1?9pHPISm6EDOrDwPvRa6MYS33#VZiG$sRUPlPB1)8#jKYcSUq_ zp%#nS(TzBe`&dKw+me3+A8i@8&kp)lMIG?|n@`=_ZF3)m9a1C~Ic~l1CBi!9<~}&> z2YuBE+}YTQu7&)tz09v#R=dfEY;^wD&F1tjWbFkA5pbS}<=u-Epqi z8fq~775S{sYd|)ZS|~*VL-c!h7FZZdSfG2c(o!o*6w~$D->MZY=O@p3%b#N-F6&Zr z`jdWQ8pisR*(a`_pQO>HQ#4XPT%|Ua&7o(2xaA~=0{{FzNnfrB!5T#)OY{o7dD$A8 zr$b9UXLDl4bB(zkN(%>Y3j1id$uw3tk-EZZAE>^Fq{I(4CnrjLJ9LzkUv3ogHGO9ijF7V{PyM{C(sgq&!udKf!+L?7;}XX>$TKz< zb1C;{hR}_MOB`rQ z3Gi<7*+T0n=+QR!+?H58N?y3`#f3Eh-KA}m!RLZgkGyp&VL|{wa!!>3LSyqykbQx2 zWjVRXaVURU_*il*sxh9Z{&oM1Y<)2+t>r?rgg!7S@RMG)uhI^~8a!Xv6*85_WC0&{ z`ch8)Gc)UGMo_$-nx~j=&ybky{<4H(wPiGZo7TqRG8nG|;#%*x^{s8w3`g}*m3HLVE=n!P4@#+JD$>E=jEcB;Ehqmx-^ih9vF zBU9hh7Ea}p=BQVpG&JhWV)w!`COQqTAD0>3g^a$E$D)9rQ2UY%r%I0|zxVrnyEnqw zk$2lwQs1XoimP)$X`$oMt#c%gM15MacVPsk%5>->X+8o9(N$2Pz1;9s`4{42%|a4g zYMA20Z&jKB)u^8vy@-L|>M_IQXwht5#|&Ipw$Xl$m!F0xD-n?FH|QXF_{-@3N^5we z3xzBFLQPGL`Yola`wSsMA}Nej2Y*;QQ0>FtCqx3vAL-lHNZru)JGRLp%4=c#-D!H& z7*>P~`AGGC#B;CTJ&E_ooCH(=SL@W@u5NJOj&3&n%f|m5i|rg5l6c3~l*`of9w}ni zgPTA9!}#YCIH}r9#WM0Ww+}G#$l&DN0lzoT^9sUQdAv+8dZGcC?E}BHOk*&ABxH6& zj{rfkel5HEh__eu-GA?QdHJg2d*p>Wy4AhajN3gOc7(b&UEe&IfD85htLCHZzztQd za#4*chsTFq(B8d9qoh^rB8K7x$Ifp>5|xME7VAt)Fz^(Lir8Eh7$anY{)bjzZ z-p<@*tXQ<8tdP`S_m4Z z%Hlb61t!9@#cdB0nkzK&4V?+`U_!UeWM;$Qgu;!rn!D4Z)=xZypR|L z6_Qp5GD$5C44&cFlJj8Tix#q6866oHFN_F`;irZpD=V_bq(%z6?_cJe;mU(2o7cE9s==vy6w6K zk%{98`W+$H%MtSGM_7A}P%ui#%@t}IUQei{4YGz7>x|o7pjvlG ztE8vKwcs96LQ*GT_h`qpXgKTM_(m@r5WL=*pNF|nFMnGs1Z1YY@`#P+w}Qi}H+Sq| z<%uS*{C9jx7*rY@uQRG{Xnkc;+-01$tyGS8{#=Yu5I~G%dkN`OmK^$eyHIjUDyntv zDGursQ9W$yUP`Mz#vKuILZU};o}m^wqcsE{dn-a@_rk9@R}|%YM55GyEVgmp0Kci+ zHAtDW^X)FFZB*AC&rL;R?^mlDG?CV>;`xLWG3kCMHEBTtAPm!wgrxBbWMk#ekJ5V89+!B^W# z^Z1y4=L0rrkkFk3N1r+cWa6`A7fo4SAORklx@iqt38w+QWbk_G5|(;5$P;X}$Njnz z>$W?t2whz?5*Mx(OtuZ)L)Q9N1E>3ZGYBnyf=F&IOz5-^AkIU~0aBJoZ9<)xl4J}{ z0OlRn|3EC0)Aj(BL~b(A@O+l@fiGf$i0DOW7*Qv|(PELu7u19Vkx1j-%%UAs2}Sr+ zh@Q_fUzaN(fUXvW#BOYZ0DUcC!^H$%%U+P#8)UXFdtW?e>R9b~6|?R=Gkz6Cq2Ul# z?10DxH`Ev!vgwg&I3$09sAEe)rJENd%%{k{0jR6Wt#&Z*kyMa|;^Lh50TMTSCp8;J z<2A0|jEMr{ZJVgL+PMFTstz@QGQr!|^l;nNRZPA+!tbTHx;hFnnr-hxp!9zC^Al4# z7a7e6zRX4a>9BZA1C384p_^uyes5ny;*Ce-K9;zs@yst`5)yF&GXNU|V3)Wba@@lm zfz5s9vgO)gb&|FV3=e12Nz}$%w}%E;lh69UuzU(lV zZHE>c!JnYmRFSp~bjl-%qD+V6DdhD=`n>1iJF&H=dvtPb&xz7D+?%Fgq*0JhIrbGD zYA~+iLt8ETaw83~4XrM0z5;h!00`e$Z_h?%_S!ePbrdrnS_KS!m+oVms-AiL(bJPG zXDglr$X`XOmB2j}NCTxO{Kvces$4XOzfg z7>$iaK<$`ZSS7iU=Lf@R_xYBy1f|Rqdw+0!>pjYsK8{mpKuBlaiA-%Pusb}zmgxG+ zUbdLnT)VIXgl+-W<$O6taO?Aub(cyame{Vurz;^M$B`N#blb$5TDTz|fQ_=0w=IJ?5!{~6706(&#CbiIDt@5t+}i1I6WPWnttk^_KV#y)@+J2-Gj z(B|<>fxX)4lQQVf@lp#e#ese3zksZ_v=$qD6M%oE$sW`)@-oKG{|%Bw(G+R}o`Rl@ z*>CaSMD9ahRGGYKUMI~WSVx3QZ%2h%4hxc4%h67)HGS6tRbJ8R4CuAYaRDHLh>o|w zv}qi^=@QR~*8to|OUm8kW9`ku2`xxSD6#7z(*Ob(e$}s0a+K_un@F?X%;FgVPX{j~ zTOR#H8%C!SpNv}zG8W)7M<8suujyGw(gS8pA}4;<epnswZW&ySl@$oMyZ-%1M|k6*f}sz*{K`xjahJeMC>m_3W`PM+%7$~c zfnMY=SIUlyiy;9(p>(2M>oFIJl-y(v)-jd@3HO0}E3@@;ig|Zy?V)OJ`>mF+Sf5b^ zr({cL$N<-Pxk+B})6wemMI5ubBMzXr7HDxqHY&Z6KEUW}FnvKAz9HJMEN|{5oh`r` z30}Op+xd?6ncIZfjceK`qcAHO`19K`fJBH%ags?myk07crcUrEp>Mp*5Ps`EHRMSp z{u(Z(-y~zou1^G}G~wTQ*`zbC#O{``lw`~X8ZOcrp(F0VNT?o|Ppnb_LKILA4)!a@ z+T>C@3YdB?dnXi!F2T>Nua8IYhIL2T#(hj6e}71D6Pt?eEoSJiZLd02QVrrTW}*>M zV!s*G7313NybaU}^XSQa25P#5)*@EIAR;G>+@A?|y^I+oRG6h=DZ95I)F~KdY7>YP zF=ENtv_*7+AnR(jhfs_?=o-yNn#}P-x~+xW!Du+GK=lSIY-UoSC)#TRB|n8$3$6CL zT)OwHCR<)v(Xq68w>rKXaBM97>hJKgs)b>u42L`aIgCkO16HjXtwlN7Cq3r;_HzrC ze=0PSV$mLCLEqj*_#20+=Q8;0%rH@tEgYMokrARIU6rHEI|TN~frf(@%U{t&NUg&H z9v>~$b$;-45iR;Una%x}W+3~|wCp|NvC8~QBthLb*VHgp{rES^Hrl~YgXSNlu3djI zCUU-0iv;|$SiJ!Pei#m!j-xN+>1f)tCWGGRN2N8yN7yDSu3&7 z2M8dQ_fA=9cf-9S@+{-WCoNe2LeY>X^=Y@ERR2iu)Dxk zXbWbF#Q1G1?u;@TO}D}pxHx+6{DGa-qKmgZveD)zb_!|m0Eru*twB-(D{~pa3N&dF51-5|iDNdcWz1_NY`#2uFZ{(+Dl&VS2 zu;w!YBve|OD?1=>KIEE>c3^vN%^_U+1hZ712;L!Dg%`2$Ryl4F{IlOsyCTMsybfCy z6KnxbZD^FG)$`Y7xO%Wxy9q_?-&rXmKi>|ocGgr|9N4zC`V3@+X1IFMqLI)Ypnpa} z=nqzo+EmsE+4qAN-Urd1Gm(F{xsk{H!P>UIvy5K$PUdxg;#|OZ4*a&Z8cov`YP#8m zxYk%FG^8xO&$A2$C(;_hO-8e08*w}ZRfe~iJ2nl5xk?BG!HV1Sf-55-{@=W56)?(- z4Yy&>fw%{_lS4=dqc+W@nL(`z{aGG~#gn1N6vK#H|JLz^Y^;=qgD_rcN3gwI7n={S zOV-(|<6ub>V^Hw$coz$$ftc!d=uY4GqQoT6_5yf>!S<)xPI2cEat4)!eO z-_W$~>$Bqb)tbEN4tc-$%2`R9SQ&sx6p@*H>x4qBjy=0QyO@0t;h)sESqpu0hKz4N+0pz`>-&z-$-gdiizmx0Cp9>93#ZVfJvKCd7 zN0ojfYOFXaO4@gH^!02y>4IX#oa$4Vb_D)=hoFHgljH_UB6bpSEV^e-_@XX{H5LHQ zJZqXu7r-anY+k;ueV0kUUgwYPjQoSIOSZTb1{KUChPDQQnZ?fk@m`*mCh_-n8!Y^$eidN zPw{PY!y29niwk9oC2o8=rE1xX0}G4!MljE=6~%NRQLZ8`ow2i7D0&zjXN;Hm>%O=i z-cv13zH)UxM6V% z4;4Hla(6E>2-U<0HwyYT#{$c2N|5;~6%T>0CeKZDEngo&r7&)=ZMiHZwOA%k;sp{7 z$#cElVykf3pH;i{Ag88qn_tJ;2xhdt|Slf93rJ5=Ma{Ilgvl3yIx)$3I`1XcEct zQ3|PmcJGv4n<^NlwU86pE#w`NDBE+k;B|#~_M^$jnO7nCN%lgC&OMhCk;j@&s(zbA z_1f7l=TG+J1a|KKEM(PF6q zW{Ud2go1Q$0;T3yxzhT;vhl}uhAJE%cyZt$1%;d;;4IWbqb;e8%Ic>oyRtpxf;IUN zrX30mR_{fpi6Js=ZZ*KyigW|EiHm|i32qxarS&18p@p{`v~~gd3+<}|Vm`7oi0Y2d zVVnE;RXF;?UV@468s#6o!OZ%?(|1f6Tfs;tRl`S4XzCyRZ_>)2lVEm1vY?7T315*( z$?crsxpU1!FC(LGjOEp0)aaA;&V3pX&2+{JTy2w=%Tcpe90;u)-0zs_eFZsz7>tNf zr|>tOLecF)ln?LUWQ|URu?}lh`g-a4kOj)R3i}vpP0hHw#>>uEX$rD81iqS}TUp=_ zwymmf>d{JHI6(~hMmuxAI4~OCiHFyXh0;NZ#m1uaj)-&_f5w*g!VY3Ak|vg0B3jX$ zwE1TJiWip!*F5UQZ_}Nka&_p{D;*s%A%4Y|6hPHwk`Xl4;%VDsXGakhn1 zyzGL+)-aX$L$nL#l~p60{MXQoiaZtG+U=T2_KY=1pZMn&NwZ~8$8E7--aED!=4%K7 zneG6?YP4BvKK@W5Fc~Wy z5sP9boh`JyVO3|Gic@!zEiOITvR@b7wNVA<-u~Gx@BW$W2!?g1LJ63a6HO8rkR9!Y zaJB=P7;LpY?sQ3pgzrpEA-9nz6&LtJR=xBYKvf+f-NKUgUkQi|U^O#d^rHW(-FP_u zMb_8OaYUAl1F0i=jQM%s3yDlUpx;pU||@g!N0X67&3e>X@i|%aJ&wSMQ-j@~MVDNX+r+QVTTS7-qyW9BaCY&Dft$ zsq0ArWEADA2An;);bNiCKK?n}WYB|ATv@4HaIPR>(J(0CxvxaUVJIz0@<1+0=Qi0c zU4==i@t>tVcm;eL1VCC%8GYt3ujnmAhW!&-CN$tt!s2n7uoadXI}8TP*okX;K9Xuw zm|sE6Va8t)lmC%6cVz}dGQ>wF>cvR2do+8mib)egjoE7*=#Qt5;C6E1ObqDxyE_b= zDj;IN@0N#E!ltk>iGX|X2tk?Sca>SP!wMZKphCIvZpb{SbL;PRPx;FWD z*D7!rKlw4{il90~|0wXj=kb+G>k~go(f_hS7@L&*MTOw{w+bQ1QEP3!RN1keF@*?H zg}#TsP;%pYZ4zg?xG+1ABJ5qgkN}uWfN=!wl2Ikd%wqi&)gyV|j0L~P59D;k6Z$HU zu07mS+Qw!#c$P?oYe`rN*gCK<#{Rln{OjSLa$fNpnZ2J($-#qGsIfhvv-`3?3ps*LTC9O~u1 zQP`A+M8&GECa?8s!oOL*b~*pFeC2Ru!q9`(@DXC~hUL|is*h$dx5g~HB6mf;0ky=j zYlc&U97MUbFSU?2va7eR>Y=~|tROuLlzcF|5bHQP&0?po=p^^J@A}$6Yc?cfuyJUr ziuuWb26%F_B!LKWRz_%b1x=7~Fb8<_#RbNeo|%a{u6P-t_!|Sfwv}`_OPoO2a^LRJ z47PL*amCpj-ykyqjZ_B7>-_m@kYuf;wQsW^R&=-%neVAat+a#RY0wcy)0h8o|7PBJ z|J-p!0%(AfeQWj_zKxH8PLFEd-31|_=Q?{DyxmJs#UJ*{|0H0xtMHFaKuG8W>F>>J z_|cRTMZgjV_vL2CzSi6DK70K57WMt_<5pe4AtN5+VB4y>HmM}|Gyf4+($(JpT0Nl zS97wn^F3X)JkwgZe3VRKtMM5TLDWA52-Sq`mKgB;@wm9s1BS{-Tzx$_e>hx%1h72# z^Z1WV6sllA)PY6AwN{ZGq4K1Stn|V-1yruo&nK z^sqI2%COK00c~O(UL*(KNp}YF8U#E~xR&|*thS?vwt3mJTZC=r`p)SQXisAWB{zHI z7-0qf1XBJ2aaP%H*XLjdP-B$kT4=$fFNY|u?l>nYuP0#9l3rRE7TK_K`&UgKa_=4)_U2Flwy5bjy zN_L#msu*aItf||d1H{5tPw(uCms;V21UnHB@q9tPe9|{u5O1(nqfZ#sGmOeo#08N~dEL*x^Rl9RDD!~=5GKYUdV-g!x4543<Zy^<1cWH!yb z$3~diQUpSaD^ss0BmS3XXnp0z+`8gxrr>T!G`*cGI3f{y0@1uMy*-82+uV zfr4t2^sU}i%*L7@xNIEr08HFooZuD}8AZ;cT;ZaM^!ckByxbgPMT(w{SHpJpJ`!Vu zsvfoJ4wLF8n>82;ywZZJnP(V_UIfDK9N;SA&^}GA$ME8<>B&c>X6&t%6h`X#$^iNjA4)ZnlqIgL@^RX)BmxL2lNf=x7&4$H zAwc7)j$;<6S@O`xtvnr(CF;1KdplZ`|94|M5 zCo20FNCIcUt>_P3-`Tv06)-akJU{2U-HB``sO3hyvFawe?xpQUcYbMv*^&(RCTByy z#S5@UAKj32Mz{t9Vl>J^T|)ddD(XESAmvdgo$Vet^$p_hsOs2_diSWyNy$bZ ze~sIn22f$)KumsIwG!8S;+C)~gBj~u>2hz5Yj8YwfYtti&%D)>kO2xDsUz{Aq;FMaAmrVOl#vLv#6a99PFd0!U^U&>W3Pumf$lKkL z)2k)lUFV1!q#vJlR2}F3Q0LqRtlRk90~Me}rx?LFb{qs>k=z7}fzMftp5>Jwp|)PA zC1S-{P#mJk*N>;6gL$vHl-(I|xni+8vWz$MTWFtZ43w$nCt6eCKL$!*w{@QCburL* zVDmy^)3-sPQjVz`U8dfE^s;*S_zjWkbtBMACO3iYq(W zl_BfC(#EVCl-^AgBmEJXgta(sz$N=|E)|< zo)wrsb-26g`k;R?APBf%A9F|JPI-lYd?4dfbsRkYoPfvS|DJ#^^N$5((dGz}7_3aHEmj&{HTFdL+@WL22*-f$J1@cjmb%0pmftTblxr<# zjF^x}+??}EeiF$JGO$860f$mUo8)I-9g7^Yh>x!SEj^F1$^C)9&oANjCWk4)hY$uz zJoX+U!SBsR(171~mjQ=94Y+}16jcM)MxvFWVw~ z&iDR=AI_?c+^?L~LBjYix%09Rtdoc?b+(9$GR1PmJDcytTbdC9M1JBHE71_W7@-0zTlv}4*VVif3Oq*yA7X2rebEC z!^82#3LvUbm_rIYyzZCUo{X7QQq|2RT+1ZZk^VkO3vW>~;2_Vg)%gj;k+?pe#Ua+2hpJdvJWQW`wp7TF`@G*;2kZN$g1k9?en?3RQsD>--a63Cc0K8OBcU+f`G&{z~=8 z?gYb`@PKVj7TG>|T5K?bLU@!#c>ZOM`^vmHq5y8p+N+!XV7BAr!6zTx3vXxRoHvRY z)p&NKudym{m(!B+)99rzI(8K|HSTl{qAYz6v*sfg8fD#$ftzUN`AZssi3q`QUia!^ zK(n^Yf|9%`4j(da z?QQ1Uu)m4;8g8%Yh;_Uv)*lI*X?XAUoOqH&&lLxjOi@Q!dCV5f&$M( zvFOaG*LW+sXXg#4X6K-8xlx_X>EsO;(S3S(FF=nP8Nk{&_qZK0=hQigmK1^WZ%_oj z@5O<=>504m=3z8>>Du1c19nUo<{Ny(A8PJy3mHHWLBIw~XkpBLe|q`=vVoMFIg$Vl z-SVWR?(yo8?Zdk{S(#gH1h*H?KQxXOSa%8nIY@&4s)Wdi^|VvTL8cq61)eCE1ifT- z>O`gClt z`fVHnX&NsR>BR3#7Q6IsQL^RgMW5-(`0?@@Vw$0rfU;FAY6+;X+g;_lPgXd*c!A^T zPjk<04ZB(@x%W8Z=A3w6qVRlQ2qa7ju8pGq(Mw;)uWo?g3CrcLW;vLDo!{JJpHb=| zs$f|D=#EXK^mI5?V1kqK4eMlNtEC&Xq*W6(4bf6puHmpKI<$-?%wTiy`g`M>qldRuhJ=)VRiksiFrl~8e5l;+Wq?P!ga_u-*I@AE@FF2obL7;o#`+^~v*Q$6h67UDBz3xQLt(n_oiW>yVb=w~pM&ro1>t zQh7h3T}xy7G>}KEFcM?WY_A$gtD&Qe{=8VRj6yYhgSZni^3|159CV6Gz=VaMDeX&7j z!BJ5n0{X}0g~g%FXvI%JezeC1CzEB2@7E=7kDEd!jU1wrZBOMB9nu2u$!dkxm+vt6 z^zg2E&3z#GhP(6Ls|l)!wg9r4+^LmlyPGW#ZBK8YmkRf{u|G(W(*pdy7VM4V`2tlG zoHQ<7LTC4um8Wvm=OGU9lsNXoR7Ti^)av&930z%G-P8Pfc>w5$!T~@Q!-!8ZX+}P<%FXi|BW+I640=xbnUnl<$TIVC&ixUSaS9$=#~d$e28c!eD3Kbf27l4b+6^l z>J3lYFH-1a-}=m_z*d5Ubo(eI3|DrG!e*G^iS-{uHya;RJ7mL4e`}MJoyPVp z-wtQt8)R8}{=0tJ1xqGjHn+$n1u_EzP5vR!ocygrpHBrox28{b!y71KSx>UP2vKRCgOTVkd}f!6C~`kAvwKq%@aDBwhpM1 zP(z|sufEEOF1TcBOmlB+<92!G6Ta~CDVCLUh_7mrGkRxuVjXMogi))Mt()~a!a5-< z>8Yz(R+}k7-aKFS?0dJ_+{v?CKRgEWI7R4vtv746OgV9SMR*61)K%3Y^ZQespFQvx znBofO+2z{dFe$w#-4KxytGTwmp1Js&zAz1yA#cLoiQcO1fY{_fCk%hqJ8TKB$NV1i z{SAYHQFef^b{ON<1`8-1R#e9mxPEtneYxw7*y6By1Ytf65Y%uVY^>wbh-!4EGmf{K z)u0DYknYvDxM43960AZuI;JUmJAnv)0#Ko?`YGf&-hZkL;pd%ZTDwOoMYbq1^lR;7 z_Kdc-#=C5r(6>TZ=|fWi7|RS7F`BXuqT*SNS`w-}eEwju!UmiEE_1dkz<%2!K3@`{ zt1sRECHelP0ZQbI1*XF*CsVv?+u5Nnvl!3966PZUu1{j}Y*VFKxl=wYaaWRwsm{?= zm~sJSGTE4q!Z5J=(~t|dN>jUdC{z65>hUB>rFkEMEAg0XZs*|StB?#3UelbQAug|; z0rETi;_iAxi$Y2pP7>fio`j9Jt5nX3!t31}V1)}T!{jEhUM>Dxv4KOLAg6ojd#Y}a z_|u`^+(nDhl@8{e9WC$$(oii?zii)Ue6*$VPG`*jclY~VBJ?9m5>Yx0zhyr7LPaZJ zwl?pxTAtSBW0io8!=0iOfZ zh}9>|yx-0>q!IfMSN}+jV}GgW>1pd%-6Q{RaEAPk(Xs!gpXe(EB#=sW+%0+(Lh%hE z+nxm4twdz7^YDkEY%49)2yMyQ;H-@xkWw9e%l-S#*wn6G3JR4?G9GACaf%<;%H|Fy z9XmZ;li2;D6le>GRxmmHfWxO!(^pIB6=RL;O-)PK@It*?+mI_l+n)y3AJy?Za;fk& z&Buqr&ini~tqUg&G6{ajsWV+$$@XqXEU{q?K{^$t`+hI>^dg>Wx8QIrcMt?AhAiwJ zF!U?2a;i=b+=zGDX5__OQo{`)BRU#_#}&(IDx9#nIlQe5i1lFFo!IMxv-z{1Xa4fS zNl?#K8sM-2@i_~0*x{GF^G{E}0zDvyYAo8eWq_8W7%Dk9 zFb{Bqk5P#$U}eqpKlUeoM0Wt5sgAH%96Nm}{D~bCEwDWN2w}T=*00zXbI@3mq2-tI zjvpxL$d~-4Pe$PGnsGWoSv#5$nt1Wl-K zr*-}lMoq`=1T5m6AG`px>eJ_UzXGc_A1TcDxoPohhsvb!^TrSf_~()OH5f|9i3bHxEs-l2fZ`XT70w> zuX%Xr%nS~3PjhUtdVsi53ukl44H`xiK4D%8PsTE&enriX`PGbp-X6Zs=`n5l_@?&h6m30XO@oc3|Q^~TLWs*RJAS53p!e#kQM z>{{hM6%~ggTW+I+UtgS#t*}`bp+v3z>^@>S^Pn3mB};HNkS~Zb*tqjj%|yDOTjB~| zxHmIcfKr&t@Z{hw&k1`s>6ZGholT7pOwcOj%;wqt}I&FPQiTI(f zWyCrTLOmM-Zm=vJ9=1K zz+Ll?X%6Kg+1~7f3E@BBQnw~XriewXPE=K#8b6r@KR#P7N_`b9nFm{;qKWeI1`Pm` zl#&pPX>B|{M$Q%=On1Lyk|pUTCNv&ak!lgzQb^ns%5L);YVS|KML*~{3Igt<{7>1g zif@?z=?r4;2|&-|!@Kg2k5@^36ZbnvLns-t01xKU8$BchthClMuervPVrpSOC`qom zl4TsQW&<2lemL~UDLu%2!RLJId^uYeHZ3`VAgeTBP(@glaJ|86D7pRP`$&0^ZDJ%=)DqKL+ zv8+S-co2Y45M{$*!t9yI4Z*|xiMZE}q4%=iJ!6X6Y;dyqyBu~7JkpNv8cgeo@;icD zQXzM^D1pmp0-%*0x8Oo+o$=J{FSG9!2X27u zfagUJC?zf2x*qJ==8_=TtqM4>*b4ls6Nl+T;qCu8PmprHuI;u=s;f)bY+Pb;)8(tX>?>WmpGUjms+5A%lA>@UnH@Bm;xc%u>(^-P%{x$zwA z<5nxDB>o_EyzREfaM*9pM6)>(x45S2aheA#E^rHAm1V__g1`_tMM6m9H=&3mhGvAM zVhtBVTOQ1CD3HcBrs5OEi&POCq|=g4UJ0C2PU1W-M<{$3nloegzXK};ub5HjVJEt4 zz~>cX*F1rB<*OLgXTj(+CA=T+q!wq)p~rxDCWeJFn#ZYOHwGPaH|Bs**Xi?tff~yn z4$Uv+wPnSt#XH8cp=b{0Gg=H#7rE;39_Wqu>ZP^R?zE6mgw!*@7Tufi z5Z|EE4e)|UW>nx3&-i+e-Ys~yq4vjt!c5CH08y1rhW9^Mnj`KZs;`xJ|CK}%_zM!r zg)s3{T#l$%XKY{8+3oZ-N}$xzM2TyGQX(i*51?Hv2ufBeb8m)+NzH%nfYm1P_zf>Hx=1tzXN&;BwYrbR&4>+w2Mr2i#$ z#kh0jH(%+GjHaqY@N5Tza>XkE@1*j<+%lx{J3Az7NbVC3eCeWp=~dtEgh>4M1SM!- zUfzV^l7lYWDfZwwS-tw z$SXW<-9~gP9Ld(6e{QNKtXArPerbOIiru36c=x1816QBbtT@JhhBAw%#9yXeE4oyN z0u9%@^~{~pSP5X{He``ECZ+V%;^r}_T!>2QO%GsNm3pjBz%&Ffp;?ZX(u0MGcd}j5o8RtUEeG?d)MFsyCxYBJ8CJYgA;mG5P7Kp2T(FeXTyY@KY*+o6*?1Lxp$hOrhI+u5T)?Qq;L1tOe0_RU6ZT@f!}BFEb|Ps?@60>q z@^c~g_qZr$?AVT|>dLrljLs0RuN;IIT-yUIp&9I@SW=;$;0P%2_3Uq{10sLh$%&|_ zLIMH;ZV#xaIF~*p-uKzf9b-bw;*S|?fAoO(56K!=`EQc7^NUFcncvpVmoteke!Dvj zSSsfs0c}l&`Nyrm=56_rK05>adNM{(%y@`&Kgn!8+)$Kwl{yzZ+usc0ZxcpB}XnsxmzUG ztMjpxa*lCJ!-AebHA}08^*#yB6`}fpIUf^Be3{m4~%aY0ov`X7IsM7MZ4sD52oF_^7c^hwbc(>1fcY`br}B* zn zF|9k;tg4!P;v4)xumh)!(=!JD*GGNVFC*_x8o$xEP5$Xd49zZ6(Efvu0vJ8eOYn&0 z2vId3CwC<061F3t_3$w_yRF&mf76VJeLgSDk}KQyqO#)>RzthA>smM98`cRq4AMto z5P6z~dW1`dw6_2K3VNj5X=(JEHR)Zi(~no2e`(tsS5!mFf=jG!hkg_F-0pkU#i-vY zd20dRyA`B%(iZ=Jz>Z)p;;8wa%!<^Nilj9g6JqfcM2!C*xDcVL`oV!7@AZ$Cb^QZy zyOtfQR-?^XW6*{*;QNx(oYu!f0G$YUy``*G6DzeA$6@;d7X!IJ(Od||Ndz708WI;2 zu)RE)fiEJLy{@iJ*EWhaue$0u@Cz?iM&-ahu5>O_5;NRnGjeZ__}j;`%?VTLsZLd8 z*$uO2#Ll{E(ocSw%E{kt#ah#YUdz5N!#oTEql|u%E%)8YPlif6VA^`%B2B4=3$zUH zzu+Q;k!ad2K-yGlcY2jlXu{`8`34t)-}x32)as@u``95P!QK~*DX8sFlgM?a%}9cJ zR`6)j$;Kvv56(ZWcQ0QSXW8wQF`iLNtc1_)QtU)t898?wbT9PuvdX56?asWCW<*F& z7rONwkM{r4pAc;dzS;m^uNvcZtAWL+LRyoNKOGxj=tiS-CVo3_&+UQ{b$jbfLb454 zG+4-P&@qqCFt^ToP4|RlKsM@s()xu$OBsCt6P!Lk?MCfZPhW!MI3?-~d0X$K3N7)p zOVZu!6KmX-==3zd{+#;oX9g&Pyb|GhN9=HoOR)L9q8Nxl5f^SF+VKH5M9&9Hv__@r z5w_hVY4ujsUT8+S7dD*d_0$WpYc08|rzZ6kxGQWYR zfuKt#-+I7JM>*^*Zg0jjLTGkIxxc~>=;`3xWjO-W}=Yb8I55O{@cHk;}F zmi{XcIjua-*QHU4CDp;-fzXvIN1{WWxnqjOi_P#IA=EO+Z7J516F*Fa~LyWuu;NLf`p zVDtsfJS{Uz8^4amcq4QYva*&3+f@D{3eibKY;!*6dFwnSn=*BI&!v}i?i}^^Fr~1= zFd#vq4;nO%nl4&}Qj^Rx&cuqqSHhR(l1yY;jSq#9 z7l6bmGVmh;n|H4H7vu-WLq$v!ONz3XCc~1V$Q#YCt*eoTA0kY4l$syB+~A1Gxz63= zT@?`{QjsSolqr|YhoU6bm=&qRAgpuzOTjCsFIoP8gOi}j4dChTeie9cTJLm4Q>xh# zT7xZBrwas=oWq*o216>;Hxb8S;d6>Kjg^rHCHS--!U7!r{?*Qq~RK)+vTAne3;WvVm z)#bHHtUj>n4Mq7a$Ek39zyVq`mTUOjL^L70WFMW88jEa30!m0g)t8gn4!8psy1upG zUXGUZ#h4!FVIf_7f^CF4?c|fMv~uaXPXj+EYyxgV zKqKVQFWKAOn&~+^U$ZIqxtIN{Mcg{v*7YCn6kVblzc=K)FYWvZ@vXg7K~V&*fek_; zBtk&8b5&Z5b%<-$)n87gKCj|-B!DEZz%G>vl9vc}d!KOOtlbY>G$|FRt0{sB>@l;d z7-MGq>3?o(5Wo8@?r7s=3iYkV0HP)O8}t9lI>ijCWXXZb9B38Fn92eDwh!{D{1&hdTrmbo{Eoj~gZve5mvHrV{Fe8n% zf&t;lh7`v7pxg~UyQdUCLvCDabhAz?{VCbgnq-vLM7{%lWRII{p$Ch@B<&r>_=kXT zlM9KSOGp1kd{ER%;&<)Q| zKRhuf?4U=Q>G6x6!_T(~dH_67pRLh@5Z+3dl8#jOOX;llL3$*E*qG=UL?65eag8T7 zd!e$5_{f*`QEtcy@|V#k?#ZbtRW1+S$49i)1)b4N$bJdAwb-&VBR}L@t2ILI{<0_g zn52v$S>rgqI=G^!%_=A{OSL4z62%`=aQMoWa+BWu+f#RL|9B#W0 zxzjowd|>)L!6S~*k>LFIY%0Xi9GsrC8&5w~VupJ!*w}a9y6t`aFHSK^@TTB$qligAoYo~I$M56|7X`G1!sBEUs2b8;J&%? z{=$9Rw=GihSh`;--`-pCe{er8(hQNq4#!(Uh~~Q`sP>*t;}H94WbB>RsP9@v8I zLYx5+M#rAUxNVBr||1I~!2e2!gp#019*Tbv$4lpn9kLls2tceK? zK^t7p-JdVd|L{Ntv5`<2gS=LMBxR%qE}qn-bVV|o7o&em%B~$BKbzSfOe435ly(_S zD=FF?T1h$_ypE!5TMwT?6Z~Y+Q#wtoYaDlDCU1fu1_BCR43&P?Xr9F%bj{G_g#ly? z1yzK0k(l#K%=ggA^SgmTvh^R0(-HBCY$uzYgIltcm2$7J_psvb+mw zQ1gk{ICUt3IJ#e+!#VUzZTMs|@Yjr4K3eTB=zjkb*W>PPsR$jx5TD^AxhOvbM{-ER{lJuwr&@2s^So55_JKJrXjsrsb2gi}{mme%&-YYnF+TX*xcd z5Qu$T{R-vYrnBt{%{862f6C49>Qn~*HM1>q{vBNUfAK^_y=k0Ol8`~aql)9X03wZ@Kw57x_l^3C(3#(@k=;O z40FPYn%r0M5dxr9N>?9s&~3?A!22Oh@By>d{$Ohpt=9jX7%u1kRbn_wo9obb$Q6$T z|Cqhuv~QZLV&j+C&FRpg+&h8hau6 z+iWbP5IyGV>QAe^d&~_w_3>du#GbUV@A2FwE(SyZx!(r3soKJ_=N=ogUX|MQE5Wr5 zMIU3fF(N7^r06}3DYb6WZ3R<@i?u&y=@upoWXJwBOShnN(h$j+dw-jv!>Ay^@dC?= zVt69oCwPZpf6C__?dp8|+@VtFNi|@l!CV1Lo*o16Dv3Ts$_q}N#3g6PJc5jvwStKL zWRP%E{$-HtEv1Xv-@38=*jitIE-2W8nI4mEW)VW3#WPWm3VeX{78~g`(L%4cp#?hM zVYAgJqvZy_6k6GJ#346TIF;wLbVBrp*IacvpJDNuJE$ppL_CsHli}D;mkDdyC0A8f@n|!VV_c7 zUB5~;`ZeTy(s{|jg0G9sYRkst8x&4k!pnSgC*UAh3#qAT`LW!K$9MgFKrq9*!GPX< zavUPW37@d!gPLOT41^^YThQ`n%Q?g+3Y8^j3;YmXEXSL-Kl+zl_{oYwYo9o?9vist zOjvzyWWpG4iACnby2@EuE=ct;74O(*sDtDwDaVJ5Red*26PPaeO3 z&<64EE|fr;?Fw~xt>I~oU`NO*d?F%+l)>Zc-B4Ljzzl5jq8dczl(EJatc>YuSp zYwL7yGv`s^-Cx+~3Q6|vF1OUC6weQLA?tMBp`Hklan}3&F<=UADOL5Bd>j*#U`uYw z>byL-vGj^SB{z^Q<;*TmITw~V-fD#vE`YWQ)Iy!?JGK~6|8W`qznd?EMb7{C7$r=z zfa6D|Um#D*c!48k93xAB%Tn&{b%Wip`HHQ8Ih@H%>~6Jb!I$7o6{t}E{% z4QqBMGq8hD`R-JR?gr;AbcYb2!Z$~+j8y>U(UtcX36bjj&!JUc(Df%iNr(=+Jf+vt ztoef}qXxI(fV^vo8t+;{T~gP`v=Cqc?pn8tBkl8}X3H>tVsyQf%W?S*WZyGoQFFqb zgMED60t;L|#fEYN9XgIW$r}K!oVF^^t<=Bo6`tzBJu#cb)>u>L``m>gg#RtbfH8!F zPI9_B>f>7hVLb6rLlYGzp`yYl#LI;;h}P&0ScXD(^-pr2%$LBP7CAeF>CwDA`B1|^ zjw$G;?Pa-pYDARAixOz%Z)kgb{cIcfWb#UjXGQgncSB0w?^Ia!6=)W@R2MgHTv(9> ztGB_CAEVhY3B@aY6aX#~H-&!@(Cw#WovA>k)J`tH`exLRrko(KS*7&wYO>JX`xFKk);U2AXtV;kf_s zMIN_7z;+f%5gVxan_<6%FUT~oIpak)jIPuVtqOah7Tlk%Vraqe`bfLxEnm$aywkJZ zO~C;wOrDA;qxqgc=qs{X#$+0`AyF>Q#$nqp5~Z#Vg@oVO0xzE->A zfLIW!3XYiZe#4AHLkT;f_OOq1LgGyoDp&b@Ef|hlL&pfMoBfelcvTx-Z^Up%2B0F- zhCB>1L**5L{BmG7xZ@Oh&h@tB4c+d>@oOieCinYj*x?vygM&;=RTm;{)|*`FO)hUD zJmaoXDlaw%|Grg1jHy{n%L5?vTH5wu)(PjNXze+iXYq{R4`RWg;`(}egwKtkq0)7v zxz80iwQjFFulg=H=xUek<(ZvtpL}c~6(+Wa`~sHYsDCDU1-Gcwki`6Lx*%A$&59J0 zqLYqKjgF$oZgoat*D}6Wlj3-%4<01zd5)bbK%cEi#|T*Y53D_V_45`~r@UBd zmj>GuBra*jy#{13v`o=o->8RH^?qD59eLa&u1dP5TKLK%TBjqAgbX~pez52(LFs02l}rpwtUe(jkl|Ms ztvN4R@qySHx%z{(Le(h^P|RGYkHB3Png944OA)}kX6PyELaVpLOhbHOoo`{BemT3FcD?o@Fafb&xx!z|J= z{gd;fHK0*JRn@s$a==i(U8JJ386Bwl0<`J=74T;Xl&Isa2Mp<`eHZ&&r2Vm!K_~$IhWBRL}l2qx{ZNsXAT?2@xqeZf;gJ zl!!z$j^yU~M9PXNgvOp!F68Ed`+{4tm&SP1t=Gl=eI$Ya1ff5|dK=-2D(Y|F{`i7C zkcSZP`-{K*O{Y){;0HEg9Bi?!@x`L{?~LS2QKpnk*eq~B)KSQ@qT-r0p%jW{Deh-z z$+ph8#duJDf2Uw{@%?VIHM=Rx=@@m=RKWz=zQ~+h!wbS;tvckBCm)>Cr`Qs{(eUhQHm#e)LO9Hp6)auSBP@$qg_8*OkXEU#1U%9jfXAss7xm9Qs0oxaDIM>h)`qYgxi02Q20(# z+GK0scCheCZb5gpo;Yi15#wvK&@tH^OsvWsj_O`t|{Ethp9uPzI zl2PHb9d@^|Crfzqm59TEMycXjGLb9p-pR}07y~^p!og@jnQwGaI_X9EU51@fzOE9^ zLSYUATd7oL*ropawA9NFc!Qzk>>Kkt@1&ISuBYlxf*vw3HX4P-PP7fRd0n(R22*a` z-Ci~IvPq^$GMEqh^U0H#%hLooMP%{Q8R4yumFS6zkSAZ>v^+Gw#Un3PQmDNV2fmT+ zV;q5{7&m*}>N8|RXITtRMZJIRXf$HQ2Mlsk;dTUbpUq_^`!0Hr_HHHb;?8yf&)p;8 z)T$t_Zm`^4zTXuawsy2Cae66vg&fYHc1-HC2dCO{cp1$C{*jU+U@~;E-^}IbBk{V> zhB@h;*Gy$@;2y*56SljY|4(gK9TkQ1?Nvk+5kv%}mF|wEK}tH7SXyckq#G$EBo-E= zSz=W>B$jTF4(XDvr50E^78ai0=l8wyJMaHDf6ke6=bV{)KXd1EZg5L=sO`WgJ1&{t z5s{?lHQ{Us!gpmy6v>M=y#%h{1Nop)TtO zcbmM?D_aXWqV*o~yBYH6+tXZhN}z;bsd`>>&*|e$fVWh}91;9j!pz=pR#cCne>h}0 zUt`y|@4NpItGX+m7$Lj|b1LA|sMDTL5_^~r5&^$NH=mmw@xM)^lGU!b4&aSDuzj=0 z`ixF)V(;Ofc&57JmwZ3$>+p!^c2pM!sgJ8z?7^QYZl4{G#1-iRYX%5fXr_bl8aH{t zyAEDUK}i)$4M#yXz|U@Xi@Y+sFq3O`-^9Alob&nJW(<6kBny`elVh{u9SJ6K7cB=5 z9?XLT!h8dX<8{@Ug1QZYXE=7R{n?Gbo7r^so`Gz2Wf5G-TjG&?B*d3-n^;uY^((q* zxxWV#dO^elwu;U^51xL~S@V$#XBwAE2~E@O2U|-R<=R=DKuhA*ss}B5AH0+jH#qFN zrPPv82sg6PPH;*{u9zH$e_eig&tLMwaSuez6-~t|sjnY@R(TUpL{{+Tb}~Hr>XD`T zZPX+3|2F!7^=?PvggE3BeiQ-fkqiOI43McioURXjRqu^sM* zeMBJD*&Yyt$k9w%cc|~5jV#0*pNhIfbcEkKm~V+zHhr(v=`$47ucSxNLquKyM$!3P z;kmK)!X^Lo0n$!)%WKxKpBau|y1q_lx%t#iEmoI8BV9Mv8u=^XnCpzp{EE(UF2YE{ zceM8950hrms#6A?W^6ILEGn{fd95+5&#`2j5i2+&u4!pq`NB&htMfC*wzab7sEIhK zA}G~W)QdD7x?tyG%K+y&Vd1xdN{(A0@6+5W_Iy=;^F1$*Uqp%IHZDB&BxqDO-%ncU z0M^4;yYy)0R@^CpZ25;LrjC7`D>+gxCU1;uL!zK5}Wzu=u-cUfBS30A{fkR<5lc(FWd=?_tsTC$wL z0`|;q9-GE<+29k;Q-j)16MTnz?o5Zdg_E%g=?jO_b#!GvR8uj({e`!N7o5-WyQ>7d zKL^h!e`S+{w7Y}OucYqbU}u`oR3Ee&F)MIxdnf5NT3B9{_&ps&paZmO_Xg&YbTZW! z8BTeJr#c3$J$VwPyww6)9{*J+Gr~yhw>qnm7h@q=BsK0_XOwd+C+nHsi=#Y}FUlDw z0Iv(4dWLh^Y>(`OI%jqDws*KLxTRyVTUH>YHZkJ-*^48WYJm*2!@gENt+z6t1Nak= zb3|^Ub`jCvt=QsL0yEOe*A#yih??f$31pn28i>HYAcxgfKW0&ny~1||E9>$};w?VZ z8egC~ASrefvj=LD3}S@>@p^K0gcI0IDHMP~S~vXQ$J%aGJ?4y&Jw+zT~YblYy6XJrR}PN3Vl~O{Ihx?Sv|y#A)McT z-aoK)I)WIo@M4nn@Cwo3_vr#ls?-Jkr5soZ0){A_a@CW#MI5ua`@#AJU90j__4JGc z3p0ww2@f+2`99uK31$ji-ELF_f9egRBHUObq?AaxmU&m@Y7?&(nuQ#65mA~uKYImm-A&8O1Au%!bC%>-2dbR%9{L(od-aZHG*2%gq4e)sg zzNMjFMh!^{f3_;$>ts#KKRrAhBxqv{`$JRisjR93i53Z#J{5l}n-H6F^y|vYs1oxE3 z?16)#aU9J!tQ;O6vlfQ?*3p6)QGVZc>#cT2ZOs|AqYJ2&w& zk5nrbO&7u56w^7JI@oof%T$jJ^BQ)(&UV+!j_zQX3EQA*Fp9>cA?K^}-)_kG<*UeF z_&kM4#E-Nk{;6yy3a9b8z$1P}`Fcc(=w1hKmvbLqmiC64P`s}%`>um|`MsrZq@C7M z>K(ps{#7E-)k^7zm$wTgsntFFy-%)2a8iuXUrpPgghrO- zUyv2YA88|E(rAQsm;Z#!7ovv+dIM<zGdA{qwMPXfka*#peH@3TeluFuO|55dqJ z_q9n}E>?TH$Xp9Hd)f9*`}8+<{@e6*F4r$^u3}NBHq)Q6p{pyzKSRc9bh#bo6?rOW zw3BvvztScFD&6jtO8dDUd}Tr9X}(4Nj`Emx2J-8@0#7+f$X#7;5%loLOj7E(BU1>6 zjm=8J(>@sPSTsDBNFkNPk7R()p z2k;Cr+|wVliYb?IKIryas1;TyYrR2+p?5<9qv>&D-by{0HG^iobq0lxK`Ht2V z5#BrMoE}b>7KV0I#6{i1trQ_|J{!`zfof})r0SiSL1LR_WJaFg118#~R!oh#YR_+E zwH#<>tA(2Mi@(dSfpvqZqs?7*`kD#m6hj@r+hgVrBYVw;^&OW;H<>FgN|)M^9^Ogr zJ^c}9vd_<}!Kvhhs#jBdyK`kLF@)8u4sS{nw}X<|6Q1@c*M4fkdP$RElRn&s6}_=j zF}AO7au(m0?E$!Nu$m2QB5ri~X1zWn<+$32=6%E^+jxq7*&S-FG?n!OYC{vEVrgh) z5MsYW6-v{b1^B=%2NuNmK%{D}4+J<{(kU`C4Tq~f%h%v1tpryaVd0YnlN2Tauo=E$ z{czZZ$iqi1uaDaHF9xXbTcEx%v)vc{OtX0NjYJAjWO_F;eGEs6l7kk$)wFr4eeQ;R z?kqt!ONxK(`AdLZ18!QhmcR+3YWl)c$NaR0}TY?^mG#>(B+D1a@nsT^Q#{lVW!p05lvu(aH!wX4&l=KkI=v-V5o^wP_V3(dDVc$z6WbG6 z{gjU>#itl<%IQ9xBlIZKMzo5@(`OX+Y&LKFesdfDYc&=s2r_o^?9rlFv@h z0b9%KTh!?5ig}nFJGu6^IN1ylyzCvURaDxYg;82Xy2Kvd^mW-wisa;~87+c8beMyy zMCx6EN}rxFl@D&qx2@s5*jPqzg41gF_{Ny$En`-w@R*g2dlL6p243hs8^HLC?5JvH z#u@gjN#u+8M-9ZiBH-iXCYgzwc#)F>XsPP#dH*DxJ|u|Hc3Ipx5_|`OUL4!j zr6&F!S+HH&Shxl@KVE=|jWt9@$?ad&T}{}f`;ML)P!qMD^_)y+GOSxcg>eiW|ZjD#&z@T65KkoMB9CV=5!*tST?OEc;a;~!V(GhkwL2D zu8@bBq zuFM7ekaMl6w@tzT*fREBx1~_eX;$#Ir2-w#`Il0TB{nI4O@DP8- zpfX>j@Moc-ZlS91*H1s7B;PC^^+kKI)s;FxT7OA<7q885Mm78Ke2tL~Ng0-dwStEi z(rcfQ=wm$Rrs!mlf?0;#G_D@I5d$TzRSX|bTeAe5++n81x38xy0pky9@-%*7Jg9|; zH=|ar8%{?O-s|c}snm>`+1$1qgQ|Aq2Tl~7w&@?8JVCmKd{Uzwd(_h%W4Cxj+MiQt zm-l>+1@(`~H_Yu{*CFw}&^&J!54fwGrUX|&HtLd_jH1uY)4HJhQ?&Yy%_=cO8%Ms< zECCB!ZGHkRt06lEziaw4=QMG(HWF*4**VPv5|R}(*XZH^c5jWscIwOf-RT-PJbn2^ zN8Ixu1Z`Luf>S(ATTpoSMB#WSv?HIt+=&2h9w$P_NJ@vueB?taF>)9=2p8`^XZRN( z>T~ut6t61QA*UJK3z>G1flyF+i}u=TMF0;~NJjR3)px8GHm%S$-QLZeY^e?clmTU&|;O6`3 z6^|lgv0oLd4~@mct1a|BvKr#TFpi5w=ECHq&3B%NRpqWPE&62iT3={*4Ji4gsRN~~ ze)$?>G_IhNs42uM>eEC7pgoequGOF@$NzamtC~w2?*2k!Z~OAVHq>mqiwv10MUvL~ zj!ce9vEn`EA^0jUjaS7lY_It2z`TR_UpKlPcU?a^8T`f`FVo{L1W zQgBy!2c&$-$|1)^9=OnUcM?TFH6O=v9LO^2=7{q6lOu3H_k)_n=n)drFprxKc}3mq zpz`{4d*Jh~ufZT@SO?@Y-`L(m{mky8$uYE{L?s8)X^)s2+ia-2M!djblJBlO$@MRn z?h?YDmZeSqN*Bt2q72)~R3gI{CDvOOlqz%Z;OZ^uiASd`Pp-Y#nyZRn@ux;wJq@g+ zMz#6cU^yoDgflv?2Ixxq9rwkIy3onu*0o9~_`biugK1Oc!`#q`4srx7xHIuK!TM~T z&qn%x@F=RkymTE5S+lckQ5s?j+dd`hS-PK*Y8p~_;G=M`4|(bVL!43nfdwd zoF2cm+?5cqr!rDE>GRKjKZuf!=b})HEk(F|y$wcF7&_sh&p~fM!aogxRsgCg(G`)A zv@Vw1WJv=ad&xoBOLYCeTHotj&I_PQcocxGnit!dRqR~s|(;au}PUf%(n**flyYAZuK(2=+&g&F)r z(p{_v;dmri1CG-bDw~M9VNhH%VTsQ)FGJ1RTd7_>l}SdmK~9i`@A$lod4OOJBbY F{x6MO+V21W literal 0 HcmV?d00001 diff --git a/hooks/screenshots/mv_command.png b/hooks/screenshots/mv_command.png new file mode 100644 index 0000000000000000000000000000000000000000..89bf88badad90a8e3cdf4354d4a99e19d4a8a60f GIT binary patch literal 8321 zcma)?Ra6^J^yevUaVSt60>#~3io1Jp4el0P3Ir)qAQ0TOK%uxpA-HRCC{iF4x8Q7l zyZ?RKm;IbGa_8Q8xO3)VX1)`vsjh&HL56{ZgoLf6D65TxglzP${{ao`<0irf;3XiBnA9Gr=66V6cSS1SIkGtxBqN(cSS=lBqV(A|H`Wk zTPnYQgQVVa2HrYuw%)#$o;FB!u5K|WNMHa4za_HN#%uY1Iikf^qlWTkZd%#L$W z{dDCP!#39^Tm`y5rQkY#xWa|YwKEljX+LUaT}9HcR4~zi&Sff<@k*8?wM5>e(tJdt zVK1bk&LxRF%x*Wh8CZ zcF08CgSZmDm zrNhO!cOocje{in4Ye_C!QU;3qS~~Q=oizwCbXun!)|`=}Kt+WBI#Y?kDORG(>vN$K zs4h&T8cS{tDP&D3?;ZmNmTshT%^iiD=F^zJXxqxfZ8iA&cxwb5Q48<-3WT)BoIVT8 z%Y7vwZF~L9IcafJ9c=I{YOEP;K9ohFAOxzl`6615A=R z>LTN}hhT`!opm57Vhx`Iy19(}Roy)1 z+P5^&mtj00_BrI*y;;9-M4geJI>owb@1nO`sL}J2e9gw>_^!WqppQfwICI`^e$*{v ztNm+c@y32e$Qtg}spn~(^X<7@&BW1I_zpPFSe!?+eZq90j}dqj8c0J$p)PPSQ*=KL z>KX)xi|~1Uf)M&7b~zE%0T;2QiU)^(90}y-6TOQxNA|bf@`%@^rl!_X>qj-h|Hohk z2)J=We|t;;^BUtwXJewH6d%QWWRWuS<@?(m{6&ZJGGZ%@BMG3O6JI{ncfqEGAbaa#i@I+{&tV`h;z55u- z-~ww?s8eLrnNP7l2wtkdrO93VS5)+N9ARfh(w34G-Sz5aE|?k_&$ltv#e8pwp4NSF z(97e+QQV|4_irhJfS&Hz+~Z_T7-p_FSEfb|Ge^+8{g*^j6m#N+j|W#8sad+5f~)qM ztyn#mT}3e=ZF5CMMcsG71$u@^da8P!+6DCgQvVOFs4R@e61zsa|G^QRc{=64%p8D3 zd{kl)Z7^dCt5#j7Vdn#iAE^txB1JqVDM@}zccF*GrO52>5ge+1ZrtWCKVqpuH+~vF zIe$Bdjf$p}8tK}1!(FM1oggP_u+S!6bF8#GUd00lWkv=l!@!}K?>$B1>8vh76+yLf z4SGfp<66o#+o(l;DLD}WJt$Vfr@(E6ct)!g@4hnC4~_<*vlLZrt`!L&y?zNpDB5nG z(tIx2XSgi)R2V?{c^gHLGZYUZEd*NEzO!z|CQ_Bss^=pDl zs=TWrJ5_liHOp29?5?}+MX};r{qS=plAsV7Zm&~g#bFv>v!7|gR6C}`d0mqrvS}^x zLK~uG8gI*>vw}sgrcu>naChO&kche4(?bqWc9675A6!B&44P-{(3bm_bAn6ao>n_Z zx26AG|IpnzX;@`y6L@}v+np1ffOvVYVy>a)U^C)L$|VrV?^R1yii?~@s@^ap7( zSqPZ(ceyh+xS^bA2q}|0{fNtRK_kN z-yS?Jj)y4r1OF{qJ9n%Gh)(bUA7`~?yV+U-6uK#!X4s{p=G1LUc&P!UM``mBS8hT{ z3nuNkM@&$~h1Jj~4B zfO~0?hMo-+D?SOf8yN;7D|+IE%tzW8LTq>%xsQ30-yMPWsUr2ThD93eY+Dai*eZ{w4Ahh_6j^Rp?UOtFoHnTIMjSyAP!K# zQ<9SC%W@CEE;(-Cv)cR}h*q8Y>onpAm1eXEIpakt+#p9h!3KzRSb@e?6IlRFtR+(G zrMIII2*maHnd3;+ZvNf?xP2i@RtY4xieO`|@GQ`rzJ*KL_6UqIy*<$l_fE(Ag#MI>@ssd%Q&b3qFWXjW8>8^1!PvAZK zZB8L~KF{@7&)B2)7o4fdmJEH3fkFs$=_;mhLXC4n%pNqA$;&gx79~z4^Hk-tAzC0xh#D;j(jO{b~mq&>1y0!(p6Ggb8bMa%S>eJ+E)pBJJWN7LS{?$1~c^zE%8* zEDO__^8_H&yx#P98trbRUPm*3Jz}iw4bbyl%AmMHrQR~SYV`KDKqpQlaay@2O_z$V zHhF7`8^dm1euUD69z`=?P^|Xsbt%)QXsgjmP5FrX1HsmOHSW$DE?; z9;X%c3C6;`oL2x!ML-}f%XDW7K3$Kewm5Ctc22=fqGn%){8of zG%K<+7X|#AD2sp}ZbzZmS;B%*F+TT=m--cEL@UQU_^&AY2~;N%C2kU3h;LlRp;W2{ zmq=p7wglky0bZs=y8IxiA2z`|)7tv2^{I=MdBny?hmu8PA_)EXOIl7Rqfol#uQTQ{ zEWNtG0f8FZRC5`*Iju*;I* zXl={wV{b<$dZ37P7m@#o!}4jw@0o}tkZe#x*^_I)u-7agP zNhaGJT)fG-)!;S-(PpRB*`9S3dav-VNGGmcsIm!2ui)XjBWCL+Kv8B4#u-?OQjLtC z4B;9>bgD7JWTgC2_tpYkGnI_@jtmCS7V9seP|;OnIO62IC||LPnPAeJXy;fCNlajr+eJnu-1sav(D6s z-nENLF&R2ws0K5_zmpXth{nO5k6T>`fe6Y$0h);q1vDLFxZ9Ks3O=q9HLDRnlzet4 z=cL6c<-cD3S^w#A86YNjFh}IPGj@~ty!^ee-^X3re+#L`IHV6QjUt?zPhFUqCMBOC zMt)5%PqORjjkomJjk#zZ(tCH{&t^UMfZ4D^x^%Ya?APL5?$GxQ|eD7EkD zDS|~;_ZcS+*0cTmglIh$3p*}FOZ`Kp{8ofOM<2{X?%r)sErS>1m`(!tV&73wkA?PH z=2F1tN27PS4{ytCh-GHrzH?7o)Oc{s8qss}G4-3Ffj(iDSq)+QhJjpT ziPjX7(t&mjF&tMEyYc1^ArXvQf7g}1uBEXI249YEuIz-&U7a z$bcv0)TdB%ptyfe0m6LiNoO`*H#71peYR}jC`M&5`# zkKKdS&q3ZRv3Ru!WofKB47Y zQyN=kaoxb)V)<~Muwg#h)$Cq+T%)gE;qcmajv-pexI;#W$1hqcqT;zdk5*CZw=XRT zobyJ!VZfr6eJF(J+*Sh{RXlf~m3WZrZUJG?bWGPW+O=9x;BmU)$4JF+P(72%$|io; z>>nS+Eb1@nj;~|OY2}+m^=hogqTPJN%Y-K|G>`OIh=^G5V%eCJ*QSrItmmART)m$p zoSUDWp%(b3L_X5ZKmX^sj=>N9Wc@h_yREsJQkNd>U7>6LXdhN&`|^*If>hn|7C%)t z?jKQaOqn>U906bIu(^5KQ$K6~1TuPM1{)4fF>FV}I@3g+Rt5;AZ_@lO)w>Zr~f+&p;DN@C?X%H&``Ft z3HW&Wx{Wsz+eK>)P0ln=5ZG@?NeVW+wK)Qv{uEeFY#a(2ZLCFFm{2i@T$cvW4j;Bm zosiH)4vUn6*(>Kd+{Jvc8;lE=jyA!jMc59{@^=RpTy2ne9z#W6ZOznmUv9*fI=Rz0 z&P({tre(!Wah(u(WN=3OQ5iPVbYOa4ZYqbkJ?*=#o4isZM~2im|`gy=3(nr$Z*@}v00}W zGc56G74q}*(-F37d)3C&_aA8Lt`0Pgh3#eis5hSaMYY&qWnA&W6eTyjQ%Hvg1{Oen zZDg5LK&}Lg=Uts%93hvn`>lr>E^fYU+?$77>2y8wu_Oyyqi~|K_Wt z1HrSE*Uz9=LUkE|V#d+dUUW8LM9L-m$9a!m@8r9PaF}{dn}K#gF*EGTWRlt03Dp~t zw(%};a(8vm>1R4uv-V-CA&%q`K;Mgqg64ESP-t;L8q^*D_G`ywHKfys=74Oc+81c^*@()5=+R@&Ea0HUbdh^St zJkLjMIC8|tCtZf{tj+_u9i;O&^5_5<+a~_Lr8}Hns>6)8OB~$+E&}G!{+BtLRMFZ2 zQ-AIY&C&07_(Jm@Aae>4LWE|ind602@AG{3`^$OnfPbkTec%*5p$%bwq6r%V4?S`c zXxQ7ny@l%Q8^9r@Z_5U?GU^G7%Mq}PD;kE0+niGb#ojJlhhn)S?PF*;@uXpqj{(>v zCS}GJmkLkaPnJN5P<_6QP67O$2%_B0kw2fo)dvGlp)qUUYVxa9YXfnDBbx&~yw*)N z?rkL)!y|+}4mhn69y$dMO~6lL^&uYlXbeu$RZ3&GdsW|V>7DJ?H*Oye?K2HgB0B8e zdoIPQ&`=U9$2I;*F9t>GyOW!xI}NmucijQw$UCerN%?5LFq$YkmZ{a*3{;|YUbMaY z%-VS?Ww7i^MKZ#5>e*PVvirN;yzN*pjW9c%w=lsnEZs0=!9}R0OpVcMoC|;DA<5~x z{LXS_Yx!!5-O)6%Nctdf1!(2*eLOejVww(JD*g9?dd+Su)`@pwz_Rze7Bl^w)2Rhf z8j3o{!thXnNhz~oFati35G;wotD}(Fz^Hh}&K0g)Q;Oqrz^_U<+yGY6HGmsW=PcFj z#P}CsI&6UKSv57Ay!|$ZM@~et&Y0)E%7HR8;tq)*~L-CP!<3ky`iyV#J5 zIYzwwX40h_+#}(!tXtR~0wv{L*laN>??$Gif*pNxZoph!rJJ0CPxuB)xN^I%MgadW(VTGkr~6qh_y^I^Rr4y8l@g|+2H?^%u^JNtcJ@|r{La+x zVSH12R}TguA{%RSGPEZK`7#m+sNwD~_oTlzV{q;J;;5J%WbAw)zg|NiCQUTNW+L#j z*y;#*9L8E-i33%@FQr}0Y5&noMiodXWT$*blixwVQasIaw&HEKt-jdvq?vl2$iTNg z?<7QkVSzhn0Pc#&{4MCk8vT`GD@Gi!(HH-V9z1NCHPh(G#;%2cs7+NGSLm?Q31cJ;)8$gt2aRjgjXcQLi+h24@pTpDf6jO1usI zi~NXcwjcLN=JsMi)AsI9vAYuRxmvN^knGI>O?4;a{k%zsC9K)Sm4Z;es>{d@)76)= z4Q`tw^t*573i&ZBm6n#beBr0K-@wsULmTGpvv%?bJjXcML=b)Gy2@R`k1*3t&}vYP z-pbv}8xN03MYtw>O&*(hxHVDkXuak;#rkV?S47;#0~j6Vc*-Cq;KrszzPMAzPLk%) z;aHGZFPiSL(TMW3%T8-YYBX#{7Ux(oapi-1`or{AaN)q!n)+IcYp+^<$O_oUivRVS zWD$bR8#4LOxEklh5sV+fJU9xR!f*xCnET#@jMoCYslNM_SM1_=1Zj*Ay-l;mb4O2p z5#8q&j~Z6SKVbkv#0i7$XPks0Y@NlLy}#J<4ij`Df=95i)A;gS2rdaaCZyr7pa)%B zWq{+1eI$B4MP2;Ps&K}E+$LjIp>C7aG-N~e_zvs|lB|qHN#ME~-lZ>Y%`n4V)yX19 zT>=T9_KaX=?3(jAkV1&NBm7|~To)5+CP@$6&kgL4?T$PyE~CmU4jbjfCoQ402hwSl z(69no>ZNI7rht&yhjf7`^9{eeEm-uX8Y~KDYuSMujXm=t`rQdXgL=l#H^Es#(WlBA z1k;4B?;NkEqIAOz07P1KE~%Q}_%r!G{Ij(3d#{pNBehz>kb7S$O9l^3OdtGpAGe>G zoej;5kXt5?cI-3w*^T_s0Xo5+H68U9KEZpTreA$AAcEu^*#%0C)&qhyZ_*91i~`Hf zycb3JINU^=*rcZpv5m}@KX4wDNH-0Cbl7uJy4`wu6U~_9dJt&%$So!S=ojNfwb^}3 z(Na@-lI7!9V;Hx3*FSs5Y%=0(0Uk-vR2}F?<8kslG@zA@vq5hDl1z_LC7nPhOxa97 z$ksx;Dl^PTZRR)S{vG4$F5~dhgQvH|J$sa?)RwS`m6mc&l~nltXE{qC3lpw*v2<0X ztY&t?;hz5yBIN7-UU7xVO7dFu`0I~W-b`g|c*8nYS_*Pj`WcaW>i!TQy-n<_CAZZI zN2Mu`b5I~}5RfX!g%vsn7WJ^s#B{J3;_|0-BnJ2sA1mSzhG)wX3UjkujG2y(P9N4* z@gx46F3=3*H1_TH0(YPOC~B7+z?Wi>IVhhX6e=p!)$u7^64sk!GRCNdgjvAJn)|SU zz~V|1xqT{g#nVa7hK|l`UEyBmb;?Ai7wW5HiTG!n(1z=MR{ju$g5US_HG3V~`=^Gp zs2H4-m%g+TS3H7|$OFR!#VTSGqhZ|RqCi;1RF8!|YR4?7vivtrVRWbkoZQ+!U>Dz& z4={99{!tlKag^u^4wrs(L#;SS&ea}XcUT`(%b1o%e=jUgXwspm*p0~^tQyPjEVZtt zHImv|m3ULnnQVG%6yAeby~4w@b`-%}0`hjs=0*8Xg$rw|>$(uDn5Z)`WEI8Sd+e9u|WNuBJKUi`@c;TTs zAR<)Kn8!kU$-Y^c?aZBC*N6v zv!3iW<=4-``ARv4H`|qBUOZHPC8JuBK6RCq2zc@hXC%!vw@jZ50z2B0{%R{Qje(Kd zFKLd0hLDAh0$JYJ%Cn&E#iDtBW#&W)Kf0FzSDDQ>0KjG3EJQ3tDw{d~ z?Ec3Pe5p-vN}WPA?xH!$v0~EdsGR42@A%QWgNtaRHfx?&JADRR98opRB1c(6JfT4( zwV;~E-?N@Gm8}7DsnrxZ9P9Y{cTr<<_S`KaWn(*aZxUmGgFoM_WLGU;H!vF0e5tCE zbIFVjEf*@oAqJuljvU=B$5Fwmwn~cd80H7Dr%OzSF7n)N(>}~)q^y=Po=4n6v>C~( zo+Nhs8btPu|5)A(X|$p+LiQKaZ$AesHL3K09yFE|^z8136@r6UPoisRYv-W-uViA# zm%i@Wn(by2A!>0TtQ{4Xf6?bL+_VF*=|IC4t_Wv^vTQv#(uY%q8`KwBo8^>9DQUAO zkrcQ|5M3C4uC$tZwpss2gxCx$I%#?ZWg*j4$c$e=*F&Vbvzn}I_P^_q^Xl9FMiug* zQm+}HZU5O~>Cz!8owOvReh9qVA9Qp-xOEqVMH!*ZHNivASXyn4_W75gLW-+H-4Wi` zz#d@rn&au2cJ6cUefA;7NlmP5_D}Fr;8IgDor`f-tSK?a`9nHCA&{^zVRa(vqf$j0 z-s9DTWpT8fQ;KT1M1EiO^%ft=HBLTfNOtmupn;$hOWdjJtC(cw2!Xs!@r;R!=|R;y zKY`q9;}1N%1T5r~cXGqJve0_U%)`e>FNtI|nUBhadu_t*4f$YX%)H^zWiRRZ1$RpfPXMV?C zCu(lz{X6{)t0?!$ce(0EE7JTJS_H?8sf45B7T{`E6fEF~EdNW%V6Uup8fxn~x4hPGz)i*OK+9J=a>z zri?h5_1<;B+1ApvpOjWhIAX2ys2@fVeiU&%zC0<0$@fP>DGfKjyI324F;sab3n%e- z5`1e3We;YUgURcOq$VZZBDjx-JC9Y)R)wSXp{5)k9saiMJJZA}V7IL*>c8+Ll=wgS zu~$UBo~S(9$1AS9towo&nh529w6=PWXEnn%$0&`B$=N8+gfl~lNm*3L3jsHNw=Ko7 zawv5knG5gg-*{7s1G7pkJlMCW(QttTd`F7)`J9;>*CY0eX?#Os^U3dg&o12In$prm zV)HH!v+Eow5vW+G!OCLW74|DHZ}Z>_cs=^x&rWZ!UR@5&@;sqEIBwZ_&^<6y6+cA= zakvVN>Mi{H5rCv5r!HF~Z4v&z0K=0np#T5? literal 0 HcmV?d00001 diff --git a/hooks/screenshots/necessary_files.png b/hooks/screenshots/necessary_files.png new file mode 100644 index 0000000000000000000000000000000000000000..cf0455f3582603338da06d2f36d7e27c99cff8c8 GIT binary patch literal 28831 zcmd?RXH-+$`Y)UW0#f4^5RekxDj>ZHNR7G$QJT_w@4bVRMA^Vb1XQHAC<>ciLT>>P zkREyoz4uP&>A)HfNZciUJMgHA)Z&M5Fll zm**f52@3f4{SpK?LT(5F1OI{DpDR27<#(|z120Ie?)@0lnVrk zzJdmUl%D*{gIT4dOD$;zRdvMKPJC!q&Jb201Pm$+8oYu&kvrXKH$E3D@!nlDn`^E* zpBqma63B;@3CwKl9yq7U@ryw~YWh!6SH}~oUYwnD*HwwFq*J9a?&fNK|27hK2LT2R zTC9LlHj^lv`xe2+hs*c_RrsDClnYd!=%9s~s5%)k%g(x_2111aFSTB0W&8|$hNE~M zd`)s^{baPAGYVe2(N|avh1c1%kh&sR&^MI&)_YPsQuW03cRfqJiB0Vr!tLv0f+KM#(hUG>4ImZ ztGtUQujGe#$xzlGm6n39Q3)5+OS7ud;!YDSg{RHRr=n~3PuHOuCO_Q}cV7RZpTEu~ z2xKyEMYe=IKD7sd zbngAfPy7E=7kW=o=f^v88*Hb$#Dg^A?fr2g0n6A30uA4{r@+aL*B$&Q2x6l^)Azi+ zG-5aLq!T|x6dtB^NKPW2Zyn0j5nCF(n*C3vMjDYPLj`1b?T#e3kgwsy+?{T5@@k2O zbtIwcY!*9VeE3M`N9i1IAWtR*h+fY9oW_Cb9quX=YC1)PIJ3=$b+-<;-obmFQC%^Y zWW4hO%^l@YA;!C{;*BmALg<~%$*!@|s?9G={`26 zg8?LGni#@5fHQRcfY@#%>(dlVludSyCh@3SuTU01n0l_H^&G+4R;Jq5pb zJ)d_2F+_}B&rO~L<GJ z-V3neh|SGcq)SSxCxKG-kUhDxExGf8^3AlfA%|J)>A`{X#Ay_fweEZf=z=HW&Md?s z;!(kP{BY!W{;i*htE(0##53e6&Z(fO@2=nEX%zAf$sSf_l|ZSkqY-D*L{QqTzH#2a z8kq9UEsuTh^`y|W0s1_QLy&UP-B{f?f zSk>i`I!%}sz^r*YVn(-=R?}B@Id=E3F^uuq#&+ixN$VKnmvi&iq$l?j=bfMM^O`gh zIdpTcgs2t@%^KS57Z}~v*WcZVl&P*5{l%YoR$OzJerI#ydf>_J0@JF`61nH+yw_nTkV_?(Q(XDsi;jye}C+8Rw;3?23re6to^MH+Id3L0zX`9+4yY@5BG^DEFt z_#_TIk9Mv(_bZ)vF1_tPzs$mBT(nWmZ)hr)Vc1<}1da$hnk4EM-_|g3==2|g5odK- z&>z*JD^BBx6W`2=Y9ssWoXnHFVEA7hABvRO4pq2v(@;`{Y`UrI(A4p$u_7N52t&lZpSXYuWm zEjI^-;SAoI1v=-s_(sNfZe1^7Vb*y5Tz_bB-0wyYrBusM4N1>mb8<84whrFIZ&8TJ z2j-Q{Pz@Jko~I-Rq*+Y9bPYag+`~)AJd@EBuaNCIL#{T%j0YUDJ^W5S?+l6gjm)|e z%ZW@0A7g5l*}1?)%ji%||#ey{l@G;Z=~!l~3}f1#Y( zvugNc+$mdkn$?+#CrMp^8eGcT@r#>Fk-dW-;W_W|w_Wd#hIVJHn8p*+{gZQNVo$BX zyq|`2)*k%`mDGuXtA9ml`BeMK)D61T(2u`_ho`Tj8oqQs;rj08=PGo!He__oQ8ndF zt|!}=Be?kk_C{6O`_jEkQ{t@WFA~|zmjfXS5%1YjQ(avMdo51e(H?8w__5nf+pW&w zGY87D`p$!d>4F>wi^|bY+$-SD%`j+28uTRRxADt{w&=KnU&Sqz{N)?Q82OxR zv8Kn9zs=J*eBN}nS{qj~4y}kktzS#xOm-a#zAuh01;q1%3o zG74*1TeLaXcIdr)>Kvl0T4Uf}C6mT?&-_g9Hs*FF%e`mEuR46&Otpzsn%x26awLgg zj4m}SWLpOi160(l9(<=Q_b!!;%~U+0HTU+hy~-rANUpmhWjh!(E|GD}Jo?-2hdVf0 zmZP3~JL*!*%z=(`*xKUBv7_#hpOzKV8>Y=w{THZ&Pd#HEDOQd)NVkpvj}=n)YwWqF z3Nr1ss>>X7S;eC_Nm$=~#maF+(tY{PMdhxt*v?6P6h270A@}r%w2zL4Lm)0L&u)$7 zdflb7;?QtKtN5casp+k?6PH8&y=V)2mb=6NYQLN&@s-89$s+pCZyG-=@?KDE-+X)6 zJAZSQXN)h)FYh(x4PTa4-B5VVLQKVb4@Yx_rnsD>Q+m~!z!Y+#orS>uR4sp+`A)64 ze^jwL4EuByT2f^=aHs4j>g{m`tN$%gqtDc%4^jYBU^)t28usjyMf7!tWRP+tI}o}z zw&?PXC?s}*DvtQxpF2dz?T5%Q^N0j`h+(fh>HJ; z?)puEetPgVBy7KW;Y_w2N#gl6OvPV)Azb9ZS$8`HcS$u{rm4d@dq?w0IV@9IOTx?K zMeN?%{o@BTb$;#hG)4obrB6}==_hj=R>!*3@-4Xhif1<;V81rZkXzM8ZHK*)Eoewi zCsuzleyHO@P_j>-ly1!GdcUFyZD*+`>dLD?I4>Pi^p5 zDOXWsCk_j3Je;+6I3LG~@BY3=pDqxHsWp%nJx_U?Y=77(C_S6Y zXea1fS<{e{sKP+Tn8~iMWWC_Nc59LXGf^5CJX@e-cgogUm;P$bmu7*LnpOo9GVxtw zygf!{`?}xF*O-pLccBF?V%-MIJ8kJ-nX8?HmPA!+=u~^^ZkmSZ62H82^!{3YuBvZbUh9ex{j|RWnx%1w!ky6+;{NF zR@HhsqsMkMgMnA}LHoTT;Hh66Q{#8R%HQCcJ_<*xhAiOHZqo-V!xiSyQP8(6@S zaktI@(;2(2$`x4h(TPJ>!F2S*2fAMy@d>jUBjOMFl4}iDmY!I#F-6oGFXP#?rA`i9 zy$ILOn&b{Q_~PPd`Hgnp?OvUII{zf3ck%AI-fdOK!o!!_&igxTQT9$(#gg6Un1gNc z!4YX@{aaFblkerWPw9g+BY3lf{DWJJ8eL6=*1wCspE|2aAtZ9`znf2x9`Dv z22|xL^6=`*_vcHzf#Mo7s*QW(_WOcoJ-X)>@RJ_hFgck%W0CQ+PeYmC<312;Qg`(^ zRgwfMo}K@uYPF;yWA1kzU2Eiv{pwaa(&s>4@>J3{+seWxa-4Ad``kRxO#MgOCJtbu zjHc&F@3S(LC*Hk~8HugP_E$RhCwEppV-r4(ESb?g-v74PEBD4PTDu~(5ZR!Bq!H(# zHnz@W4;UZK6gi|-ocyNc5AD6@eC~HV)LgDOao_?Bp3C9)llWTu{q_NV z2Zyd1=BCgy;e|85M4rjt_8cP$+{^oC=o>kor-}l<{gi=LzgJvzBsI6vN4YqC#_YXi z`PG0Kf@pU_ALu+DS@|=&ihHgd>+N zs#gYCM+U4c^LW2|I1eOln+kIfZ!P887u8emM~x~St2=D5@Gu*%w*`85$?>PJw7fXZ z`NA}E3R>TjTWutqYCqWCl{|NWDa2s?X6`fx-q4&Wu2vjP8+x`=ZQ4w*$Qo#t(5wcA zHC`|B#3Mow3~& zL11fMkO0zylNIW!K0wWG|0lx3|3t-p=oWUsuifU1%ikpW>=H~@R`xf;O+lbFG(acM zVc651I4sTj_-Q2*1*kq>_fLcf5U4O9gBEvGH}Va?7nN2QK-{}u&W7VWdxvntE_@McJt{Y*jX$L_)DnD0mGwSfo4k9+S$FT%{d%Wj4)1{=B` zO-yfN8u3Fbq1qevYAC)UpgFm_%)OX4@#DoxZkpW;$%z@Cxcn;Mm#a5?s&rx$#lg^m*lyC1?VAs%5_R?6(|mGPXH(Q07_lXz7m2Ps$m zQF(wiq6OkHNLca!SAEkE;Y>@52hJ$|hn7*?jS8``mwQN&0gE*UOMlI7VQ-QYtD{+)$$YCmu_V)EC+Jq8W2&kg=k>HHGekNcIJ!r^kF9 z_7(XsnLZ;rW(^b5y1sKdn`C%D757`rMm;~d>fM3U)?L+CRzq4B0~w9H(w|mO;t=Xc z4QLEv!+Math4n_Dgi0=$yaov5knras1_Wf%o_irlm|Xp@H3mB?RG{xZ83>e{%=7QH zxc{gm{z1wEG&(raZTa{-O57Qqk#9t-VX_aNw@i9Q4RM)u4k**@|{|Dt(?c51Ns>BCR?yt9To3-ueZFYZmx z3hHBTcZk(M;i~=F-Zah)nIZ?CELtqGhKXyAQFg?&J9t@Z|C@##V_7D<1~mb!&W`Xg z$w;I;9Z=b1d)4o`99V?LSm9EI@8L-sG?_V{_}E+kC2@?wzCbQ}M79E&#$O>-_BKM1 zO1MKy=FPf<OGy8(1I7E9mWl#46@q+I-liYFt#VYKl==vYagP=Mi^S6=)KtMo-VS zsd1<$(Gv!9x;?_^moq*vsPDHez&;G)U&xAD-{JW@ZR$C}e?o}N&N>ct92bLygZM@_ zc53y_!Fn>K=U*+PKPO=+TAaT65S&~HqSe>gsc_f343dBMWy)|&dn80{a^`gh5q+83 zc<$pM{Q2ID$>th}?X=dPCrR++<}o@=P__9wT>2Us)4aHvOP#2XDjsR(Z*P)heOjgS zO!t{(WA6l|@xeZG(|dbiexsz0X#V_Ld3azzy4)#ELkPc13-Poj2dPKpmuY{nJTKJu zdf|n#UDHAhMjY-O5-ZdNRN z741;x11+g`&}E@FszD6fuGlfzDf&eawBE<6+p` zp$;#fkIHCcwj#G-@-;(s@nNT_k498~Lf_Ozh(JDcit`}M%W1yyu6S{FX@|uU8UTovMr{9n(bZ?96$OBxj;k74%*8Qcc&W)L#u8RV9DIrM|`f!1+DDFL6fMgnRc6^ zl-CgtwzVX@)Cm(WJ*iCxT0Ud+I8q>WsN>{-}{ke95kt!F? zTwHvDwqW*c{dJe5Rja2CeiKH|cJ`Mg?-9kcBGH~8N=ASGA2@2(EZ?|jbI{36mp zgL_g`c(_#p*3)p?wd>H~uD|ABh=SlhQQ#T~cXLQehA-7~=iNp)Pss8XyWIyYlIKd$ zHC`}fVctbIHazs-oLc05s5;+#9tHH~ta!>{T?oC2KJXE+Y%&Nsto3)`^gl`6|MbrO z4$#@o3JsulS4J10&gF=G#E!kHAviEaUBb}})SiPt`tm5J-#7NjLBqljkB6QGa-jOy zpU^DXerI4%Sw|L03BJqDHwBDhyMYi79%2R!{{-<+_xXQ%08qxG$$ zq%rl)(r>HOAQyJvXCELB=i{)ajy9VM-A&(Ps!vYOk`|-FBXWv}GxK<0@*u}+TZ9cU z4_g67MO4%QKdj98wY{H0H~WMflLNWLh#sV5sjGirbrMM!D18PDaZS861%$p#?QxNH zmcCz*5MBYeLH0jA>M>LNR}VZW^^S@C4Fsff;6 zIuutUq1l|z9bt=(W%SQdyM3=CUVd>jM-dE%EzFF6-8O}%cxHOV=t}2eij%l$a1N=0 zxoNu%Jb&@bQWh_O?-vL!z+1q0tWY zoe_)q@9QV=pOw|Tz>?34cq;B>~7IOG-)vKKca`A9sreYHwM zTr}9-M>;Hj89WTXOHZEta)oUB4w!GVtjb1F_sU1_r2D%zy_xUV4wOJ*s zQ+r7+Z4BOI z-($Q_t|`hH)@CSFR&~n#e2w0=^wZrptPBDc{B&p$rwUlg&j^KuH`WGQnh>rv4OHa$ znc9)yJ?RjugF+b2%fioe-xgkK9#I{C+sJx2@S~r6TW(jz{nNX%eO9$@*gM(7?jyCE zw-DrK;(n=L+XmHGHRZPFAtiy5woi=eymL0cI}O`HN|G^wT!@#q*L6@@D_#yX8E+D}g)N3A+}9 z#UW2h-)y;5f*i>`FrL#D{wS?p@lSe6{=nJ4IOwBNji1(Zfk2Br|Af`6FRBbQB@KYw zK#%F&+Mu)?YCxVkkqbO+{bH6qe(`Ah(Rg%U{a@>F|FTO8{ouaoTEG6(w`7C_q4?YZ zrBEsTN56do8dn=%g@Fn!NtZ%ae*E<&LR{nEm;|63(SQRA(t7+aZTWvg1^!di&*Ob? zbRu40Yyfv4A7=g=#-}a3^QXZ6G0EO>v^PK!s*+%!VJ`r5d#IBy`A_<9IUau@5oHBn zf6-4Sw}A#3=;N_DV{-C>mlYki$;J-fG-P)m*TDb~T;H3hr{t;kbn+D=y8aljw)=l@ z>Xi~wuELk3TnM`)647cX_AiZp7FAyAH58=19_xQHDHkYNT$HrZl|E!T)XwRhxL&`~ z6f5a;`x(Rge0-q4#}~P|sljcGz8|*A4LhzMH#zZ4yl%9*f9kr-=JdM#I6?4XY0X=Y zqX~?|=2V@R)#y}O)Lq3(wc8O{`3M34v^2g1X?;!YSVxprQ1!c} z#MJ~qV6teBRF4p-9vu*<@QDHXkUlukFprv1_T)Ffu}|To)P9)M7^)7ozg%Tk68PZO ze5F?A^Js&jJ148iN2y{YqnD&Qr*xbHI_9V2wM}jtK4(3;4^GrD@5f3s{>Cbur6Ey> z*JMx1t7LmCGtc~$nURSMH(aT|HU7QoCZ6ikBUH%Pn%AT4*S0d(093PqG^pY*z9+O(d&C@Ghc>FE<| zOF#+m*s6kfFu7;JXmR&By!0P1UiB3h5L(?9A%tX#h#)Z9R;j{>MqN@a7fMKpZ#+%@ zH=>yOah$w&T6Zd?PN?YTS4T+bq>Qw(da9F)?}S}&`RGa_9KAbRD*IHnf*QBC#jDz` zIRL{q&23+UDdB|zyzcuSFH^ZWZU$aS%#+B~M;O$ApehehUiwwU73&v|iC~4 zY@?Hqau?ZvNT>Dyz4aGlWb5IRX-;(129uLP~m_ia8$==NRY-o#u_70fmEmn)}wm`kv z5A9L+#PL{|GJJ*}UD$J#7KhV6fW403aS4Q!aH)VP9X|ZEahBR~k}f>~(jTQFFdRd| zb#eyI!d5XJ1W%Jj4hLddnhs4xhpy+q1JdL?Be8?)r0>&k;xb`zxlr<9s7xYYY%`Qk z!E2t7eJ)>sa`{&T>2G)2d`jo>MKvc4jZb*_G4(m-ih7r~ki&j8{fOV$^&%`& zM}>C0(I; z%dppO+SCi`F)^hoUdDiAa>*8Xc-hN>FkP!|g~u;0qd#6#h~kD@?Z)-0 z)5EM&*-W_guc9?+X%hu*Eq8DM40PmWm@o2I`UPii3bn)QAES5hOGU9(1~m=^Y=f}S z7OS0xtRnSqxLl0eWwbd@t{;E1S(W%bFDpp1(1>NfTEhB^-hyF=yJ{6QIxCjnA;?iMUEXULYDb}HdZn2 z^W1fWb-v4I*Xp$knd{l3vaYr9-Z4<#kgtdRo+d;GyH1JQwvPq)bVYpPShcDJ_!>vD zYp^EjEBW{51ucd;SH8m{hURRR3=6X?5MOFP#p@Yf_Q8TV)9qrA zl4ZLriR7SvdmvfK@IVqBdL#`24AbB@EeTk!JFS$pIsZ#h6L5Id>b_3pX(V$Lp4f2d#Dz!rsWe@oS*a}OFV_MT{ z;(k7NX0ck1sORwDz-dG6Ws3pSOwBmzKJCQlQ!D<_qW4|V$*v=yyw91doXm-Ud>y_< zAM`sQ!2s$n%JME)a#$9qOS79GP9VEDQ4^qJG5_kqGl{1(c>-{C$iHZlg15nxf-g$} zt@|Fp+&QlQ4?(qm-P}5V-v0IJ{vMm_2e9(2D!<_*aT}1gFDUch zqHDl3a5OJ~WErhKau?d^FhuTbbgnWEk>g)FdOL0|jWnK+2dzK6@GaOT>coa=>)-zw zNTidvgMiBgoSXxEIB31zmlFZ*>gWecPKrlQX2@xxsthRge?dW#6aLoHoHFg6fd76F z&}geD)P^-7^iv%UmZ*@E1!uiwPKl&fW5{jcwHF6ASo^lUx7yyjSzoC;Z=|?5i1QWF zGCITM08M&(r?wG@c&6+74G@?j_haogx88bfHO{8oX$c%pujPYL zbb##wq$iNsPsgxhz3&AKuRHR1g@MMUvS}TLE{EExq0~o`rl^Kcv9Wn4;3&*hI#4MM-&b+t2@@{(ABK!LKP!oyhID3<3DUca`P{xC3 z-0zz1+?M^>+$qC*SRqN+h}4lkkrMbm9|bVs;g%d3;A{LY8HVvNCMC^;fC%@l3FaF~ zaH@I=;Iq-;DPhux-B-YLQlvtM4x)-aLcPu8p}_eN0{}vi+_qr|HsnR%e|#t44R9Z* zsDG^*D$)Q9o$4=J1N1@xvDe)wT9Oi*!9(w-0QiMLxsnsqp*F{!JYnBm=7nJ`Yjj_1 zooEj-7Wsxi^;ZCQ1`zuKRqi)~0@56)n{=hutXZ&4u-v9BNJ-fD&-Yu-*J^!+6|9rF ziFm>`h)=$`;UL^T2b65G3MqL`%#71pFYPdqM&$4b>9edOF?YE6vj&OpGG0>$mr7Zp z$b=;I(+|dyYFqr`W*g_St=$}oyj=!Fh_`;1dr2;l_y14Z24p?=7bA`g|5STJ&hHQ2 zW$?EuJY!Ku`%0{8hAf8Wy>G>KYJrqRhvGm5MSd$L|_C-jod-UC&!shUMO>1}vzf zIhAtzx#4mEqWSg3kOFuiYWx*)E1R`%{gokuULH60hgBr=#_VmAOKv%JmEeM5wKvXp zd7=?dZi(XHd}3`Rh-fS%Z0Y;yqPi-fX#y3B%^ulkV~hB*aU8ZTR&qn9)I&zk>Jge9iaRZj7TfX>?fTJ9ID~d=2Wtsz(HBuy9FqM3MH;b9q_g_@jff2#;vIa zFFg!&PlfPJ!|&%Kp&aS*j*lk zLIGw_-YZyyQq3gD5qOdq@0+ib!X7Ft985A&<0R>#$s+o!7$tJDjvHsgdPiQjb?l1f zS5B@LOntoi%3&JhD9KCFz+{Rc48l&F+Gho5D<+eVzk%`?W8Z|qfcU?E@VYX=izXBt%*ZE&U!L(2?g?GWv(nEFSr~|@{`0%&Ls9GU zRHt{2p9g#6e%ANDD=Q6LYhg6F8na*L+Jhs<2ow= zd^|tE0z@#}^8Xv1>tBaRF0?;D?OQ#O1JGOi#nKZqCJ;!jyb{fe!~(~@1rW+u=s(6(Rshnv%|LS ziA{2S{%%=mo0)S)cn6=S_Ic%8E-X58AVSpS=0; z`lt}kSD)9WWnL{^2JUR=O(PCPNq(tuzyuApT(l!8pJ1?j3Y4k3X1MND>8sn{Kr*sx z_RFIa&ot$Z8aM3BI@0;yjQTjTM>;0B`*0ODrq7Nf3{Wqni?C0~8WLWURT9xbeZ9wq z2{AYQ+M!F|pV>lVjTti|OGjB{{W?1pTjEj_RmA~OYM^b+!xCk+TS7A0NUq+%9TBXO z`=zS&!LFs>L+vl`MAvFO+G`rkUq%=luDzlDEr$*EP0(Gc!a%;G`qsH*ka`EJu9(wX z+3GFNqQdykopX(g&z)L>N}1oL&`y0QlR;R2Hp(p;_BTH((LjZa^s0rwFXQ>SxaL>g zH_4?7GoLY*(-N0(rz639>n~oHM)2M>5OR!-W52{B{^om0a&LCd7YYgKvSHzaEjkh& zYA*dk-qGVK^Bft^qBkC$mD6{u3b;vauJ+mH(Wcp2Zbb8`%IM6k#ejIN@IZ#o$1E$a zC_cdT>1bp-@Gk$9Z4ldPnZ?}L-KeJ3cm}A=@Le6nt7xhkwZi021E2U2yEUtPOz2%0 zyxG{)0RhAp-kYY`Oi#Z7DSOX{WX7ii>_SRZ2^zg0W;vgf-wLwFXhRC)*=it?KWEI< z{K66gU3vjb1sp(nE0YeLJ0SS;YI1g;_1;$_YoE2pN1f9C!xq6s;PI4=o##t{7U&mQ zy3H>gfQppsb1?#NN=hD)+Gfzm{WD*w28shZqR1n7CRrJt66pX-D_;l$*lrdDxJ|EV zBVj}{h`WutG5=ybdA{o8JgVUcvd;Hq z`VGU&f)1MDjZLg|f_x}_Jsv&rk#=0<#PqjH*`yFt@p7tF@cOjnOPX)y(L(#rlR|~g zt7NH66H{&Wj%|u>13^rG8q@YG;L*Sdw7_?t=8fsYuwDSL=~S*$;W8$A^;eoQxs!Nv z{EA}4?E4}cYN**(`#0(vl68O58eO8!o?=>f=2~%yK#bOV{mA~7J@vr2c)>39e8xu5Kt5@-)N0O$k)Hv5 zzpr8Bj@`xW16r=$4Jv2 zc(q`ok;y=E5cB{2uocVv#o|aKY=tbaDD3s$P`Nlh=H$ZC<`RRf6_=9Q^rT#@wIr_P z*r&@@ZTSjyAIk(D6V3qZVcmWn!fh{3#BAduITAhgeGSytJu|)i+5u5B z?a-kIzk9Ya9F*yfl$h)mrYAw&S>OIn_yjQ z0Ta5lNNBdV!4yIF+l{LjU+z|oU?$(AZ22$YQP!JRFz=Ow-3SibyOC`C93h(>9%REz z5Dx+aZY1q0bWp%UXXLq}M@F!AxUXKhp07i3&LBC^&>TANkBg~r)#&`HDLXi78~T(+ ze`ccLE4Qvz*F7(-n%ILSFhG9TXmEs&Fw|FFX64#Cs@?w86iLKM>Qg`i(Z#R&zQ%7* zkuD97Fw)`*pVuzhQ0_ z&NSW{D*m2BUed$t#di(u*?XITi9Akmlt)XG$l#yb?Eke1QgXz5zr2i)4yA(H)b{Ts z7~!kaYOaHl?C_A1-T*I=bB1RLQJx>Q%)Gm3aaJLZ1HS6oe!@5w@4PTgvM-gDS7k)6 zqnxsJ8MXBHAGOI>rvh>9tUje=_`4&_?^f;nXQ>_z=U3*tubG%TngTk8^$kRy1lh3c zKR1-MSRdbkU?r1kK>e4i+B%wO=++ELZHqXA%bm|X7ho8X;o7JqaYzFc zJ5YG5Tyr8)n88HEDk4 zeuMf?=t6s-(Ikvyg97%BFA_VLh(Eye-u>OJKa67M+A4e$E1Au3FU??vU@f3UJUlY2 z^Y%GiGxe}97-erPnh;+4>hf6XaC*BZX2cKX4F3R5Oy)|_ML}}Mw4gR??V>zL&%e0T zFyl!I-^4W> zc1}J0A5z^%rF`9Hu%mj&D_(wI4)Kx3qE?OjqXn(s^`y1O(}A?dPoxwOk<5=WldUp|9+@bc?c%!uBLnjW{tZ_q}B4zFfAlbEce!l-uqH9cF zZcQMuIjag-*g~=)o_Q4w;wg^+kA9o~4sEZfjWK|;Rj%k>sCVOe{wBhi)VS;N);YP& zVv<3#we79@J$mPgIDWv-1TO20qLtkSV)z`uqUL|p)a{tJXTI3Sfw?U9ZTX7mLKHaRta~ZB48FZZki_H00@WJ{h#DtF$)V8ny|iE{pq{ z>*xiF^|AuLnC>g+D!acaizF`Whpz^$m;OU~n(v|j?_ofBbrN{8&I`f`tXP&^I-pd$ zh-XZ7B;Q93_8bArdQMk{h%U=35mgETfv#Ve#PWmzeeQ=g&rG^hI!Vlm?CN(J20|Er z6n)oq1W%gbv2tmx*1^(JA&hbJGq#}Cf9LQbB%Ugl@9kMW)yTLWf1Q?T;+1NsOu*$= z`+(r@s!dnWS~&s05EwOk%}OL6A;dOE1V|)^0P*CJh_ti&n!`>2l@8sh21wuGNaNC= zQK?Lbd95Wbc={iedM+UI^-K~2JouFu3S57aKz{BWgt@TSPrjV})&uTApa|F^l1!96 z36`JnghbR=C%HTj(3Z!u#w%xpCu$wE8NM+&lGA5~u=Vblu^?}dMn%D*R;cq>$Fd0o zsBWk&JZ1l|wtczL$yvW>eA-W8pwVlgLRK+-L#+7CTK}Uwv8H?WV#BwE5GSu3^3^zocxO>%cOKrq8Ydl{wDuQ4{|`Ox z*+q%!v!EOOmLB2x*wIJd^_YyuUctv%7MP8&)Q|8Qw)c+SIBoD3eBHPdkPu`1xEV+X z7K0r_~q1li$WVzB%e8uAfD?Cz5f(gM&SV% z`vzY-$lpA&aC=-slm+%q82km;HUdakP#J=iIpNiou>DS{xLZ5dwi`cW!hlq; ze2ogQxkM!IO*v0y^m0)l9@uZ<54kKgeuVQ$dxH9pb_d{++EN+D0t3Rdj$ASWdfu{2 zT5`6;H;?}sdOz$vS%k;|j2x^7d(2t0yL2>lje89R*ap!_ia{YR$(6uuG8TkTJZ zIvofVoGXM7@IThnjHSvZV+<{rpdavZ|7PBL?)(RQ={Rb zH3)7|?UP#DPMrVVx$#J2K9Qa2q1Pkr=U|Xm*9($8r7v(0jqG)1^BRsZe{zt{=v8uH zLzSfeDzpV`RuQM|3v6(0NYYQ-3rzuP$(ln;EW;VPhuQ8*g{o;n1uO%|{jI z{O^0ih#~OpujdI3e21T|MTgp=X!D&o;>744b#+4ePQZ&>>eWK%<-eV5Uc|3}S>Y59tmN&7 zXRVpJUPgOP2ai(SJW;jl!a^jUN^qz082qd(kZZ+hcndN9%DSAU00k+$?R5jeW9tz~ zYC{)pE3DOP$BLbn&30;O!RVE-L_=+ad}WPL#nEFfTFpu{e|rlP za(#SVOJZw&)q8}H3&B|A9>_W#7t7i58TI%NA!9uRIj&m>j$nZcuckRP*Ye=xIDH zh|C{5(B1;!O@QVQS)&&iAhc&QW~5>6YU$xJDgCR7oHb**C5o0x_&3Zz%f*g8sr(c; zJD2k8Ct^t;%h35W(tS?gxCV# z$xL}OsCH{vTT_?+_iF~MvUzXcoGFOODZTJ?{|P|g!*~9?Gb9h&M!V*#m*)cT!>jED ziN|i!o{}8y=*EnKoGS9RCh!nTSq@$7ViWe?5Q@J?`R{4}*D->BzVIpyE@yHbe%d#U zgCnbUn}#N@1_7uF3j9KDq3{h}8$i!zD`^Iu9hduQT51t_%ZB!kTGBS`aADFu`wQ#9 zD#oC75XgGE1G4uu(>O2v{vY!7Nk34D@4-=N3x4Dzf&1P74bZ{^h&0uST0SxW@cU#l z4C_t&7B_ZXruxYD8=-@{(?EYJRP5qKeUm?r8sB!fV;gcG@eo`h?(`dxwu z^FTgU^!k*}>n6iP-4-kSDpdL!|IJG`(_|?a?*p4r6=MDg65XaI5mjxadnMqHz|S#cUmTmvUpfNupy6p|JnSNs@ zGkWi7#O{H6vo-?d#$IYVByZ}_nZK1->v+ki(l|o;2XnfDDWw?HNi(bgzSHU+IB~PK z%v(-o^N^lX~WU6pDe~Zo#iBf{p z*NYj0BaAsIm^vzFBr4`~>E~=oMAKZFJJ0T|E(hPkglEk^YgKH_KfQB8&9$icTjiSA zi&*cV+K@s$LD^7#Sd?;>llIg3Y=&M)h0^GP5NUeUqyN6oP_SaDyYkq=#QK1 zFz*YG9Dx+)q&h-F=q7AmotbwtK_sYRjy`zuQi?H&X(hTL*pc6GK_Ym{dWVSH?*@lJ>XNfMe zK7{7ws z6X!?qUb@x!>u?6n>}s~-d$jF}2{r?o(B5C_nq-v)$rJDFZ{s)uPUBQfxb?Nh9`ArF z>#1E{xEG8`TMY_PV@VZ^=v#KI)H1_4z!+Q2u zghTbV#^#)Ac5tad-^m(ocVXEk_WKa5OnrLjIo0;HBdr57_19A+?ColomUTbM4}a`@NPFfzW!EdBrQa`;9oXP?4SS^ z-unBdN$WD$Z|q5}DGv+AuRx(TGAz|GhzX;FH3o`EiO;!|*oFmH$?( z_5GxjOLF9XC!;_Pk9xKqoyB;&wU&!ElPzz>8N}AAeWr$G9&;3I!p_-LF__S)!2=0dN} zXy3=b3`b*>(=f+h4W80&FQY8dvs4fwb zfxJ+p)sy!08kn-N#4JHf(N+&HCUo$=lirra;anOvUoa8rJM>#2o6YU>uV+AIueD6T zN*@ny#&;I^^IBTSx%qr{J~)(3{n2rDyJpFUi@StL}oo{zASR0sVA^}cPfK>`4 z%7^)NUjkC}c~6UXomB6jW6+*_vUMYHvMHy+WqgFg-?iLUO*$QQC7^tE&T_?)Xx|26 zGCw_^xs_MsM;pM?{%+i&oo?>6(bbxo4Hoq@uaY`xmhfV|Wx-a?MQ5FN zRSbl*opaDQPh#g{dL%~puVXQJOdaW|E8KSQFfGeBqLq`yrZs}L1^h7s-4JG@saUS4 z4BoZj$CNM2wXfAfY`8=nB9e*@$0sYCMcp$!Kf$=qN&s$c^sJS=>05#YFOXcG{tP6S zu2BlSiE8ZW|fw@gZnqzAW_ChEXuXZx5eu z#b8A7)mu4-taj28Dt^S;al%bT+uQ55JW$aGOPazcISd_+UX`hjf2y&X~7xI zktH<{wmH`<8ZY+2vVXrHCu>JvMPv4CNtmYPiFs}rGr1{1kNeQg76NDdgFTgJ&ind! z0n3diU9OXW1=0Cn$kHK;@2c`TqEO(a5J=SMlA2f-DA@K?6XCP`MVHOoUm9?G+~vYW z{Hv$KqCj-&WfSh?)?X+#{BdXh z*_@*Awh?cVt7|}MOwhs7&^-|DAKH=*bvBwc*bB#8-9sK|4xU&kCM^h>pl60gZLZ~H z3vu3M7GW9Yo>=d^BRjK107@|{9VkT0?w{Aqsor$~QWtNSSLu{NjeO(C30fE*8_+vi z*@g1Kop%usoq~1vv`CZE-E#kS0v}mkMnvf}vYQnRcdUj%<^UG^Ur##SBZ&S#LD`EL z64PKAhJLG}SAMq?qRL%&`>Fi{5TEK0e}j-p;p~_~)xFY5g|Pcd;R>vOivVWhxd)8b zXfZc9RPX9kW^CYUj?pgeO^(#;k*ZC6v}qsMl;;orR%3t1jxq$0Cb54zY-_&DDSM%; z{OZ*Z;nAOQ#FPp$=wPaY4po7+g6|M_{*dOp15EtzP4#HrXblhGNC~W6s&v!$?BOn! zu``*J*4#WUlg$Mmm;%O~n}Jj>qmm!jATbBq@Kboo+r<*-?eA`}@rU%3N|AU;LKDit z+hnJ=X16l@q2R5lFfV$zJ|tj?P}LV>u&<-TvtW_mZ=eqFzElnBbl4LaB4)McoG2j3 z&*ceX<{xKp4s0?W^Y-`mzH4PYF``xWPV}>^+Rk)t1Ih3PpQqoz(*>4cx$GlYv4*M{ zA3(5N2ZBW`-Qjc<&g!-k#cg%KH;?N*k4!AEQ~4o$0MqZ2Sj_ur?1#iYYO}P!So}db z-k>B5M$BNM(=}L8b1V)qi^{rRY83Un%sapQ>Ko)NEHj=m<4w=*EE;igxu(H*%OSCyUu|UEz;uxN^#+}FvaiZ6{_*?~aP0P+2*orq1`v$0 z&E3-8MG|a&i8ec$%(7hA-oK!{K>SHocVRS2J&6w!%}*Z;7Wb-SUGH z?NZ4$oY*#d1;6mp!3+4WOx>|fw)Saa(nA$V27Gw{kyzmHWT>uR)O>;JXrsgbcuA1s{;Yb-FCs{iwF-U(hhs>wmPT(Q@Mu1!kb*3 zp$(wurRS>7gP<2k6u__Y{5@x+2pf8XX=%8a1B)q2K|iR#8xXT*^z;n%g|Fw!l|a<5 zVyn2|;)k?Wy>R8>jT=M3+Kp&!ukm`BL-}71lDO@|oVTcBPu&LU^q1^WWn%@uJ}Iqh zr5oL^C#rT9@1zfI9=K1gU%Nlo;&Z!|`wD$XWPZ#J&mlZA32>t+vbTE zvn2jClsGl0lf+C3JF_b)TNWG&@41OmSA$!%AGaqdJ3FE$J>t|a$RWq`f6zU- z5qDsO2}Bw3YhMaIU~mboH8xYsvq;0n!R~6i_S!5ntfo)X#z^u7hmeSPg!bSMA6Efd z>%(aQ`|<%gzwGW(w$%;%1eX2LkOl4{#DT>cFZ_pmh{Gp8@&Z8%x}TBzE4?W5nA44< zH@CQ2bIP}#$Huc`0%Sw%@{fst1TcnSyo9gJItTfgDs zN1EI!ikdW_MSL2@T=1L7m+iE?JDB8i-6dtqM_U8+Xj~6gdT^4jVuk3aYkd$c(cc77 z7rEafRaav7x=6yPhNu6%8k!@hk8LuG4;8d>wg9D#99^4@w4W5#d83KY2!=S+XSVuI z1RnA{_IYK8q=II7>b4>OP}V%k;sH`fi(Oxgth(hmeh{iBB%zo)^}(N{RWnk^q>a26 zUY7iB#H@?JSFJ`Wk(#aAvS@w-DVO5Wpv_fM^{!2XEm9&^+Svw}5w;h-oG=qZ62>k9^lx_t zAus^h&$KdhY+Q=3xSblkx{_bFB_WC&V718BK!3#2ojyb2hJfINY01@mD1_%4+EMI( z?17(KL;BN~XEuuCLAkAkyVAKQ`t$lo6f{U}dER+s4P^nP8Bb6OUGNB?EN51Ly8gQ`YGc&_T%Uc{e)mXKuX}Dh!Nq`T z`S)7@g=&5*flhT%tK>7PNVEOIch;u!702fa1;6h_;GO3w%=_D+pnXk;8_FVa_cZv} zc%<-qM;*WbyH$notIM8cN->Lt&LeqMrsKB(XV@|nVPju=h`wipiKaB7O+%DCsa-<} zQPp<(qTst6Du^$%5e{HPp+6E|LS?ZQ+U5vPZje@&c?@ZimV6P7+99XTPOCA8sH>ts zP9C0|>=*CgKY+n+90GfUwG}Xrw2P24h^QPtclW-79|u>bB6l74_}t+y#Kajhsh*0( z78-Gc51>vrJ-9m5bXtd*=OnQMiFESb>{8X`14+{$W@F+rgAk~Z@WVAUzV2`QxK5;z zA6OSxslRb<@O=NBRIq^X7%0&%rwTekMwcv|gyPKoeRjR-X)GC4%YzPR>i`L4(1+%N zTwK@l)`afp($tsgDSDK0AXP}bwCyVaKZZL86#yD(aS(mf1FI}lFGA6E(o2rN9xJ7GiAFJrH5>g~X z%(fQHtb%ZyNK;L!N=uJ2tuF2VB_YE%?osD5@N7wGNO})}H;IeLmZ=ze>ejx)to|{U%3x(%amx5Sdl0N4?l55c5)d>dmpHgnd7Ij|I zw34a6!D!n#5N+9?un3G({?M&W73#(Uw-M1Q+rGK4i{r_Q+h_F6PrAsyGAqtF6D1{_ zeKKrP#2D**w^yyqJBD)GE~tEM^ao3Gie7bL5lb`YIq&0Q4?Hpi_BRWrlIDB2c$Hj? znX%`Q&2ymv2K=$|z}a*%b;xiKmR^j(!c&5K+q)M$*GFNFw?QhGHdU8IHg%9Xew~Siv(qa?+jqx!xC$Q6r&VwIQ)5 zjj98sie2I%!aE=*04K32=&jRot@V@j+maLWtu)h3WJ$$ITI=~2B66n1LYx>k&Sxq; zU)^=O6kJSv;P|K`h)8>b3|vzSw|k#6a|wm3(qE!U`)FGf^R=WY0hZh$PXO$ObE%3q zHr=R_su8Ys2lBkTJF^Iej-xHZOo>YoKlT>?Iixi$ljX4%N*UO@brZiO>Fh8uaockB1|^d2Y+8j`vijH!B(QlM3&!R@s7k)L2WKfaBBMr>>p$@ zM2N5?$0PL*d0^SlWeLwO#+&Fl+)EA^t9=W^Awcet-Q;?aFfz6@Ap|B#eA@yR{OfWF zGzcw}=OByR&Zm3S>E)m4XNXUrdPB%z+vX~}+U=x1lZk+D4n-vzyj(ynmpqgN$y3_? zAh)h7BTC|)kB|X`*nB_sN?)6zi}9ELiQUb*5i%ivc5hNShV;6&oC2gJ4AAmRPrtK4 zM$}sNZ1&gK&)|ISR%QZX`y+b?;`ZKm;I1Zgz0#^3}) z6z*g)+(c8K=DYAcC_7C|f>-h|u+xAzyiMajZ9cb;D1`@o75i~0^;rOZV?*?wUcn-T zT#&EiqaDstj()q^FU>%RcorDI1O*{*6P|;>5gCG*bM~E$_dSt&RgD zgFqQ)MM+Nm*#7C`(CrnEAn)Y3G-mD-WxHl+743;OAOvd5muSh3nJ>S2`D#N*00Lyz zDagN%HaD8j^VHUNTEN$M8!Nq!%T&ZR7d(3uLhz}WACSw=nB~KI49E-7lv-Cbe|HS0 zvtIwQ%^0=(%#p&+Tfs2iUs2cJJ%8U@2pKHx=JI;dBn}(Efj@^{9{`QKs0M(&?}v5K z0t*v=I8XFNU7n=Us7O{9`3tWg7E|GdF`*vPG44gLi(fZAdQ-(-y*MZK^i;38L8WB8 zfT4kk-aAk@lBd`{GA-QlG1Yn@L!WuX(4pXT^AD4E^K0{v?!J$L7@WHxa6)KJ&cI4r znx0>>)U1XUHZIu-rl5GB`El5XqNepOZiQLf;NuA2mrB zXrytUSI&YwY+|TlcRO2iuD?ff(^m@}!8u?Hx)e!m>GVB6aRyG052S=v$7*mGld+;m zo}$qN{ejD-98FcP_G$28fMt-F5Fm~kb!=VAZ6lqW0y446!hTMdT(`8>g%@hb^|AMsX?bB zz)gHt%EBlRce6SF`lCc*C`gC}`bo<>1S=I~VbuXoq+Wc$QLQocg+S7@QC$Px{QGd| zl!RDNXml@Y?Ea2Y3lM=m2Yk82-CpqqQ9*1MmjlH;t4q>IwtcmsU!rr8!{6(g>)lfz zyLZwir|P;Mt@WGmHjLgzQch-}obs0DD0@`?^JuL^qi9u{$9jTcvygs zooIYzT(;o25}fa=-P8b<>nol}vVeYTlSGdL6{1?~wVOA?(n@d9|RLmALe z`!BD*8zzso*X&R-WwN1!!m--cvPY1r9r?s_L*PDd2!JH_fQsw*N`lsU8`@FZ>h%~r zg#8O}ENdA;ULZM9YLqA-x)lQ9bCgGddu@@F0w^9uzd#_0U%k1(e+)%cDr?CC7sh@TGG3#;^Ix H#`pgX4&85` literal 0 HcmV?d00001 diff --git a/hooks/screenshots/readme.txt b/hooks/screenshots/readme.txt new file mode 100644 index 000000000..a87f9057a --- /dev/null +++ b/hooks/screenshots/readme.txt @@ -0,0 +1 @@ +Screenshots pertaining to the readme.md file in hooks folder. \ No newline at end of file diff --git a/linter/yapf_contrib/fixers/fixers_api.py b/linter/yapf_contrib/fixers/fixers_api.py index f1425a9af..96dda284f 100755 --- a/linter/yapf_contrib/fixers/fixers_api.py +++ b/linter/yapf_contrib/fixers/fixers_api.py @@ -13,12 +13,11 @@ # limitations under the License. """Entry point for refactoring via the lib2to3 fixer.""" +import logging from lib2to3 import refactor as lib2to3_refactor from lib2to3.pgen2 import parse as pgen2_parse from lib2to3.pgen2 import tokenize as pgen2_tokenize -import logging - # Our path in the source tree. MODULE_NAME_PREFIX = 'yapf_contrib.fixers.fix_' diff --git a/linter/yapf_contrib/fixers/line_conditional_fix.py b/linter/yapf_contrib/fixers/line_conditional_fix.py index f414a2232..63d02eed8 100755 --- a/linter/yapf_contrib/fixers/line_conditional_fix.py +++ b/linter/yapf_contrib/fixers/line_conditional_fix.py @@ -15,9 +15,7 @@ import collections import re - -from lib2to3 import fixer_base -from lib2to3 import pytree +from lib2to3 import fixer_base, pytree class LineConditionalFix(fixer_base.BaseFix): diff --git a/linter_log.txt b/linter_log.txt new file mode 100644 index 000000000..e69de29bb diff --git a/official/common/__init__.py b/official/common/__init__.py index a25710c22..e04127d3f 100755 --- a/official/common/__init__.py +++ b/official/common/__init__.py @@ -11,5 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - diff --git a/official/common/dataset_fn.py b/official/common/dataset_fn.py index 4ac16a31b..5807a7350 100755 --- a/official/common/dataset_fn.py +++ b/official/common/dataset_fn.py @@ -28,7 +28,7 @@ # ============================================================================== """Utility library for picking an appropriate dataset function.""" -from typing import Any, Callable, Union, Type +from typing import Any, Callable, Type, Union import tensorflow as tf diff --git a/official/common/distribute_utils.py b/official/common/distribute_utils.py index 4dec9346c..62740f69b 100755 --- a/official/common/distribute_utils.py +++ b/official/common/distribute_utils.py @@ -16,6 +16,7 @@ import json import os + import tensorflow as tf diff --git a/official/core/__init__.py b/official/core/__init__.py index e419af524..e04127d3f 100755 --- a/official/core/__init__.py +++ b/official/core/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/core/base_task.py b/official/core/base_task.py index cefa5e916..835aac35c 100755 --- a/official/core/base_task.py +++ b/official/core/base_task.py @@ -16,12 +16,11 @@ import abc from typing import Optional -from absl import logging import tensorflow as tf +from absl import logging from official.core import config_definitions -from official.modeling import optimization -from official.modeling import performance +from official.modeling import optimization, performance OptimizationConfig = optimization.OptimizationConfig RuntimeConfig = config_definitions.RuntimeConfig diff --git a/official/core/base_trainer.py b/official/core/base_trainer.py index 19a39410e..776d13497 100755 --- a/official/core/base_trainer.py +++ b/official/core/base_trainer.py @@ -19,16 +19,14 @@ interchangable and independent on model architectures and tasks. """ -from absl import logging import gin -import orbit import tensorflow as tf +from absl import logging -from official.core import base_task -from official.core import config_definitions +import orbit +from official.core import base_task, config_definitions from official.modeling import optimization - ExperimentConfig = config_definitions.ExperimentConfig TrainerConfig = config_definitions.TrainerConfig diff --git a/official/core/base_trainer_test.py b/official/core/base_trainer_test.py index 5fe7deba8..b79154acf 100755 --- a/official/core/base_trainer_test.py +++ b/official/core/base_trainer_test.py @@ -15,12 +15,12 @@ """Tests for tensorflow_models.core.trainers.trainer.""" # pylint: disable=g-direct-tensorflow-import import os -from absl.testing import parameterized + import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.core import base_trainer as trainer_lib from official.core import config_definitions as cfg from official.core import train_lib diff --git a/official/core/config_definitions.py b/official/core/config_definitions.py index 0e6072431..21369d517 100755 --- a/official/core/config_definitions.py +++ b/official/core/config_definitions.py @@ -14,9 +14,8 @@ """Common configuration settings.""" -from typing import Optional, Sequence, Union - import dataclasses +from typing import Optional, Sequence, Union from official.modeling.hyperparams import base_config from official.modeling.optimization.configs import optimization_config diff --git a/official/core/exp_factory.py b/official/core/exp_factory.py index 7536c0f8f..6cd3dbd37 100755 --- a/official/core/exp_factory.py +++ b/official/core/exp_factory.py @@ -17,7 +17,6 @@ from official.core import config_definitions as cfg from official.core import registry - _REGISTERED_CONFIGS = {} diff --git a/official/core/input_reader.py b/official/core/input_reader.py index 6343b34bb..fb8cb4269 100755 --- a/official/core/input_reader.py +++ b/official/core/input_reader.py @@ -16,9 +16,9 @@ import random from typing import Any, Callable, List, Optional -from absl import logging import tensorflow as tf import tensorflow_datasets as tfds +from absl import logging from official.core import config_definitions as cfg diff --git a/official/core/registry_test.py b/official/core/registry_test.py index eb43dfba9..9e75f99cf 100755 --- a/official/core/registry_test.py +++ b/official/core/registry_test.py @@ -14,11 +14,10 @@ """Tests for registry.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf + from official.core import registry diff --git a/official/core/train_lib.py b/official/core/train_lib.py index 1c73858ad..4dd95879c 100755 --- a/official/core/train_lib.py +++ b/official/core/train_lib.py @@ -17,14 +17,12 @@ import os from typing import Any, Mapping, Tuple +import tensorflow as tf # Import libraries from absl import logging -import orbit -import tensorflow as tf -from official.core import base_task -from official.core import config_definitions -from official.core import train_utils +import orbit +from official.core import base_task, config_definitions, train_utils BestCheckpointExporter = train_utils.BestCheckpointExporter maybe_create_best_ckpt_exporter = train_utils.maybe_create_best_ckpt_exporter diff --git a/official/core/train_lib_test.py b/official/core/train_lib_test.py index 368023876..d4db1bf13 100755 --- a/official/core/train_lib_test.py +++ b/official/core/train_lib_test.py @@ -16,20 +16,16 @@ import json import os -from absl import flags -from absl.testing import flagsaver -from absl.testing import parameterized import tensorflow as tf +from absl import flags +from absl.testing import flagsaver, parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations -from official.common import flags as tfm_flags # pylint: disable=unused-import +from official.common import flags as tfm_flags from official.common import registry_imports # pylint: enable=unused-import -from official.core import task_factory -from official.core import train_lib -from official.core import train_utils +from official.core import task_factory, train_lib, train_utils FLAGS = flags.FLAGS diff --git a/official/core/train_utils.py b/official/core/train_utils.py index 8bc38fd13..ffd5da4b3 100755 --- a/official/core/train_utils.py +++ b/official/core/train_utils.py @@ -14,21 +14,19 @@ """Training utils.""" import copy +import dataclasses import json import os import pprint from typing import Any, Callable, Dict, List, Optional -from absl import logging -import dataclasses import gin -import orbit import tensorflow as tf +from absl import logging -from official.core import base_task -from official.core import base_trainer -from official.core import config_definitions -from official.core import exp_factory +import orbit +from official.core import (base_task, base_trainer, config_definitions, + exp_factory) from official.modeling import hyperparams diff --git a/official/modeling/__init__.py b/official/modeling/__init__.py index e419af524..e04127d3f 100755 --- a/official/modeling/__init__.py +++ b/official/modeling/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/modeling/activations/__init__.py b/official/modeling/activations/__init__.py index 086e1fb97..96ec9cf63 100755 --- a/official/modeling/activations/__init__.py +++ b/official/modeling/activations/__init__.py @@ -16,6 +16,5 @@ from official.modeling.activations.gelu import gelu from official.modeling.activations.relu import relu6 from official.modeling.activations.sigmoid import hard_sigmoid -from official.modeling.activations.swish import hard_swish -from official.modeling.activations.swish import identity -from official.modeling.activations.swish import simple_swish +from official.modeling.activations.swish import (hard_swish, identity, + simple_swish) diff --git a/official/modeling/activations/gelu_test.py b/official/modeling/activations/gelu_test.py index cfe1950d9..2b3b70e25 100755 --- a/official/modeling/activations/gelu_test.py +++ b/official/modeling/activations/gelu_test.py @@ -15,8 +15,9 @@ """Tests for the Gaussian error linear unit.""" import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.modeling import activations diff --git a/official/modeling/activations/relu_test.py b/official/modeling/activations/relu_test.py index 215f189ea..91dbe28ba 100755 --- a/official/modeling/activations/relu_test.py +++ b/official/modeling/activations/relu_test.py @@ -15,9 +15,9 @@ """Tests for the customized Relu activation.""" import tensorflow as tf - from tensorflow.python.keras import \ - keras_parameterized # pylint: disable=g-direct-tensorflow-import + keras_parameterized # pylint: disable=g-direct-tensorflow-import + from official.modeling import activations diff --git a/official/modeling/activations/sigmoid_test.py b/official/modeling/activations/sigmoid_test.py index 6aad90ef3..6cdbcd93d 100755 --- a/official/modeling/activations/sigmoid_test.py +++ b/official/modeling/activations/sigmoid_test.py @@ -16,9 +16,9 @@ import numpy as np import tensorflow as tf - from tensorflow.python.keras import \ - keras_parameterized # pylint: disable=g-direct-tensorflow-import + keras_parameterized # pylint: disable=g-direct-tensorflow-import + from official.modeling import activations diff --git a/official/modeling/activations/swish_test.py b/official/modeling/activations/swish_test.py index 3cb9495d8..72c98ea7a 100755 --- a/official/modeling/activations/swish_test.py +++ b/official/modeling/activations/swish_test.py @@ -15,8 +15,9 @@ """Tests for the customized Swish activation.""" import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.modeling import activations diff --git a/official/modeling/hyperparams/__init__.py b/official/modeling/hyperparams/__init__.py index bcbc0aedd..1cad5540c 100755 --- a/official/modeling/hyperparams/__init__.py +++ b/official/modeling/hyperparams/__init__.py @@ -13,8 +13,6 @@ # limitations under the License. """Hyperparams package definition.""" -# pylint: disable=g-multiple-import from official.modeling.hyperparams.base_config import * from official.modeling.hyperparams.oneof import * from official.modeling.hyperparams.params_dict import * - diff --git a/official/modeling/hyperparams/base_config.py b/official/modeling/hyperparams/base_config.py index 871f76df9..d0f2a6f56 100755 --- a/official/modeling/hyperparams/base_config.py +++ b/official/modeling/hyperparams/base_config.py @@ -15,13 +15,13 @@ """Base configurations to standardize experiments.""" import copy +import dataclasses import functools from typing import Any, List, Mapping, Optional, Type -from absl import logging -import dataclasses import tensorflow as tf import yaml +from absl import logging from official.modeling.hyperparams import params_dict diff --git a/official/modeling/hyperparams/base_config_test.py b/official/modeling/hyperparams/base_config_test.py index 3e64ec532..b27be5c4f 100755 --- a/official/modeling/hyperparams/base_config_test.py +++ b/official/modeling/hyperparams/base_config_test.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import dataclasses import pprint from typing import List, Tuple -from absl.testing import parameterized -import dataclasses import tensorflow as tf +from absl.testing import parameterized + from official.modeling.hyperparams import base_config diff --git a/official/modeling/hyperparams/oneof.py b/official/modeling/hyperparams/oneof.py index 61591496e..311d8c059 100755 --- a/official/modeling/hyperparams/oneof.py +++ b/official/modeling/hyperparams/oneof.py @@ -14,9 +14,9 @@ """Config class that supports oneof functionality.""" +import dataclasses from typing import Optional -import dataclasses from official.modeling.hyperparams import base_config diff --git a/official/modeling/hyperparams/oneof_test.py b/official/modeling/hyperparams/oneof_test.py index 2cde73c15..b0f82ef56 100755 --- a/official/modeling/hyperparams/oneof_test.py +++ b/official/modeling/hyperparams/oneof_test.py @@ -13,9 +13,10 @@ # limitations under the License. import dataclasses + import tensorflow as tf -from official.modeling.hyperparams import base_config -from official.modeling.hyperparams import oneof + +from official.modeling.hyperparams import base_config, oneof @dataclasses.dataclass diff --git a/official/modeling/multitask/__init__.py b/official/modeling/multitask/__init__.py index e419af524..e04127d3f 100755 --- a/official/modeling/multitask/__init__.py +++ b/official/modeling/multitask/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/modeling/multitask/base_model.py b/official/modeling/multitask/base_model.py index 976b0d8e3..c30d7187c 100755 --- a/official/modeling/multitask/base_model.py +++ b/official/modeling/multitask/base_model.py @@ -28,7 +28,7 @@ # limitations under the License. # ============================================================================== """Abstraction of multi-task model.""" -from typing import Text, Dict +from typing import Dict, Text import tensorflow as tf diff --git a/official/modeling/multitask/configs.py b/official/modeling/multitask/configs.py index faaf41ae4..5e776e268 100755 --- a/official/modeling/multitask/configs.py +++ b/official/modeling/multitask/configs.py @@ -13,9 +13,8 @@ # limitations under the License. """Configuration definitions for multi-task training.""" -from typing import Optional, Tuple - import dataclasses +from typing import Optional, Tuple from official.core import config_definitions as cfg from official.modeling import hyperparams diff --git a/official/modeling/multitask/evaluator.py b/official/modeling/multitask/evaluator.py index 806f5a6d3..054f10d65 100755 --- a/official/modeling/multitask/evaluator.py +++ b/official/modeling/multitask/evaluator.py @@ -17,13 +17,13 @@ The evaluator implements the Orbit `AbstractEvaluator` interface. """ from typing import Optional, Union + import gin -import orbit import tensorflow as tf +import orbit from official.core import train_utils -from official.modeling.multitask import base_model -from official.modeling.multitask import multitask +from official.modeling.multitask import base_model, multitask @gin.configurable diff --git a/official/modeling/multitask/evaluator_test.py b/official/modeling/multitask/evaluator_test.py index fa141292d..f9fbf9793 100755 --- a/official/modeling/multitask/evaluator_test.py +++ b/official/modeling/multitask/evaluator_test.py @@ -13,16 +13,14 @@ # limitations under the License. """Tests for multitask.evaluator.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.core import base_task from official.core import config_definitions as cfg -from official.modeling.multitask import evaluator -from official.modeling.multitask import multitask +from official.modeling.multitask import evaluator, multitask def all_strategy_combinations(): diff --git a/official/modeling/multitask/multitask.py b/official/modeling/multitask/multitask.py index 701d7141d..9b10b011e 100755 --- a/official/modeling/multitask/multitask.py +++ b/official/modeling/multitask/multitask.py @@ -17,9 +17,8 @@ from typing import Dict, List, Optional, Text, Union import tensorflow as tf -from official.core import base_task -from official.core import config_definitions -from official.core import task_factory + +from official.core import base_task, config_definitions, task_factory from official.modeling import optimization from official.modeling.multitask import configs diff --git a/official/modeling/multitask/train_lib.py b/official/modeling/multitask/train_lib.py index abfb9a6b1..4759930a9 100755 --- a/official/modeling/multitask/train_lib.py +++ b/official/modeling/multitask/train_lib.py @@ -15,9 +15,11 @@ """Multitask training driver library.""" # pytype: disable=attribute-error import os + +import tensorflow as tf from absl import logging + import orbit -import tensorflow as tf from official.core import base_task from official.core import base_trainer as core_lib from official.core import train_utils diff --git a/official/modeling/optimization/SGDAccumulated.py b/official/modeling/optimization/SGDAccumulated.py index 34b33e6c2..9ea6e9cc6 100755 --- a/official/modeling/optimization/SGDAccumulated.py +++ b/official/modeling/optimization/SGDAccumulated.py @@ -1,7 +1,9 @@ import tensorflow as tf +from tensorflow.python import ops from tensorflow.python.keras.optimizer_v2.optimizer_v2 import OptimizerV2 -from tensorflow.python.ops import math_ops, state_ops, control_flow_ops, array_ops -from tensorflow.python import ops +from tensorflow.python.ops import (array_ops, control_flow_ops, math_ops, + state_ops) + # from tensorflow.python.keras.utils import control_flow_util # from tensorflow.python.keras import backend_config @@ -216,4 +218,4 @@ def get_config(self): # train_data = task.build_inputs(task.task_config.train_data) # validation_data = task.build_inputs(task.task_config.train_data) -# model.compile(optimizer = optimizer) \ No newline at end of file +# model.compile(optimizer = optimizer) diff --git a/official/modeling/optimization/SGDAccumulated__.py b/official/modeling/optimization/SGDAccumulated__.py index 4ceaad57f..04b07a530 100755 --- a/official/modeling/optimization/SGDAccumulated__.py +++ b/official/modeling/optimization/SGDAccumulated__.py @@ -1,7 +1,9 @@ import tensorflow as tf -from tensorflow.python.ops import math_ops, state_ops, control_flow_ops, array_ops -from tensorflow.python import ops +from tensorflow.python import ops from tensorflow.python.keras.utils import control_flow_util +from tensorflow.python.ops import (array_ops, control_flow_ops, math_ops, + state_ops) + # from tensorflow.python.keras import backend_config @@ -181,4 +183,4 @@ def from_config(cls, config, custom_objects=None): # train_data = task.build_inputs(task.task_config.train_data) # validation_data = task.build_inputs(task.task_config.train_data) -# model.compile(optimizer = optimizer) \ No newline at end of file +# model.compile(optimizer = optimizer) diff --git a/official/modeling/optimization/__init__.py b/official/modeling/optimization/__init__.py index 9829683a2..6bd63845a 100755 --- a/official/modeling/optimization/__init__.py +++ b/official/modeling/optimization/__init__.py @@ -14,9 +14,9 @@ """Optimization package definition.""" -# pylint: disable=wildcard-import from official.modeling.optimization.configs.learning_rate_config import * from official.modeling.optimization.configs.optimization_config import * from official.modeling.optimization.configs.optimizer_config import * -from official.modeling.optimization.ema_optimizer import ExponentialMovingAverage +from official.modeling.optimization.ema_optimizer import \ + ExponentialMovingAverage from official.modeling.optimization.optimizer_factory import OptimizerFactory diff --git a/official/modeling/optimization/configs/__init__.py b/official/modeling/optimization/configs/__init__.py index e419af524..e04127d3f 100755 --- a/official/modeling/optimization/configs/__init__.py +++ b/official/modeling/optimization/configs/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/official/modeling/optimization/configs/learning_rate_config.py b/official/modeling/optimization/configs/learning_rate_config.py index 520a0b961..6903ad077 100755 --- a/official/modeling/optimization/configs/learning_rate_config.py +++ b/official/modeling/optimization/configs/learning_rate_config.py @@ -13,9 +13,9 @@ # limitations under the License. """Dataclasses for learning rate schedule config.""" +import dataclasses from typing import List, Optional -import dataclasses from official.modeling.hyperparams import base_config diff --git a/official/modeling/optimization/configs/optimization_config.py b/official/modeling/optimization/configs/optimization_config.py index 09db8a158..4ef9f4167 100755 --- a/official/modeling/optimization/configs/optimization_config.py +++ b/official/modeling/optimization/configs/optimization_config.py @@ -18,13 +18,12 @@ It also has two helper functions get_optimizer_config, and get_lr_config from an OptimizationConfig class. """ -from typing import Optional - import dataclasses +from typing import Optional -from official.modeling.hyperparams import base_config -from official.modeling.hyperparams import oneof -from official.modeling.optimization.configs import learning_rate_config as lr_cfg +from official.modeling.hyperparams import base_config, oneof +from official.modeling.optimization.configs import \ + learning_rate_config as lr_cfg from official.modeling.optimization.configs import optimizer_config as opt_cfg diff --git a/official/modeling/optimization/configs/optimization_config_test.py b/official/modeling/optimization/configs/optimization_config_test.py index 02b99f592..c95525cb8 100755 --- a/official/modeling/optimization/configs/optimization_config_test.py +++ b/official/modeling/optimization/configs/optimization_config_test.py @@ -16,7 +16,8 @@ import tensorflow as tf -from official.modeling.optimization.configs import learning_rate_config as lr_cfg +from official.modeling.optimization.configs import \ + learning_rate_config as lr_cfg from official.modeling.optimization.configs import optimization_config from official.modeling.optimization.configs import optimizer_config as opt_cfg diff --git a/official/modeling/optimization/configs/optimizer_config.py b/official/modeling/optimization/configs/optimizer_config.py index 16e216208..d263fa5f1 100755 --- a/official/modeling/optimization/configs/optimizer_config.py +++ b/official/modeling/optimization/configs/optimizer_config.py @@ -13,9 +13,9 @@ # limitations under the License. """Dataclasses for optimizer configs.""" +import dataclasses from typing import List, Optional -import dataclasses from official.modeling.hyperparams import base_config diff --git a/official/modeling/optimization/ema_optimizer.py b/official/modeling/optimization/ema_optimizer.py index a20bd7c80..ee5e57542 100755 --- a/official/modeling/optimization/ema_optimizer.py +++ b/official/modeling/optimization/ema_optimizer.py @@ -14,7 +14,7 @@ """Exponential moving average optimizer.""" -from typing import Text, List +from typing import List, Text import tensorflow as tf diff --git a/official/modeling/optimization/lr_schedule.py b/official/modeling/optimization/lr_schedule.py index b5c1ef349..dcc2a8b70 100755 --- a/official/modeling/optimization/lr_schedule.py +++ b/official/modeling/optimization/lr_schedule.py @@ -14,7 +14,7 @@ """Learning rate schedule classes.""" -from typing import Mapping, Any, Union, Optional +from typing import Any, Mapping, Optional, Union import tensorflow as tf diff --git a/official/modeling/optimization/optimizer_factory.py b/official/modeling/optimization/optimizer_factory.py index 27d74d71e..5fe9bb314 100755 --- a/official/modeling/optimization/optimizer_factory.py +++ b/official/modeling/optimization/optimizer_factory.py @@ -15,14 +15,14 @@ """Optimizer factory class.""" from typing import Callable, Union - import gin import tensorflow as tf import tensorflow_addons.optimizers as tfa_optimizers -from official.modeling.optimization import ema_optimizer, SGDAccumulated -from official.modeling.optimization import lr_schedule -from official.modeling.optimization.configs import optimization_config as opt_cfg +from official.modeling.optimization import (SGDAccumulated, ema_optimizer, + lr_schedule) +from official.modeling.optimization.configs import \ + optimization_config as opt_cfg from official.nlp import optimization as nlp_optimization OPTIMIZERS_CLS = { diff --git a/official/modeling/optimization/optimizer_factory_test.py b/official/modeling/optimization/optimizer_factory_test.py index 5e670c296..0899c9e56 100755 --- a/official/modeling/optimization/optimizer_factory_test.py +++ b/official/modeling/optimization/optimizer_factory_test.py @@ -13,9 +13,9 @@ # limitations under the License. """Tests for optimizer_factory.py.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.modeling.optimization import optimizer_factory from official.modeling.optimization.configs import optimization_config diff --git a/official/modeling/progressive/policies.py b/official/modeling/progressive/policies.py index cf9598d97..4b7eba5c4 100755 --- a/official/modeling/progressive/policies.py +++ b/official/modeling/progressive/policies.py @@ -19,11 +19,13 @@ """ import abc -from typing import Any, Mapping -from absl import logging import dataclasses +from typing import Any, Mapping + import six import tensorflow as tf +from absl import logging + from official.modeling.hyperparams import base_config from official.modeling.progressive import utils diff --git a/official/modeling/progressive/train.py b/official/modeling/progressive/train.py index dc1fb5d9e..ef43137c6 100755 --- a/official/modeling/progressive/train.py +++ b/official/modeling/progressive/train.py @@ -14,17 +14,15 @@ """TFM binary for the progressive trainer.""" -from absl import app -from absl import flags import gin +from absl import app, flags -from official.common import distribute_utils -# pylint: disable=unused-import -from official.common import registry_imports # pylint: enable=unused-import +# pylint: disable=unused-import +from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_utils +from official.common import registry_imports +from official.core import task_factory, train_utils from official.modeling import performance from official.modeling.progressive import train_lib diff --git a/official/modeling/progressive/train_lib.py b/official/modeling/progressive/train_lib.py index a096aebf1..2dd5e904c 100755 --- a/official/modeling/progressive/train_lib.py +++ b/official/modeling/progressive/train_lib.py @@ -22,12 +22,12 @@ import os from typing import Any, Mapping, Tuple +import tensorflow as tf # Import libraries from absl import logging + import orbit -import tensorflow as tf -from official.core import base_task -from official.core import config_definitions +from official.core import base_task, config_definitions from official.core import train_lib as base_train_lib from official.modeling.progressive import trainer as prog_trainer_lib diff --git a/official/modeling/progressive/train_lib_test.py b/official/modeling/progressive/train_lib_test.py index 178fc6df7..6ccd15d53 100755 --- a/official/modeling/progressive/train_lib_test.py +++ b/official/modeling/progressive/train_lib_test.py @@ -13,26 +13,24 @@ # limitations under the License. """Tests for the progressive train_lib.""" +import dataclasses import os +import tensorflow as tf from absl import flags from absl.testing import parameterized -import dataclasses -import orbit -import tensorflow as tf +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations -from official.common import flags as tfm_flags +import orbit # pylint: disable=unused-import +from official.common import flags as tfm_flags from official.common import registry_imports # pylint: enable=unused-import from official.core import config_definitions as cfg from official.core import task_factory from official.modeling import optimization from official.modeling.hyperparams import params_dict -from official.modeling.progressive import policies -from official.modeling.progressive import train_lib +from official.modeling.progressive import policies, train_lib from official.modeling.progressive import trainer as prog_trainer_lib from official.utils.testing import mock_task diff --git a/official/modeling/progressive/trainer.py b/official/modeling/progressive/trainer.py index 6c970ecbc..46f19a9a1 100755 --- a/official/modeling/progressive/trainer.py +++ b/official/modeling/progressive/trainer.py @@ -18,21 +18,20 @@ `StandardEvaluable` interfaces. Trainers inside this project should be interchangable and independent on model architectures and tasks. """ +import dataclasses import os from typing import Any, Optional +import gin +import tensorflow as tf # Import libraries from absl import logging -import dataclasses -import gin import orbit -import tensorflow as tf from official.core import base_task from official.core import base_trainer as trainer_lib from official.core import config_definitions -from official.modeling.progressive import policies -from official.modeling.progressive import utils +from official.modeling.progressive import policies, utils ExperimentConfig = config_definitions.ExperimentConfig diff --git a/official/modeling/progressive/trainer_test.py b/official/modeling/progressive/trainer_test.py index 7d4ab3e22..ede4b696a 100755 --- a/official/modeling/progressive/trainer_test.py +++ b/official/modeling/progressive/trainer_test.py @@ -16,12 +16,11 @@ # pylint: disable=g-direct-tensorflow-import import os -from absl.testing import parameterized -import orbit import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations +import orbit from official.core import config_definitions as cfg from official.modeling import optimization from official.modeling.progressive import policies diff --git a/official/modeling/progressive/utils.py b/official/modeling/progressive/utils.py index 192170cb8..105634cf0 100755 --- a/official/modeling/progressive/utils.py +++ b/official/modeling/progressive/utils.py @@ -14,9 +14,8 @@ """Util classes and functions.""" -from absl import logging import tensorflow as tf - +from absl import logging # pylint: disable=g-direct-tensorflow-import from tensorflow.python.training.tracking import tracking diff --git a/official/modeling/tf_utils.py b/official/modeling/tf_utils.py index ca9e2362f..254a69257 100755 --- a/official/modeling/tf_utils.py +++ b/official/modeling/tf_utils.py @@ -16,8 +16,8 @@ import six import tensorflow as tf - from tensorflow.python.util import deprecation + from official.modeling import activations diff --git a/official/nlp/albert/export_albert_tfhub.py b/official/nlp/albert/export_albert_tfhub.py index 8b48476f3..b7b8247e8 100755 --- a/official/nlp/albert/export_albert_tfhub.py +++ b/official/nlp/albert/export_albert_tfhub.py @@ -14,12 +14,12 @@ # ============================================================================== """A script to export the ALBERT core model as a TF-Hub SavedModel.""" -# Import libraries -from absl import app -from absl import flags -import tensorflow as tf from typing import Text +import tensorflow as tf +# Import libraries +from absl import app, flags + from official.nlp.albert import configs from official.nlp.bert import bert_models diff --git a/official/nlp/albert/export_albert_tfhub_test.py b/official/nlp/albert/export_albert_tfhub_test.py index 8e87f325a..a55845c56 100755 --- a/official/nlp/albert/export_albert_tfhub_test.py +++ b/official/nlp/albert/export_albert_tfhub_test.py @@ -16,12 +16,10 @@ import os import numpy as np - import tensorflow as tf import tensorflow_hub as hub -from official.nlp.albert import configs -from official.nlp.albert import export_albert_tfhub +from official.nlp.albert import configs, export_albert_tfhub class ExportAlbertTfhubTest(tf.test.TestCase): diff --git a/official/nlp/albert/run_classifier.py b/official/nlp/albert/run_classifier.py index 5a0b5a109..7d4376335 100755 --- a/official/nlp/albert/run_classifier.py +++ b/official/nlp/albert/run_classifier.py @@ -16,17 +16,16 @@ import json import os -# Import libraries -from absl import app -from absl import flags -from absl import logging + import tensorflow as tf +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils from official.nlp.albert import configs as albert_configs from official.nlp.bert import bert_models from official.nlp.bert import run_classifier as run_classifier_bert - FLAGS = flags.FLAGS diff --git a/official/nlp/albert/run_squad.py b/official/nlp/albert/run_squad.py index 154e19e38..0257db473 100755 --- a/official/nlp/albert/run_squad.py +++ b/official/nlp/albert/run_squad.py @@ -18,15 +18,13 @@ import os import time -# Import libraries -from absl import app -from absl import flags -from absl import logging import tensorflow as tf +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils from official.nlp.albert import configs as albert_configs -from official.nlp.bert import run_squad_helper -from official.nlp.bert import tokenization +from official.nlp.bert import run_squad_helper, tokenization from official.nlp.data import squad_lib_sp flags.DEFINE_string( diff --git a/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py b/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py index 64c2dc222..3a0ea5b66 100755 --- a/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py +++ b/official/nlp/albert/tf2_albert_encoder_checkpoint_converter.py @@ -19,15 +19,13 @@ """ import os -from absl import app -from absl import flags - import tensorflow as tf +from absl import app, flags + from official.modeling import tf_utils from official.nlp.albert import configs from official.nlp.bert import tf1_checkpoint_converter_lib -from official.nlp.modeling import models -from official.nlp.modeling import networks +from official.nlp.modeling import models, networks FLAGS = flags.FLAGS diff --git a/official/nlp/bert/__init__.py b/official/nlp/bert/__init__.py index 8b1378917..e69de29bb 100755 --- a/official/nlp/bert/__init__.py +++ b/official/nlp/bert/__init__.py @@ -1 +0,0 @@ - diff --git a/official/nlp/bert/bert_models.py b/official/nlp/bert/bert_models.py index c4f73ffbd..1dbaee579 100755 --- a/official/nlp/bert/bert_models.py +++ b/official/nlp/bert/bert_models.py @@ -21,8 +21,7 @@ from official.modeling import tf_utils from official.nlp.albert import configs as albert_configs from official.nlp.bert import configs -from official.nlp.modeling import models -from official.nlp.modeling import networks +from official.nlp.modeling import models, networks class BertPretrainLossAndMetricLayer(tf.keras.layers.Layer): diff --git a/official/nlp/bert/common_flags.py b/official/nlp/bert/common_flags.py index 22292673d..82aa264e5 100755 --- a/official/nlp/bert/common_flags.py +++ b/official/nlp/bert/common_flags.py @@ -14,8 +14,8 @@ # ============================================================================== """Defining common flags used across all BERT models/applications.""" -from absl import flags import tensorflow as tf +from absl import flags from official.utils import hyperparams_flags from official.utils.flags import core as flags_core diff --git a/official/nlp/bert/export_tfhub.py b/official/nlp/bert/export_tfhub.py index bf40bd066..14de4921c 100755 --- a/official/nlp/bert/export_tfhub.py +++ b/official/nlp/bert/export_tfhub.py @@ -14,14 +14,13 @@ # ============================================================================== """A script to export the BERT core model as a TF-Hub SavedModel.""" -# Import libraries -from absl import app -from absl import flags -from absl import logging -import tensorflow as tf from typing import Text -from official.nlp.bert import bert_models -from official.nlp.bert import configs + +import tensorflow as tf +# Import libraries +from absl import app, flags, logging + +from official.nlp.bert import bert_models, configs FLAGS = flags.FLAGS diff --git a/official/nlp/bert/export_tfhub_test.py b/official/nlp/bert/export_tfhub_test.py index 98462ce92..ea7d68fa5 100755 --- a/official/nlp/bert/export_tfhub_test.py +++ b/official/nlp/bert/export_tfhub_test.py @@ -16,13 +16,12 @@ import os -from absl.testing import parameterized import numpy as np import tensorflow as tf import tensorflow_hub as hub +from absl.testing import parameterized -from official.nlp.bert import configs -from official.nlp.bert import export_tfhub +from official.nlp.bert import configs, export_tfhub class ExportTfhubTest(tf.test.TestCase, parameterized.TestCase): diff --git a/official/nlp/bert/model_saving_utils.py b/official/nlp/bert/model_saving_utils.py index 059fa8080..2464f4991 100755 --- a/official/nlp/bert/model_saving_utils.py +++ b/official/nlp/bert/model_saving_utils.py @@ -15,10 +15,10 @@ """Utilities to save models.""" import os +import typing -from absl import logging import tensorflow as tf -import typing +from absl import logging def export_bert_model(model_export_path: typing.Text, diff --git a/official/nlp/bert/model_training_utils.py b/official/nlp/bert/model_training_utils.py index 7d03e0ffc..fd21795b7 100755 --- a/official/nlp/bert/model_training_utils.py +++ b/official/nlp/bert/model_training_utils.py @@ -18,9 +18,10 @@ import os import tempfile -from absl import logging import tensorflow as tf +from absl import logging from tensorflow.python.util import deprecation + from official.common import distribute_utils from official.staging.training import grad_utils diff --git a/official/nlp/bert/model_training_utils_test.py b/official/nlp/bert/model_training_utils_test.py index 32b88a36c..131f415ca 100755 --- a/official/nlp/bert/model_training_utils_test.py +++ b/official/nlp/bert/model_training_utils_test.py @@ -16,18 +16,14 @@ import os -from absl import logging -from absl.testing import flagsaver -from absl.testing import parameterized -from absl.testing.absltest import mock import numpy as np import tensorflow as tf +from absl import logging +from absl.testing import flagsaver, parameterized +from absl.testing.absltest import mock +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations -from official.nlp.bert import common_flags -from official.nlp.bert import model_training_utils - +from official.nlp.bert import common_flags, model_training_utils common_flags.define_common_bert_flags() diff --git a/official/nlp/bert/run_classifier.py b/official/nlp/bert/run_classifier.py index 1a5a9462e..bf9ab78d5 100755 --- a/official/nlp/bert/run_classifier.py +++ b/official/nlp/bert/run_classifier.py @@ -19,20 +19,17 @@ import math import os -# Import libraries -from absl import app -from absl import flags -from absl import logging import gin import tensorflow as tf +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils from official.modeling import performance from official.nlp import optimization -from official.nlp.bert import bert_models -from official.nlp.bert import common_flags +from official.nlp.bert import bert_models, common_flags from official.nlp.bert import configs as bert_configs -from official.nlp.bert import input_pipeline -from official.nlp.bert import model_saving_utils +from official.nlp.bert import input_pipeline, model_saving_utils from official.utils.misc import keras_utils flags.DEFINE_enum( diff --git a/official/nlp/bert/run_pretraining.py b/official/nlp/bert/run_pretraining.py index 76d669f19..f00bd070c 100755 --- a/official/nlp/bert/run_pretraining.py +++ b/official/nlp/bert/run_pretraining.py @@ -14,21 +14,16 @@ # ============================================================================== """Run masked LM/next sentence pre-training for BERT in TF 2.x.""" -# Import libraries -from absl import app -from absl import flags -from absl import logging import gin import tensorflow as tf +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils from official.modeling import performance from official.nlp import optimization -from official.nlp.bert import bert_models -from official.nlp.bert import common_flags -from official.nlp.bert import configs -from official.nlp.bert import input_pipeline -from official.nlp.bert import model_training_utils - +from official.nlp.bert import (bert_models, common_flags, configs, + input_pipeline, model_training_utils) flags.DEFINE_string('input_files', None, 'File path to retrieve training data for pre-training.') diff --git a/official/nlp/bert/run_squad.py b/official/nlp/bert/run_squad.py index ca1e766f2..7e12d5b2b 100755 --- a/official/nlp/bert/run_squad.py +++ b/official/nlp/bert/run_squad.py @@ -18,20 +18,17 @@ import os import time -# Import libraries -from absl import app -from absl import flags -from absl import logging import gin import tensorflow as tf +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils from official.nlp.bert import configs as bert_configs -from official.nlp.bert import run_squad_helper -from official.nlp.bert import tokenization +from official.nlp.bert import run_squad_helper, tokenization from official.nlp.data import squad_lib as squad_lib_wp from official.utils.misc import keras_utils - flags.DEFINE_string('vocab_file', None, 'The vocabulary file that the BERT model was trained on.') diff --git a/official/nlp/bert/run_squad_helper.py b/official/nlp/bert/run_squad_helper.py index 3c3630cb0..a1301a4ec 100755 --- a/official/nlp/bert/run_squad_helper.py +++ b/official/nlp/bert/run_squad_helper.py @@ -18,18 +18,14 @@ import json import os -from absl import flags -from absl import logging import tensorflow as tf +from absl import flags, logging + from official.modeling import performance from official.nlp import optimization -from official.nlp.bert import bert_models -from official.nlp.bert import common_flags -from official.nlp.bert import input_pipeline -from official.nlp.bert import model_saving_utils -from official.nlp.bert import model_training_utils -from official.nlp.bert import squad_evaluate_v1_1 -from official.nlp.bert import squad_evaluate_v2_0 +from official.nlp.bert import (bert_models, common_flags, input_pipeline, + model_saving_utils, model_training_utils, + squad_evaluate_v1_1, squad_evaluate_v2_0) from official.nlp.data import squad_lib_sp from official.utils.misc import keras_utils diff --git a/official/nlp/bert/serving.py b/official/nlp/bert/serving.py index 4085fb822..7d84e19b1 100755 --- a/official/nlp/bert/serving.py +++ b/official/nlp/bert/serving.py @@ -15,12 +15,10 @@ # ============================================================================== """Examples of SavedModel export for tf-serving.""" -from absl import app -from absl import flags import tensorflow as tf +from absl import app, flags -from official.nlp.bert import bert_models -from official.nlp.bert import configs +from official.nlp.bert import bert_models, configs flags.DEFINE_integer( "sequence_length", None, "Sequence length to parse the tf.Example. If " diff --git a/official/nlp/bert/squad_evaluate_v1_1.py b/official/nlp/bert/squad_evaluate_v1_1.py index 1cc4430bc..eb3f8cc04 100755 --- a/official/nlp/bert/squad_evaluate_v1_1.py +++ b/official/nlp/bert/squad_evaluate_v1_1.py @@ -26,9 +26,10 @@ import re import string +from absl import logging + # pylint: disable=g-bad-import-order -from absl import logging # pylint: enable=g-bad-import-order diff --git a/official/nlp/bert/tf2_encoder_checkpoint_converter.py b/official/nlp/bert/tf2_encoder_checkpoint_converter.py index e81c0d6a5..87d136fd4 100755 --- a/official/nlp/bert/tf2_encoder_checkpoint_converter.py +++ b/official/nlp/bert/tf2_encoder_checkpoint_converter.py @@ -21,15 +21,12 @@ import os -from absl import app -from absl import flags - import tensorflow as tf +from absl import app, flags + from official.modeling import tf_utils -from official.nlp.bert import configs -from official.nlp.bert import tf1_checkpoint_converter_lib -from official.nlp.modeling import models -from official.nlp.modeling import networks +from official.nlp.bert import configs, tf1_checkpoint_converter_lib +from official.nlp.modeling import models, networks FLAGS = flags.FLAGS diff --git a/official/nlp/bert/tokenization.py b/official/nlp/bert/tokenization.py index 684be103f..16e09962b 100755 --- a/official/nlp/bert/tokenization.py +++ b/official/nlp/bert/tokenization.py @@ -23,11 +23,10 @@ import re import unicodedata +import sentencepiece as spm import six import tensorflow as tf -import sentencepiece as spm - SPIECE_UNDERLINE = "▁" diff --git a/official/nlp/configs/bert.py b/official/nlp/configs/bert.py index e0a64b78b..bafd847b8 100755 --- a/official/nlp/configs/bert.py +++ b/official/nlp/configs/bert.py @@ -17,9 +17,8 @@ Includes configurations and instantiation methods. """ -from typing import List, Optional, Text - import dataclasses +from typing import List, Optional, Text from official.modeling.hyperparams import base_config from official.nlp.configs import encoders diff --git a/official/nlp/configs/electra.py b/official/nlp/configs/electra.py index 527cde521..eb0308c75 100755 --- a/official/nlp/configs/electra.py +++ b/official/nlp/configs/electra.py @@ -14,13 +14,11 @@ # limitations under the License. # ============================================================================== """ELECTRA model configurations and instantiation methods.""" -from typing import List - import dataclasses +from typing import List from official.modeling.hyperparams import base_config -from official.nlp.configs import bert -from official.nlp.configs import encoders +from official.nlp.configs import bert, encoders @dataclasses.dataclass diff --git a/official/nlp/configs/encoders.py b/official/nlp/configs/encoders.py index 9d2807bd6..481de3996 100755 --- a/official/nlp/configs/encoders.py +++ b/official/nlp/configs/encoders.py @@ -17,15 +17,14 @@ Includes configurations and factory methods. """ +import dataclasses from typing import Optional -from absl import logging -import dataclasses import gin import tensorflow as tf +from absl import logging -from official.modeling import hyperparams -from official.modeling import tf_utils +from official.modeling import hyperparams, tf_utils from official.nlp.modeling import networks from official.nlp.projects.bigbird import encoder as bigbird_encoder diff --git a/official/nlp/configs/experiment_configs.py b/official/nlp/configs/experiment_configs.py index 4f274ce41..a4dd100af 100755 --- a/official/nlp/configs/experiment_configs.py +++ b/official/nlp/configs/experiment_configs.py @@ -15,5 +15,5 @@ # ============================================================================== """Experiments definition.""" # pylint: disable=unused-import -from official.nlp.configs import finetuning_experiments -from official.nlp.configs import pretraining_experiments +from official.nlp.configs import (finetuning_experiments, + pretraining_experiments) diff --git a/official/nlp/configs/finetuning_experiments.py b/official/nlp/configs/finetuning_experiments.py index 9a8d25992..2366af89e 100755 --- a/official/nlp/configs/finetuning_experiments.py +++ b/official/nlp/configs/finetuning_experiments.py @@ -17,10 +17,9 @@ from official.core import config_definitions as cfg from official.core import exp_factory from official.modeling import optimization -from official.nlp.data import question_answering_dataloader -from official.nlp.data import sentence_prediction_dataloader -from official.nlp.tasks import question_answering -from official.nlp.tasks import sentence_prediction +from official.nlp.data import (question_answering_dataloader, + sentence_prediction_dataloader) +from official.nlp.tasks import question_answering, sentence_prediction @exp_factory.register_config_factory('bert/sentence_prediction') diff --git a/official/nlp/continuous_finetune_lib.py b/official/nlp/continuous_finetune_lib.py index 0554d3115..c387b2fb9 100755 --- a/official/nlp/continuous_finetune_lib.py +++ b/official/nlp/continuous_finetune_lib.py @@ -18,17 +18,14 @@ import time from typing import Any, Mapping, Optional -from absl import logging import tensorflow as tf +from absl import logging from official.common import distribute_utils -from official.core import config_definitions -from official.core import task_factory -from official.core import train_lib -from official.core import train_utils +from official.core import (config_definitions, task_factory, train_lib, + train_utils) from official.modeling import performance -from official.modeling.multitask import configs -from official.modeling.multitask import multitask +from official.modeling.multitask import configs, multitask from official.modeling.multitask import train_lib as multitask_train_lib diff --git a/official/nlp/continuous_finetune_lib_test.py b/official/nlp/continuous_finetune_lib_test.py index a478b2ce9..17845879a 100755 --- a/official/nlp/continuous_finetune_lib_test.py +++ b/official/nlp/continuous_finetune_lib_test.py @@ -14,18 +14,15 @@ # ============================================================================== import os -from absl import flags -from absl.testing import flagsaver -from absl.testing import parameterized import tensorflow as tf +from absl import flags +from absl.testing import flagsaver, parameterized -# pylint: disable=unused-import -from official.common import registry_imports # pylint: enable=unused-import +# pylint: disable=unused-import from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib -from official.core import train_utils +from official.common import registry_imports +from official.core import task_factory, train_lib, train_utils from official.nlp import continuous_finetune_lib FLAGS = flags.FLAGS diff --git a/official/nlp/data/classifier_data_lib.py b/official/nlp/data/classifier_data_lib.py index 80d4fda09..322172548 100755 --- a/official/nlp/data/classifier_data_lib.py +++ b/official/nlp/data/classifier_data_lib.py @@ -20,9 +20,9 @@ import json import os -from absl import logging import tensorflow as tf import tensorflow_datasets as tfds +from absl import logging from official.nlp.bert import tokenization diff --git a/official/nlp/data/create_finetuning_data.py b/official/nlp/data/create_finetuning_data.py index 706757d58..c0b4f478a 100755 --- a/official/nlp/data/create_finetuning_data.py +++ b/official/nlp/data/create_finetuning_data.py @@ -18,18 +18,16 @@ import json import os -# Import libraries -from absl import app -from absl import flags import tensorflow as tf +# Import libraries +from absl import app, flags + from official.nlp.bert import tokenization -from official.nlp.data import classifier_data_lib -from official.nlp.data import sentence_retrieval_lib +# sentence-piece tokenizer based squad_lib # word-piece tokenizer based squad_lib +from official.nlp.data import classifier_data_lib, sentence_retrieval_lib from official.nlp.data import squad_lib as squad_lib_wp -# sentence-piece tokenizer based squad_lib -from official.nlp.data import squad_lib_sp -from official.nlp.data import tagging_data_lib +from official.nlp.data import squad_lib_sp, tagging_data_lib FLAGS = flags.FLAGS diff --git a/official/nlp/data/create_pretraining_data.py b/official/nlp/data/create_pretraining_data.py index 6e3d8b557..18030cf6a 100755 --- a/official/nlp/data/create_pretraining_data.py +++ b/official/nlp/data/create_pretraining_data.py @@ -18,11 +18,9 @@ import itertools import random -# Import libraries -from absl import app -from absl import flags -from absl import logging import tensorflow as tf +# Import libraries +from absl import app, flags, logging from official.nlp.bert import tokenization diff --git a/official/nlp/data/create_xlnet_pretraining_data.py b/official/nlp/data/create_xlnet_pretraining_data.py index b8c1b962c..7c93f89a2 100755 --- a/official/nlp/data/create_xlnet_pretraining_data.py +++ b/official/nlp/data/create_xlnet_pretraining_data.py @@ -14,26 +14,25 @@ # ============================================================================== """Create LM TF examples for XLNet.""" +import dataclasses import json import math import os - import random -from typing import Iterable, Mapping, List, Optional, Tuple import unicodedata +from typing import Iterable, List, Mapping, Optional, Tuple -# Import libraries - -from absl import app -from absl import flags -from absl import logging - -import dataclasses import numpy as np import tensorflow as tf +from absl import app, flags, logging from official.nlp.bert import tokenization +# Import libraries + + + + special_symbols = { "": 0, "": 1, diff --git a/official/nlp/data/create_xlnet_pretraining_data_test.py b/official/nlp/data/create_xlnet_pretraining_data_test.py index 095361ee9..d82ad7e59 100755 --- a/official/nlp/data/create_xlnet_pretraining_data_test.py +++ b/official/nlp/data/create_xlnet_pretraining_data_test.py @@ -18,11 +18,10 @@ import tempfile from typing import List -from absl import logging -from absl.testing import parameterized - import numpy as np import tensorflow as tf +from absl import logging +from absl.testing import parameterized from official.nlp.data import create_xlnet_pretraining_data as cpd diff --git a/official/nlp/data/data_loader_factory_test.py b/official/nlp/data/data_loader_factory_test.py index 9818a4965..aceb82ffb 100755 --- a/official/nlp/data/data_loader_factory_test.py +++ b/official/nlp/data/data_loader_factory_test.py @@ -16,6 +16,7 @@ """Tests for official.nlp.data.data_loader_factory.""" import dataclasses + import tensorflow as tf from official.core import config_definitions as cfg diff --git a/official/nlp/data/pretrain_dataloader.py b/official/nlp/data/pretrain_dataloader.py index 29cb662c5..407d0cdc1 100755 --- a/official/nlp/data/pretrain_dataloader.py +++ b/official/nlp/data/pretrain_dataloader.py @@ -14,17 +14,16 @@ # limitations under the License. # ============================================================================== """Loads dataset for the BERT pretraining task.""" +import dataclasses from typing import Mapping, Optional -from absl import logging - -import dataclasses import numpy as np import tensorflow as tf +from absl import logging + from official.core import config_definitions as cfg from official.core import input_reader -from official.nlp.data import data_loader -from official.nlp.data import data_loader_factory +from official.nlp.data import data_loader, data_loader_factory @dataclasses.dataclass diff --git a/official/nlp/data/pretrain_dataloader_test.py b/official/nlp/data/pretrain_dataloader_test.py index a3441f1aa..07f6339f8 100755 --- a/official/nlp/data/pretrain_dataloader_test.py +++ b/official/nlp/data/pretrain_dataloader_test.py @@ -17,9 +17,9 @@ import itertools import os -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.nlp.data import pretrain_dataloader diff --git a/official/nlp/data/question_answering_dataloader.py b/official/nlp/data/question_answering_dataloader.py index eaa4f021e..e74c36433 100755 --- a/official/nlp/data/question_answering_dataloader.py +++ b/official/nlp/data/question_answering_dataloader.py @@ -14,14 +14,14 @@ # limitations under the License. # ============================================================================== """Loads dataset for the question answering (e.g, SQuAD) task.""" +import dataclasses from typing import Mapping, Optional -import dataclasses import tensorflow as tf + from official.core import config_definitions as cfg from official.core import input_reader -from official.nlp.data import data_loader -from official.nlp.data import data_loader_factory +from official.nlp.data import data_loader, data_loader_factory @dataclasses.dataclass diff --git a/official/nlp/data/sentence_prediction_dataloader.py b/official/nlp/data/sentence_prediction_dataloader.py index 88779c24b..c887f587d 100755 --- a/official/nlp/data/sentence_prediction_dataloader.py +++ b/official/nlp/data/sentence_prediction_dataloader.py @@ -14,9 +14,9 @@ # limitations under the License. # ============================================================================== """Loads dataset for the sentence prediction (classification) task.""" +import dataclasses from typing import List, Mapping, Optional -import dataclasses import tensorflow as tf import tensorflow_hub as hub @@ -24,8 +24,7 @@ from official.core import config_definitions as cfg from official.core import input_reader from official.nlp import modeling -from official.nlp.data import data_loader -from official.nlp.data import data_loader_factory +from official.nlp.data import data_loader, data_loader_factory LABEL_TYPES_MAP = {'int': tf.int64, 'float': tf.float32} diff --git a/official/nlp/data/sentence_prediction_dataloader_test.py b/official/nlp/data/sentence_prediction_dataloader_test.py index 06aa6f92f..caf371eaf 100755 --- a/official/nlp/data/sentence_prediction_dataloader_test.py +++ b/official/nlp/data/sentence_prediction_dataloader_test.py @@ -16,11 +16,11 @@ """Tests for official.nlp.data.sentence_prediction_dataloader.""" import os -from absl.testing import parameterized import numpy as np import tensorflow as tf - +from absl.testing import parameterized from sentencepiece import SentencePieceTrainer + from official.nlp.data import sentence_prediction_dataloader as loader diff --git a/official/nlp/data/sentence_retrieval_lib.py b/official/nlp/data/sentence_retrieval_lib.py index b89d3e7a1..fdc11c235 100755 --- a/official/nlp/data/sentence_retrieval_lib.py +++ b/official/nlp/data/sentence_retrieval_lib.py @@ -17,6 +17,7 @@ import os from absl import logging + from official.nlp.bert import tokenization from official.nlp.data import classifier_data_lib diff --git a/official/nlp/data/squad_lib.py b/official/nlp/data/squad_lib.py index 7f28f6deb..36640f378 100755 --- a/official/nlp/data/squad_lib.py +++ b/official/nlp/data/squad_lib.py @@ -21,9 +21,8 @@ import os import six - -from absl import logging import tensorflow as tf +from absl import logging from official.nlp.bert import tokenization diff --git a/official/nlp/data/squad_lib_sp.py b/official/nlp/data/squad_lib_sp.py index dfb3ee828..02f1acf08 100755 --- a/official/nlp/data/squad_lib_sp.py +++ b/official/nlp/data/squad_lib_sp.py @@ -24,9 +24,9 @@ import math import os -from absl import logging import numpy as np import tensorflow as tf +from absl import logging from official.nlp.bert import tokenization diff --git a/official/nlp/data/tagging_data_lib.py b/official/nlp/data/tagging_data_lib.py index d71fcb1d9..c79b7ca0f 100755 --- a/official/nlp/data/tagging_data_lib.py +++ b/official/nlp/data/tagging_data_lib.py @@ -16,8 +16,8 @@ import collections import os -from absl import logging import tensorflow as tf +from absl import logging from official.nlp.bert import tokenization from official.nlp.data import classifier_data_lib diff --git a/official/nlp/data/tagging_data_lib_test.py b/official/nlp/data/tagging_data_lib_test.py index 940ec01f3..bac37bbee 100755 --- a/official/nlp/data/tagging_data_lib_test.py +++ b/official/nlp/data/tagging_data_lib_test.py @@ -17,8 +17,8 @@ import os import random -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized from official.nlp.bert import tokenization from official.nlp.data import tagging_data_lib diff --git a/official/nlp/data/tagging_dataloader.py b/official/nlp/data/tagging_dataloader.py index e50cad5bf..4bdf470bd 100755 --- a/official/nlp/data/tagging_dataloader.py +++ b/official/nlp/data/tagging_dataloader.py @@ -14,14 +14,14 @@ # limitations under the License. # ============================================================================== """Loads dataset for the tagging (e.g., NER/POS) task.""" +import dataclasses from typing import Mapping, Optional -import dataclasses import tensorflow as tf + from official.core import config_definitions as cfg from official.core import input_reader -from official.nlp.data import data_loader -from official.nlp.data import data_loader_factory +from official.nlp.data import data_loader, data_loader_factory @dataclasses.dataclass diff --git a/official/nlp/data/tagging_dataloader_test.py b/official/nlp/data/tagging_dataloader_test.py index ce67640f5..6a82eb482 100755 --- a/official/nlp/data/tagging_dataloader_test.py +++ b/official/nlp/data/tagging_dataloader_test.py @@ -16,9 +16,9 @@ """Tests for official.nlp.data.tagging_data_loader.""" import os -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.nlp.data import tagging_dataloader diff --git a/official/nlp/data/train_sentencepiece.py b/official/nlp/data/train_sentencepiece.py index a4528937e..d54d90ede 100755 --- a/official/nlp/data/train_sentencepiece.py +++ b/official/nlp/data/train_sentencepiece.py @@ -25,15 +25,11 @@ import tempfile from typing import List, Tuple -from absl import app -from absl import flags -from absl import logging import tensorflow as tf import tensorflow_datasets as tfds - +from absl import app, flags, logging from sentencepiece import SentencePieceTrainer - FLAGS = flags.FLAGS flags.DEFINE_string("output_model_path", None, "Path to save the the sentencepiece model.") diff --git a/official/nlp/data/wmt_dataloader.py b/official/nlp/data/wmt_dataloader.py index 424839676..e026c7df6 100755 --- a/official/nlp/data/wmt_dataloader.py +++ b/official/nlp/data/wmt_dataloader.py @@ -31,15 +31,15 @@ This batching scheme decreases the fraction of padding tokens per training batch, thus improving the training speed significantly. """ +import dataclasses from typing import Dict, Optional -import dataclasses import tensorflow as tf + import tensorflow_text as tftxt from official.core import config_definitions as cfg from official.core import input_reader -from official.nlp.data import data_loader -from official.nlp.data import data_loader_factory +from official.nlp.data import data_loader, data_loader_factory # Example grouping constants. Defines length boundaries for each group. # These values are the defaults used in Tensor2Tensor. diff --git a/official/nlp/data/wmt_dataloader_test.py b/official/nlp/data/wmt_dataloader_test.py index 6644f886c..ffb27662b 100755 --- a/official/nlp/data/wmt_dataloader_test.py +++ b/official/nlp/data/wmt_dataloader_test.py @@ -15,11 +15,11 @@ # ============================================================================== """Tests for official.nlp.data.wmt_dataloader.""" import os -from absl.testing import parameterized import tensorflow as tf - +from absl.testing import parameterized from sentencepiece import SentencePieceTrainer + from official.nlp.data import wmt_dataloader diff --git a/official/nlp/keras_nlp/__init__.py b/official/nlp/keras_nlp/__init__.py index 68aae1196..7462fa3c8 100755 --- a/official/nlp/keras_nlp/__init__.py +++ b/official/nlp/keras_nlp/__init__.py @@ -14,5 +14,4 @@ # ============================================================================== """Keras-NLP package definition.""" # pylint: disable=wildcard-import -from official.nlp.keras_nlp import encoders -from official.nlp.keras_nlp import layers +from official.nlp.keras_nlp import encoders, layers diff --git a/official/nlp/keras_nlp/encoders/bert_encoder.py b/official/nlp/keras_nlp/encoders/bert_encoder.py index f488ea565..030494c49 100755 --- a/official/nlp/keras_nlp/encoders/bert_encoder.py +++ b/official/nlp/keras_nlp/encoders/bert_encoder.py @@ -16,8 +16,9 @@ # pylint: disable=g-classes-have-attributes import collections -from absl import logging + import tensorflow as tf +from absl import logging from official.nlp.keras_nlp import layers diff --git a/official/nlp/keras_nlp/encoders/bert_encoder_test.py b/official/nlp/keras_nlp/encoders/bert_encoder_test.py index 3fab59fd0..aef82b9a6 100755 --- a/official/nlp/keras_nlp/encoders/bert_encoder_test.py +++ b/official/nlp/keras_nlp/encoders/bert_encoder_test.py @@ -14,11 +14,12 @@ # ============================================================================== """Tests for transformer-based bert encoder network.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.keras_nlp.encoders import bert_encoder diff --git a/official/nlp/keras_nlp/layers/__init__.py b/official/nlp/keras_nlp/layers/__init__.py index 4476cbb05..79a750163 100755 --- a/official/nlp/keras_nlp/layers/__init__.py +++ b/official/nlp/keras_nlp/layers/__init__.py @@ -17,4 +17,5 @@ from official.nlp.keras_nlp.layers.on_device_embedding import OnDeviceEmbedding from official.nlp.keras_nlp.layers.position_embedding import PositionEmbedding from official.nlp.keras_nlp.layers.self_attention_mask import SelfAttentionMask -from official.nlp.keras_nlp.layers.transformer_encoder_block import TransformerEncoderBlock +from official.nlp.keras_nlp.layers.transformer_encoder_block import \ + TransformerEncoderBlock diff --git a/official/nlp/keras_nlp/layers/on_device_embedding_test.py b/official/nlp/keras_nlp/layers/on_device_embedding_test.py index 7fd4b0efe..ba399bf57 100755 --- a/official/nlp/keras_nlp/layers/on_device_embedding_test.py +++ b/official/nlp/keras_nlp/layers/on_device_embedding_test.py @@ -16,8 +16,9 @@ import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.keras_nlp.layers import on_device_embedding diff --git a/official/nlp/keras_nlp/layers/position_embedding_test.py b/official/nlp/keras_nlp/layers/position_embedding_test.py index 3dea2741b..4628bb3fd 100755 --- a/official/nlp/keras_nlp/layers/position_embedding_test.py +++ b/official/nlp/keras_nlp/layers/position_embedding_test.py @@ -16,8 +16,9 @@ import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.keras_nlp.layers import position_embedding diff --git a/official/nlp/keras_nlp/layers/transformer_encoder_block_test.py b/official/nlp/keras_nlp/layers/transformer_encoder_block_test.py index 7e422fdc1..f0fac7275 100755 --- a/official/nlp/keras_nlp/layers/transformer_encoder_block_test.py +++ b/official/nlp/keras_nlp/layers/transformer_encoder_block_test.py @@ -14,12 +14,14 @@ # ============================================================================== """Tests for Keras-based transformer block layer.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import -from official.nlp.keras_nlp.layers.transformer_encoder_block import TransformerEncoderBlock +from official.nlp.keras_nlp.layers.transformer_encoder_block import \ + TransformerEncoderBlock @keras_parameterized.run_all_keras_modes diff --git a/official/nlp/keras_nlp/setup.py b/official/nlp/keras_nlp/setup.py index a7e674141..9a0869d5b 100755 --- a/official/nlp/keras_nlp/setup.py +++ b/official/nlp/keras_nlp/setup.py @@ -16,8 +16,7 @@ import os -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup version = '0.0.1' diff --git a/official/nlp/modeling/__init__.py b/official/nlp/modeling/__init__.py index df56aec85..3522909fb 100755 --- a/official/nlp/modeling/__init__.py +++ b/official/nlp/modeling/__init__.py @@ -13,7 +13,4 @@ # limitations under the License. # ============================================================================== """Modeling package definition.""" -from official.nlp.modeling import layers -from official.nlp.modeling import losses -from official.nlp.modeling import models -from official.nlp.modeling import networks +from official.nlp.modeling import layers, losses, models, networks diff --git a/official/nlp/modeling/layers/__init__.py b/official/nlp/modeling/layers/__init__.py index 97c2afd59..92c170d44 100755 --- a/official/nlp/modeling/layers/__init__.py +++ b/official/nlp/modeling/layers/__init__.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================== """Layers package definition.""" -# pylint: disable=wildcard-import from official.nlp.modeling.layers.attention import * from official.nlp.modeling.layers.cls_head import * from official.nlp.modeling.layers.dense_einsum import DenseEinsum @@ -21,23 +20,25 @@ from official.nlp.modeling.layers.masked_lm import MaskedLM from official.nlp.modeling.layers.masked_softmax import MaskedSoftmax from official.nlp.modeling.layers.mat_mul_with_margin import MatMulWithMargin -from official.nlp.modeling.layers.mobile_bert_layers import MobileBertEmbedding -from official.nlp.modeling.layers.mobile_bert_layers import MobileBertMaskedLM -from official.nlp.modeling.layers.mobile_bert_layers import MobileBertTransformer +from official.nlp.modeling.layers.mobile_bert_layers import ( + MobileBertEmbedding, MobileBertMaskedLM, MobileBertTransformer) from official.nlp.modeling.layers.multi_channel_attention import * from official.nlp.modeling.layers.on_device_embedding import OnDeviceEmbedding -from official.nlp.modeling.layers.position_embedding import RelativePositionBias -from official.nlp.modeling.layers.position_embedding import RelativePositionEmbedding -from official.nlp.modeling.layers.relative_attention import MultiHeadRelativeAttention -from official.nlp.modeling.layers.relative_attention import TwoStreamRelativeAttention +from official.nlp.modeling.layers.position_embedding import ( + RelativePositionBias, RelativePositionEmbedding) +from official.nlp.modeling.layers.relative_attention import ( + MultiHeadRelativeAttention, TwoStreamRelativeAttention) from official.nlp.modeling.layers.rezero_transformer import ReZeroTransformer from official.nlp.modeling.layers.self_attention_mask import SelfAttentionMask -from official.nlp.modeling.layers.talking_heads_attention import TalkingHeadsAttention -from official.nlp.modeling.layers.text_layers import BertPackInputs -from official.nlp.modeling.layers.text_layers import BertTokenizer -from official.nlp.modeling.layers.text_layers import SentencepieceTokenizer -from official.nlp.modeling.layers.tn_transformer_expand_condense import TNTransformerExpandCondense +from official.nlp.modeling.layers.talking_heads_attention import \ + TalkingHeadsAttention +from official.nlp.modeling.layers.text_layers import (BertPackInputs, + BertTokenizer, + SentencepieceTokenizer) +from official.nlp.modeling.layers.tn_transformer_expand_condense import \ + TNTransformerExpandCondense from official.nlp.modeling.layers.transformer import * -from official.nlp.modeling.layers.transformer_scaffold import TransformerScaffold -from official.nlp.modeling.layers.transformer_xl import TransformerXL -from official.nlp.modeling.layers.transformer_xl import TransformerXLBlock +from official.nlp.modeling.layers.transformer_scaffold import \ + TransformerScaffold +from official.nlp.modeling.layers.transformer_xl import (TransformerXL, + TransformerXLBlock) diff --git a/official/nlp/modeling/layers/attention_test.py b/official/nlp/modeling/layers/attention_test.py index c1ea089a2..faae39712 100755 --- a/official/nlp/modeling/layers/attention_test.py +++ b/official/nlp/modeling/layers/attention_test.py @@ -16,8 +16,9 @@ import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import attention diff --git a/official/nlp/modeling/layers/dense_einsum.py b/official/nlp/modeling/layers/dense_einsum.py index 7fa14e243..02c89711a 100755 --- a/official/nlp/modeling/layers/dense_einsum.py +++ b/official/nlp/modeling/layers/dense_einsum.py @@ -16,7 +16,6 @@ # pylint: disable=g-classes-have-attributes import tensorflow as tf - from tensorflow.python.util import deprecation _CHR_IDX = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m"] diff --git a/official/nlp/modeling/layers/dense_einsum_test.py b/official/nlp/modeling/layers/dense_einsum_test.py index af302c529..5bf4aec7b 100755 --- a/official/nlp/modeling/layers/dense_einsum_test.py +++ b/official/nlp/modeling/layers/dense_einsum_test.py @@ -16,8 +16,9 @@ import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import dense_einsum diff --git a/official/nlp/modeling/layers/gated_feedforward_test.py b/official/nlp/modeling/layers/gated_feedforward_test.py index 342cf5cf1..03792b23e 100755 --- a/official/nlp/modeling/layers/gated_feedforward_test.py +++ b/official/nlp/modeling/layers/gated_feedforward_test.py @@ -14,11 +14,12 @@ # ============================================================================== """Tests for Keras-based gated feedforward layer.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import gated_feedforward diff --git a/official/nlp/modeling/layers/masked_lm.py b/official/nlp/modeling/layers/masked_lm.py index 6196f36dd..dc338a4f6 100755 --- a/official/nlp/modeling/layers/masked_lm.py +++ b/official/nlp/modeling/layers/masked_lm.py @@ -16,5 +16,4 @@ # pylint: disable=g-classes-have-attributes from official.nlp import keras_nlp - MaskedLM = keras_nlp.layers.MaskedLM diff --git a/official/nlp/modeling/layers/masked_lm_test.py b/official/nlp/modeling/layers/masked_lm_test.py index f47adf93c..241150301 100755 --- a/official/nlp/modeling/layers/masked_lm_test.py +++ b/official/nlp/modeling/layers/masked_lm_test.py @@ -16,8 +16,8 @@ import numpy as np import tensorflow as tf - -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import masked_lm from official.nlp.modeling.networks import bert_encoder diff --git a/official/nlp/modeling/layers/masked_softmax_test.py b/official/nlp/modeling/layers/masked_softmax_test.py index 6c0a5c416..695eb9e1b 100755 --- a/official/nlp/modeling/layers/masked_softmax_test.py +++ b/official/nlp/modeling/layers/masked_softmax_test.py @@ -16,8 +16,9 @@ import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import masked_softmax diff --git a/official/nlp/modeling/layers/mat_mul_with_margin.py b/official/nlp/modeling/layers/mat_mul_with_margin.py index 2a0895298..c990c76b7 100755 --- a/official/nlp/modeling/layers/mat_mul_with_margin.py +++ b/official/nlp/modeling/layers/mat_mul_with_margin.py @@ -16,6 +16,7 @@ # pylint: disable=g-classes-have-attributes from typing import Tuple + # Import libraries import tensorflow as tf diff --git a/official/nlp/modeling/layers/mat_mul_with_margin_test.py b/official/nlp/modeling/layers/mat_mul_with_margin_test.py index 7134196b1..e6bdc77f9 100755 --- a/official/nlp/modeling/layers/mat_mul_with_margin_test.py +++ b/official/nlp/modeling/layers/mat_mul_with_margin_test.py @@ -15,8 +15,9 @@ """Tests for mat_mul_with_margin layer.""" import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import mat_mul_with_margin diff --git a/official/nlp/modeling/layers/mobile_bert_layers_test.py b/official/nlp/modeling/layers/mobile_bert_layers_test.py index 7861217bd..d539d630c 100755 --- a/official/nlp/modeling/layers/mobile_bert_layers_test.py +++ b/official/nlp/modeling/layers/mobile_bert_layers_test.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -from absl.testing import parameterized - import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.nlp.modeling.layers import mobile_bert_layers from official.nlp.modeling.networks import mobile_bert_encoder diff --git a/official/nlp/modeling/layers/multi_channel_attention.py b/official/nlp/modeling/layers/multi_channel_attention.py index 2c70ecf9c..9e9949a56 100755 --- a/official/nlp/modeling/layers/multi_channel_attention.py +++ b/official/nlp/modeling/layers/multi_channel_attention.py @@ -19,6 +19,7 @@ import math import tensorflow as tf + from official.modeling import tf_utils from official.nlp.modeling.layers import masked_softmax diff --git a/official/nlp/modeling/layers/on_device_embedding.py b/official/nlp/modeling/layers/on_device_embedding.py index 85c34c24b..7083f9ad0 100755 --- a/official/nlp/modeling/layers/on_device_embedding.py +++ b/official/nlp/modeling/layers/on_device_embedding.py @@ -17,5 +17,4 @@ from official.nlp import keras_nlp - OnDeviceEmbedding = keras_nlp.layers.OnDeviceEmbedding diff --git a/official/nlp/modeling/layers/position_embedding_test.py b/official/nlp/modeling/layers/position_embedding_test.py index 4e2d475cd..01260a932 100755 --- a/official/nlp/modeling/layers/position_embedding_test.py +++ b/official/nlp/modeling/layers/position_embedding_test.py @@ -14,11 +14,12 @@ # ============================================================================== """Tests for Keras-based positional embedding layer.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import position_embedding diff --git a/official/nlp/modeling/layers/relative_attention.py b/official/nlp/modeling/layers/relative_attention.py index cb3fb374c..91f7275be 100755 --- a/official/nlp/modeling/layers/relative_attention.py +++ b/official/nlp/modeling/layers/relative_attention.py @@ -16,6 +16,7 @@ """Keras-based relative attention layers.""" import math import string + import tensorflow as tf _CHR_IDX = string.ascii_lowercase @@ -497,4 +498,3 @@ def call(self, query_attention_output = self._output_dense(query_attention_output) return content_attention_output, query_attention_output - diff --git a/official/nlp/modeling/layers/relative_attention_test.py b/official/nlp/modeling/layers/relative_attention_test.py index aff5eda21..8792b79b0 100755 --- a/official/nlp/modeling/layers/relative_attention_test.py +++ b/official/nlp/modeling/layers/relative_attention_test.py @@ -16,9 +16,10 @@ import numpy as np import tensorflow as tf - from tensorflow.python.distribute import combinations -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import + from official.nlp.modeling.layers import relative_attention diff --git a/official/nlp/modeling/layers/rezero_transformer_test.py b/official/nlp/modeling/layers/rezero_transformer_test.py index 69a2e7d9e..7aaf63341 100755 --- a/official/nlp/modeling/layers/rezero_transformer_test.py +++ b/official/nlp/modeling/layers/rezero_transformer_test.py @@ -16,8 +16,9 @@ import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import rezero_transformer diff --git a/official/nlp/modeling/layers/talking_heads_attention_test.py b/official/nlp/modeling/layers/talking_heads_attention_test.py index 1ee71223b..ce3ab3035 100755 --- a/official/nlp/modeling/layers/talking_heads_attention_test.py +++ b/official/nlp/modeling/layers/talking_heads_attention_test.py @@ -14,11 +14,12 @@ # ============================================================================== """Tests for the attention layer.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import talking_heads_attention diff --git a/official/nlp/modeling/layers/text_layers.py b/official/nlp/modeling/layers/text_layers.py index c7b80f4d5..1ce9f869b 100755 --- a/official/nlp/modeling/layers/text_layers.py +++ b/official/nlp/modeling/layers/text_layers.py @@ -15,8 +15,8 @@ """Keras Layers for BERT-specific preprocessing.""" from typing import Any, Dict, List, Optional, Union -from absl import logging import tensorflow as tf +from absl import logging try: import tensorflow_text as text # pylint: disable=g-import-not-at-top diff --git a/official/nlp/modeling/layers/text_layers_test.py b/official/nlp/modeling/layers/text_layers_test.py index 4fabd7d02..f762b1e93 100755 --- a/official/nlp/modeling/layers/text_layers_test.py +++ b/official/nlp/modeling/layers/text_layers_test.py @@ -19,8 +19,8 @@ import numpy as np import tensorflow as tf - from sentencepiece import SentencePieceTrainer + from official.nlp.modeling.layers import text_layers diff --git a/official/nlp/modeling/layers/tn_expand_condense.py b/official/nlp/modeling/layers/tn_expand_condense.py index b4f830b3b..310a76a26 100755 --- a/official/nlp/modeling/layers/tn_expand_condense.py +++ b/official/nlp/modeling/layers/tn_expand_condense.py @@ -14,7 +14,8 @@ # ============================================================================== """ExpandCondense tensor network layer used in TN-BERT.""" # pylint: disable=g-classes-have-attributes -from typing import List, Optional, Text, Any, Dict +from typing import Any, Dict, List, Optional, Text + import tensorflow as tf Layer = tf.keras.layers.Layer diff --git a/official/nlp/modeling/layers/tn_expand_condense_test.py b/official/nlp/modeling/layers/tn_expand_condense_test.py index 4accc5867..6c5c49b9b 100755 --- a/official/nlp/modeling/layers/tn_expand_condense_test.py +++ b/official/nlp/modeling/layers/tn_expand_condense_test.py @@ -16,11 +16,12 @@ import os -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized # pylint: disable=g-direct-tensorflow-import from tensorflow.python.keras.testing_utils import layer_test + from official.nlp.modeling.layers.tn_expand_condense import TNExpandCondense diff --git a/official/nlp/modeling/layers/tn_transformer_test.py b/official/nlp/modeling/layers/tn_transformer_test.py index 1a1b41128..321483adf 100755 --- a/official/nlp/modeling/layers/tn_transformer_test.py +++ b/official/nlp/modeling/layers/tn_transformer_test.py @@ -14,12 +14,14 @@ # ============================================================================== """Tests for TN-BERT transformer.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import -from official.nlp.modeling.layers.tn_transformer_expand_condense import TNTransformerExpandCondense +from official.nlp.modeling.layers.tn_transformer_expand_condense import \ + TNTransformerExpandCondense # This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It diff --git a/official/nlp/modeling/layers/transformer.py b/official/nlp/modeling/layers/transformer.py index 511ba9dbb..b5e3ae8bc 100755 --- a/official/nlp/modeling/layers/transformer.py +++ b/official/nlp/modeling/layers/transformer.py @@ -19,8 +19,7 @@ import tensorflow as tf from official.nlp import keras_nlp -from official.nlp.modeling.layers import attention -from official.nlp.modeling.layers import multi_channel_attention +from official.nlp.modeling.layers import attention, multi_channel_attention from official.nlp.modeling.layers.util import tf_function_if_eager diff --git a/official/nlp/modeling/layers/transformer_scaffold.py b/official/nlp/modeling/layers/transformer_scaffold.py index 34d06d49c..e934b7ea7 100755 --- a/official/nlp/modeling/layers/transformer_scaffold.py +++ b/official/nlp/modeling/layers/transformer_scaffold.py @@ -15,9 +15,9 @@ """Keras-based transformer scaffold layer.""" # pylint: disable=g-classes-have-attributes -from absl import logging import gin import tensorflow as tf +from absl import logging from official.nlp.modeling.layers import attention diff --git a/official/nlp/modeling/layers/transformer_scaffold_test.py b/official/nlp/modeling/layers/transformer_scaffold_test.py index 14bb8c23d..088fb5a63 100755 --- a/official/nlp/modeling/layers/transformer_scaffold_test.py +++ b/official/nlp/modeling/layers/transformer_scaffold_test.py @@ -16,10 +16,10 @@ import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import -from official.nlp.modeling.layers import attention -from official.nlp.modeling.layers import transformer_scaffold +from official.nlp.modeling.layers import attention, transformer_scaffold # Test class that wraps a standard attention layer. If this layer is called diff --git a/official/nlp/modeling/layers/transformer_test.py b/official/nlp/modeling/layers/transformer_test.py index aef245c9d..bbce7790d 100755 --- a/official/nlp/modeling/layers/transformer_test.py +++ b/official/nlp/modeling/layers/transformer_test.py @@ -15,8 +15,9 @@ """Tests for Keras-based transformer block layer.""" import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import transformer diff --git a/official/nlp/modeling/layers/transformer_xl.py b/official/nlp/modeling/layers/transformer_xl.py index f0fff2efb..62bfc455a 100755 --- a/official/nlp/modeling/layers/transformer_xl.py +++ b/official/nlp/modeling/layers/transformer_xl.py @@ -15,9 +15,8 @@ # ============================================================================== """Keras-based Transformer XL layer.""" -from absl import logging - import tensorflow as tf +from absl import logging from official.nlp.modeling.layers import relative_attention diff --git a/official/nlp/modeling/layers/transformer_xl_test.py b/official/nlp/modeling/layers/transformer_xl_test.py index 443b26968..7c2cdc4dc 100755 --- a/official/nlp/modeling/layers/transformer_xl_test.py +++ b/official/nlp/modeling/layers/transformer_xl_test.py @@ -16,9 +16,9 @@ import numpy as np import tensorflow as tf - from tensorflow.python.distribute import combinations -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.layers import transformer_xl diff --git a/official/nlp/modeling/losses/__init__.py b/official/nlp/modeling/losses/__init__.py index 7a396eb98..6fc0f991b 100755 --- a/official/nlp/modeling/losses/__init__.py +++ b/official/nlp/modeling/losses/__init__.py @@ -13,4 +13,5 @@ # limitations under the License. # ============================================================================== """Activations package definition. Subject to change.""" -from official.nlp.modeling.losses.weighted_sparse_categorical_crossentropy import loss as weighted_sparse_categorical_crossentropy_loss +from official.nlp.modeling.losses.weighted_sparse_categorical_crossentropy import \ + loss as weighted_sparse_categorical_crossentropy_loss diff --git a/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py b/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py index 9e6c09cde..5e30424a3 100755 --- a/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py +++ b/official/nlp/modeling/losses/weighted_sparse_categorical_crossentropy_test.py @@ -14,13 +14,13 @@ # ============================================================================== """Tests for masked LM loss.""" import numpy as np - import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import -from official.nlp.modeling import layers -from official.nlp.modeling import networks -from official.nlp.modeling.losses import weighted_sparse_categorical_crossentropy +from official.nlp.modeling import layers, networks +from official.nlp.modeling.losses import \ + weighted_sparse_categorical_crossentropy @keras_parameterized.run_all_keras_modes diff --git a/official/nlp/modeling/models/__init__.py b/official/nlp/modeling/models/__init__.py index c5c4eb610..2f567394a 100755 --- a/official/nlp/modeling/models/__init__.py +++ b/official/nlp/modeling/models/__init__.py @@ -16,10 +16,11 @@ from official.nlp.modeling.models.bert_classifier import BertClassifier from official.nlp.modeling.models.bert_pretrainer import * from official.nlp.modeling.models.bert_span_labeler import BertSpanLabeler -from official.nlp.modeling.models.bert_token_classifier import BertTokenClassifier +from official.nlp.modeling.models.bert_token_classifier import \ + BertTokenClassifier from official.nlp.modeling.models.dual_encoder import DualEncoder from official.nlp.modeling.models.electra_pretrainer import ElectraPretrainer from official.nlp.modeling.models.seq2seq_transformer import * -from official.nlp.modeling.models.xlnet import XLNetClassifier -from official.nlp.modeling.models.xlnet import XLNetPretrainer -from official.nlp.modeling.models.xlnet import XLNetSpanLabeler +from official.nlp.modeling.models.xlnet import (XLNetClassifier, + XLNetPretrainer, + XLNetSpanLabeler) diff --git a/official/nlp/modeling/models/bert_classifier.py b/official/nlp/modeling/models/bert_classifier.py index aba240a7c..e47e20285 100755 --- a/official/nlp/modeling/models/bert_classifier.py +++ b/official/nlp/modeling/models/bert_classifier.py @@ -15,10 +15,10 @@ """BERT cls-token classifier.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf -from official.nlp.modeling import layers -from official.nlp.modeling import networks +from official.nlp.modeling import layers, networks @tf.keras.utils.register_keras_serializable(package='Text') diff --git a/official/nlp/modeling/models/bert_classifier_test.py b/official/nlp/modeling/models/bert_classifier_test.py index ed3c5b09e..044a13a5a 100755 --- a/official/nlp/modeling/models/bert_classifier_test.py +++ b/official/nlp/modeling/models/bert_classifier_test.py @@ -14,10 +14,11 @@ # ============================================================================== """Tests for BERT trainer network.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling import networks from official.nlp.modeling.models import bert_classifier diff --git a/official/nlp/modeling/models/bert_pretrainer.py b/official/nlp/modeling/models/bert_pretrainer.py index a4bade2af..3b09866b0 100755 --- a/official/nlp/modeling/models/bert_pretrainer.py +++ b/official/nlp/modeling/models/bert_pretrainer.py @@ -18,12 +18,11 @@ import copy from typing import List, Optional -from absl import logging import gin import tensorflow as tf +from absl import logging -from official.nlp.modeling import layers -from official.nlp.modeling import networks +from official.nlp.modeling import layers, networks @tf.keras.utils.register_keras_serializable(package='Text') diff --git a/official/nlp/modeling/models/bert_pretrainer_test.py b/official/nlp/modeling/models/bert_pretrainer_test.py index cb438630a..cb8be43b9 100755 --- a/official/nlp/modeling/models/bert_pretrainer_test.py +++ b/official/nlp/modeling/models/bert_pretrainer_test.py @@ -15,12 +15,12 @@ """Tests for BERT pretrainer model.""" import itertools -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import -from official.nlp.modeling import layers -from official.nlp.modeling import networks +from official.nlp.modeling import layers, networks from official.nlp.modeling.models import bert_pretrainer diff --git a/official/nlp/modeling/models/bert_span_labeler.py b/official/nlp/modeling/models/bert_span_labeler.py index 8e4b10da1..566035973 100755 --- a/official/nlp/modeling/models/bert_span_labeler.py +++ b/official/nlp/modeling/models/bert_span_labeler.py @@ -15,6 +15,7 @@ """BERT Question Answering model.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf from official.nlp.modeling import networks diff --git a/official/nlp/modeling/models/bert_span_labeler_test.py b/official/nlp/modeling/models/bert_span_labeler_test.py index 2c613ccdb..0ff7fe01d 100755 --- a/official/nlp/modeling/models/bert_span_labeler_test.py +++ b/official/nlp/modeling/models/bert_span_labeler_test.py @@ -14,10 +14,11 @@ # ============================================================================== """Tests for BERT trainer network.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling import networks from official.nlp.modeling.models import bert_span_labeler diff --git a/official/nlp/modeling/models/bert_token_classifier.py b/official/nlp/modeling/models/bert_token_classifier.py index 92ac76da7..e9a96767f 100755 --- a/official/nlp/modeling/models/bert_token_classifier.py +++ b/official/nlp/modeling/models/bert_token_classifier.py @@ -15,6 +15,7 @@ """BERT token classifier.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf diff --git a/official/nlp/modeling/models/bert_token_classifier_test.py b/official/nlp/modeling/models/bert_token_classifier_test.py index 43c255b73..97a98d7e8 100755 --- a/official/nlp/modeling/models/bert_token_classifier_test.py +++ b/official/nlp/modeling/models/bert_token_classifier_test.py @@ -14,10 +14,11 @@ # ============================================================================== """Tests for BERT token classifier.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling import networks from official.nlp.modeling.models import bert_token_classifier diff --git a/official/nlp/modeling/models/dual_encoder.py b/official/nlp/modeling/models/dual_encoder.py index 190f63fe5..dde94d60f 100755 --- a/official/nlp/modeling/models/dual_encoder.py +++ b/official/nlp/modeling/models/dual_encoder.py @@ -15,6 +15,7 @@ """Trainer network for dual encoder style models.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf from official.nlp.modeling import layers diff --git a/official/nlp/modeling/models/dual_encoder_test.py b/official/nlp/modeling/models/dual_encoder_test.py index 2bdf71f4b..17ee59d8c 100755 --- a/official/nlp/modeling/models/dual_encoder_test.py +++ b/official/nlp/modeling/models/dual_encoder_test.py @@ -14,10 +14,11 @@ # ============================================================================== """Tests for dual encoder network.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling import networks from official.nlp.modeling.models import dual_encoder diff --git a/official/nlp/modeling/models/electra_pretrainer_test.py b/official/nlp/modeling/models/electra_pretrainer_test.py index 2d16b87ac..a1e327741 100755 --- a/official/nlp/modeling/models/electra_pretrainer_test.py +++ b/official/nlp/modeling/models/electra_pretrainer_test.py @@ -15,8 +15,9 @@ """Tests for ELECTRA pre trainer network.""" import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling import networks from official.nlp.modeling.models import electra_pretrainer diff --git a/official/nlp/modeling/models/seq2seq_transformer.py b/official/nlp/modeling/models/seq2seq_transformer.py index e8f9de673..3809da116 100755 --- a/official/nlp/modeling/models/seq2seq_transformer.py +++ b/official/nlp/modeling/models/seq2seq_transformer.py @@ -19,6 +19,7 @@ import math import tensorflow as tf + from official.modeling import tf_utils from official.nlp import keras_nlp from official.nlp.modeling import layers diff --git a/official/nlp/modeling/models/seq2seq_transformer_test.py b/official/nlp/modeling/models/seq2seq_transformer_test.py index 0ba82f67f..d3dc73237 100755 --- a/official/nlp/modeling/models/seq2seq_transformer_test.py +++ b/official/nlp/modeling/models/seq2seq_transformer_test.py @@ -14,13 +14,12 @@ # ============================================================================== """Test Transformer model.""" -from absl import logging -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl import logging +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.nlp.modeling.models import seq2seq_transformer diff --git a/official/nlp/modeling/models/xlnet.py b/official/nlp/modeling/models/xlnet.py index 87ee65a92..3e35eaeaf 100755 --- a/official/nlp/modeling/models/xlnet.py +++ b/official/nlp/modeling/models/xlnet.py @@ -19,8 +19,7 @@ import tensorflow as tf -from official.nlp.modeling import layers -from official.nlp.modeling import networks +from official.nlp.modeling import layers, networks class XLNetMaskedLM(tf.keras.layers.Layer): @@ -339,4 +338,3 @@ def get_config(self): @classmethod def from_config(cls, config, custom_objects=None): return cls(**config) - diff --git a/official/nlp/modeling/models/xlnet_test.py b/official/nlp/modeling/models/xlnet_test.py index 67b8fe4d7..ccfe96d43 100755 --- a/official/nlp/modeling/models/xlnet_test.py +++ b/official/nlp/modeling/models/xlnet_test.py @@ -14,12 +14,12 @@ # ============================================================================== """Tests for XLNet classifier network.""" -from absl.testing import parameterized - import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling import networks from official.nlp.modeling.models import xlnet diff --git a/official/nlp/modeling/networks/__init__.py b/official/nlp/modeling/networks/__init__.py index f716985e6..29e83811d 100755 --- a/official/nlp/modeling/networks/__init__.py +++ b/official/nlp/modeling/networks/__init__.py @@ -17,10 +17,13 @@ from official.nlp.modeling.networks.bert_encoder import BertEncoder from official.nlp.modeling.networks.classification import Classification from official.nlp.modeling.networks.encoder_scaffold import EncoderScaffold -from official.nlp.modeling.networks.mobile_bert_encoder import MobileBERTEncoder -from official.nlp.modeling.networks.packed_sequence_embedding import PackedSequenceEmbedding -from official.nlp.modeling.networks.span_labeling import SpanLabeling -from official.nlp.modeling.networks.span_labeling import XLNetSpanLabeling +from official.nlp.modeling.networks.mobile_bert_encoder import \ + MobileBERTEncoder +from official.nlp.modeling.networks.packed_sequence_embedding import \ + PackedSequenceEmbedding +from official.nlp.modeling.networks.span_labeling import (SpanLabeling, + XLNetSpanLabeling) from official.nlp.modeling.networks.xlnet_base import XLNetBase + # Backward compatibility. The modules are deprecated. TransformerEncoder = BertEncoder diff --git a/official/nlp/modeling/networks/albert_encoder.py b/official/nlp/modeling/networks/albert_encoder.py index 473b4f08e..47aab8e36 100755 --- a/official/nlp/modeling/networks/albert_encoder.py +++ b/official/nlp/modeling/networks/albert_encoder.py @@ -15,6 +15,7 @@ """ALBERT (https://arxiv.org/abs/1810.04805) text encoder network.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf from official.modeling import activations diff --git a/official/nlp/modeling/networks/albert_encoder_test.py b/official/nlp/modeling/networks/albert_encoder_test.py index c29031788..b7b680a00 100755 --- a/official/nlp/modeling/networks/albert_encoder_test.py +++ b/official/nlp/modeling/networks/albert_encoder_test.py @@ -14,15 +14,14 @@ # ============================================================================== """Tests for ALBERT transformer-based text encoder network.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.networks import albert_encoder diff --git a/official/nlp/modeling/networks/bert_encoder.py b/official/nlp/modeling/networks/bert_encoder.py index a0647e7c3..15cb461ee 100755 --- a/official/nlp/modeling/networks/bert_encoder.py +++ b/official/nlp/modeling/networks/bert_encoder.py @@ -15,6 +15,7 @@ """Transformer-based text encoder network.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf from official.modeling import activations diff --git a/official/nlp/modeling/networks/bert_encoder_test.py b/official/nlp/modeling/networks/bert_encoder_test.py index d58bb094e..1c46fa394 100755 --- a/official/nlp/modeling/networks/bert_encoder_test.py +++ b/official/nlp/modeling/networks/bert_encoder_test.py @@ -14,12 +14,13 @@ # ============================================================================== """Tests for transformer-based bert encoder network.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.networks import bert_encoder diff --git a/official/nlp/modeling/networks/classification.py b/official/nlp/modeling/networks/classification.py index c0a71c4d4..17fa18cfa 100755 --- a/official/nlp/modeling/networks/classification.py +++ b/official/nlp/modeling/networks/classification.py @@ -15,6 +15,7 @@ """Classification and regression network.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf diff --git a/official/nlp/modeling/networks/classification_test.py b/official/nlp/modeling/networks/classification_test.py index 600e16c47..9a850013d 100755 --- a/official/nlp/modeling/networks/classification_test.py +++ b/official/nlp/modeling/networks/classification_test.py @@ -14,15 +14,14 @@ # ============================================================================== """Tests for classification network.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.networks import classification diff --git a/official/nlp/modeling/networks/encoder_scaffold.py b/official/nlp/modeling/networks/encoder_scaffold.py index e278199c5..5e5b93947 100755 --- a/official/nlp/modeling/networks/encoder_scaffold.py +++ b/official/nlp/modeling/networks/encoder_scaffold.py @@ -17,9 +17,9 @@ # pylint: disable=g-classes-have-attributes import inspect -from absl import logging import gin import tensorflow as tf +from absl import logging from official.nlp import keras_nlp from official.nlp.modeling import layers diff --git a/official/nlp/modeling/networks/encoder_scaffold_test.py b/official/nlp/modeling/networks/encoder_scaffold_test.py index 1ca9e2a6b..95f0d3c85 100755 --- a/official/nlp/modeling/networks/encoder_scaffold_test.py +++ b/official/nlp/modeling/networks/encoder_scaffold_test.py @@ -14,11 +14,12 @@ # ============================================================================== """Tests for EncoderScaffold network.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.modeling import activations from official.nlp.modeling import layers from official.nlp.modeling.networks import encoder_scaffold diff --git a/official/nlp/modeling/networks/mobile_bert_encoder_test.py b/official/nlp/modeling/networks/mobile_bert_encoder_test.py index f2f2b0c26..3cdca4387 100755 --- a/official/nlp/modeling/networks/mobile_bert_encoder_test.py +++ b/official/nlp/modeling/networks/mobile_bert_encoder_test.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -from absl.testing import parameterized - import numpy as np import tensorflow as tf +from absl.testing import parameterized + from official.nlp.modeling import models from official.nlp.modeling.networks import mobile_bert_encoder diff --git a/official/nlp/modeling/networks/packed_sequence_embedding.py b/official/nlp/modeling/networks/packed_sequence_embedding.py index f73355985..d2a3b9539 100755 --- a/official/nlp/modeling/networks/packed_sequence_embedding.py +++ b/official/nlp/modeling/networks/packed_sequence_embedding.py @@ -16,6 +16,7 @@ """An embedding network supporting packed sequences and position ids.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf from official.modeling import tf_utils diff --git a/official/nlp/modeling/networks/packed_sequence_embedding_test.py b/official/nlp/modeling/networks/packed_sequence_embedding_test.py index 4d2b2354d..dcf6abe51 100755 --- a/official/nlp/modeling/networks/packed_sequence_embedding_test.py +++ b/official/nlp/modeling/networks/packed_sequence_embedding_test.py @@ -17,9 +17,9 @@ # Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.nlp.modeling.networks import packed_sequence_embedding diff --git a/official/nlp/modeling/networks/span_labeling.py b/official/nlp/modeling/networks/span_labeling.py index 896df2034..6133de710 100755 --- a/official/nlp/modeling/networks/span_labeling.py +++ b/official/nlp/modeling/networks/span_labeling.py @@ -15,6 +15,7 @@ """Span labeling network.""" # pylint: disable=g-classes-have-attributes import collections + import tensorflow as tf diff --git a/official/nlp/modeling/networks/span_labeling_test.py b/official/nlp/modeling/networks/span_labeling_test.py index ac1cb1f46..67bd1451e 100755 --- a/official/nlp/modeling/networks/span_labeling_test.py +++ b/official/nlp/modeling/networks/span_labeling_test.py @@ -15,8 +15,9 @@ """Tests for span_labeling network.""" import numpy as np import tensorflow as tf +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import from official.nlp.modeling.networks import span_labeling diff --git a/official/nlp/modeling/networks/xlnet_base.py b/official/nlp/modeling/networks/xlnet_base.py index 3ee785d07..fdd3d968c 100755 --- a/official/nlp/modeling/networks/xlnet_base.py +++ b/official/nlp/modeling/networks/xlnet_base.py @@ -15,9 +15,8 @@ # ============================================================================== """Keras-based XLNet Model.""" -from absl import logging - import tensorflow as tf +from absl import logging from official.nlp.modeling import layers from official.nlp.modeling.layers import transformer_xl diff --git a/official/nlp/modeling/networks/xlnet_base_test.py b/official/nlp/modeling/networks/xlnet_base_test.py index cbc008501..eff523b26 100755 --- a/official/nlp/modeling/networks/xlnet_base_test.py +++ b/official/nlp/modeling/networks/xlnet_base_test.py @@ -12,15 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np import tensorflow as tf - from tensorflow.python.distribute import combinations -from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import + from official.nlp.modeling.networks import xlnet_base diff --git a/official/nlp/modeling/ops/__init__.py b/official/nlp/modeling/ops/__init__.py index 558748001..ddf4e6b0e 100755 --- a/official/nlp/modeling/ops/__init__.py +++ b/official/nlp/modeling/ops/__init__.py @@ -14,5 +14,5 @@ # ============================================================================== """Ops package definition.""" from official.nlp.modeling.ops.beam_search import sequence_beam_search -from official.nlp.modeling.ops.segment_extractor import get_next_sentence_labels -from official.nlp.modeling.ops.segment_extractor import get_sentence_order_labels +from official.nlp.modeling.ops.segment_extractor import ( + get_next_sentence_labels, get_sentence_order_labels) diff --git a/official/nlp/modeling/ops/beam_search_test.py b/official/nlp/modeling/ops/beam_search_test.py index 6db022434..2da96affe 100755 --- a/official/nlp/modeling/ops/beam_search_test.py +++ b/official/nlp/modeling/ops/beam_search_test.py @@ -14,8 +14,8 @@ # ============================================================================== """Test beam search helper methods.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized from official.nlp.modeling.ops import beam_search diff --git a/official/nlp/modeling/ops/decoding_module.py b/official/nlp/modeling/ops/decoding_module.py index 7a59841ff..d54e34411 100755 --- a/official/nlp/modeling/ops/decoding_module.py +++ b/official/nlp/modeling/ops/decoding_module.py @@ -18,7 +18,6 @@ from typing import Any, Callable, Dict, Tuple import tensorflow as tf - from tensorflow.python.framework import dtypes Output = Tuple[tf.Tensor, tf.Tensor] @@ -284,6 +283,3 @@ def inf(self): return dtypes.float16.max else: raise AssertionError("Invalid dtype: %s" % self.dtype) - - - diff --git a/official/nlp/modeling/ops/decoding_module_test.py b/official/nlp/modeling/ops/decoding_module_test.py index 41f40cfb8..a51ed8736 100755 --- a/official/nlp/modeling/ops/decoding_module_test.py +++ b/official/nlp/modeling/ops/decoding_module_test.py @@ -15,6 +15,7 @@ """Test decoding utility methods.""" import abc + import tensorflow as tf from official.nlp.modeling.ops import decoding_module diff --git a/official/nlp/modeling/ops/sampling_module.py b/official/nlp/modeling/ops/sampling_module.py index 181965bdb..992738061 100755 --- a/official/nlp/modeling/ops/sampling_module.py +++ b/official/nlp/modeling/ops/sampling_module.py @@ -438,10 +438,3 @@ def _finished_flags(self, topk_ids, state) -> tf.Tensor: new_finished_flags = tf.logical_or( new_finished_flags, state[decoding_module.StateKeys.FINISHED_FLAGS]) return new_finished_flags - - - - - - - diff --git a/official/nlp/modeling/ops/segment_extractor_test.py b/official/nlp/modeling/ops/segment_extractor_test.py index 6c9fef013..d8f3064a2 100755 --- a/official/nlp/modeling/ops/segment_extractor_test.py +++ b/official/nlp/modeling/ops/segment_extractor_test.py @@ -18,8 +18,8 @@ """Tests for sentence prediction labels.""" import functools -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized from official.nlp.modeling.ops import segment_extractor diff --git a/official/nlp/nhnet/configs.py b/official/nlp/nhnet/configs.py index 41cfa6117..e646136e7 100755 --- a/official/nlp/nhnet/configs.py +++ b/official/nlp/nhnet/configs.py @@ -15,9 +15,8 @@ # ============================================================================== """Common NHNet/Bert2Bert configuration.""" -from typing import List, Text - import dataclasses +from typing import List, Text from official.modeling.hyperparams import base_config diff --git a/official/nlp/nhnet/configs_test.py b/official/nlp/nhnet/configs_test.py index 1260b646c..556860a27 100755 --- a/official/nlp/nhnet/configs_test.py +++ b/official/nlp/nhnet/configs_test.py @@ -15,6 +15,7 @@ """Tests for configs.""" import tensorflow as tf + from official.nlp.nhnet import configs BERT2BERT_CONFIG = { diff --git a/official/nlp/nhnet/decoder.py b/official/nlp/nhnet/decoder.py index 8f8ffa4a6..6a955c6a1 100755 --- a/official/nlp/nhnet/decoder.py +++ b/official/nlp/nhnet/decoder.py @@ -15,6 +15,7 @@ """Transformer decoder that mimics a BERT encoder, to load BERT checkpoints.""" import tensorflow as tf + from official.modeling import tf_utils from official.nlp.modeling import layers from official.nlp.transformer import model_utils as transformer_utils diff --git a/official/nlp/nhnet/decoder_test.py b/official/nlp/nhnet/decoder_test.py index 9ec4ae721..304e0df18 100755 --- a/official/nlp/nhnet/decoder_test.py +++ b/official/nlp/nhnet/decoder_test.py @@ -16,10 +16,9 @@ import numpy as np import tensorflow as tf + from official.nlp.modeling import layers -from official.nlp.nhnet import configs -from official.nlp.nhnet import decoder -from official.nlp.nhnet import utils +from official.nlp.nhnet import configs, decoder, utils class DecoderTest(tf.test.TestCase): diff --git a/official/nlp/nhnet/evaluation.py b/official/nlp/nhnet/evaluation.py index 76f9e3113..012201e7f 100755 --- a/official/nlp/nhnet/evaluation.py +++ b/official/nlp/nhnet/evaluation.py @@ -16,13 +16,13 @@ """Evaluation for Bert2Bert.""" import os -# Import libraries -from absl import logging + import numpy as np import tensorflow as tf +# Import libraries +from absl import logging -from official.nlp.nhnet import input_pipeline -from official.nlp.nhnet import models +from official.nlp.nhnet import input_pipeline, models from official.nlp.transformer import metrics as metrics_v2 from official.nlp.transformer.utils import metrics diff --git a/official/nlp/nhnet/models.py b/official/nlp/nhnet/models.py index 805557139..3a3e38a43 100755 --- a/official/nlp/nhnet/models.py +++ b/official/nlp/nhnet/models.py @@ -13,19 +13,18 @@ # limitations under the License. # ============================================================================== """tf.keras Models for NHNet.""" -from absl import logging +from typing import Optional, Text + import gin import tensorflow as tf -from typing import Optional, Text +from absl import logging from official.modeling import tf_utils from official.modeling.hyperparams import params_dict from official.nlp.modeling import networks from official.nlp.modeling.layers import multi_channel_attention -from official.nlp.nhnet import configs -from official.nlp.nhnet import decoder -from official.nlp.nhnet import utils from official.nlp.modeling.ops import beam_search +from official.nlp.nhnet import configs, decoder, utils def embedding_linear(embedding_matrix, x): diff --git a/official/nlp/nhnet/models_test.py b/official/nlp/nhnet/models_test.py index c79242adf..b110b3b0b 100755 --- a/official/nlp/nhnet/models_test.py +++ b/official/nlp/nhnet/models_test.py @@ -16,18 +16,15 @@ import os -from absl import logging -from absl.testing import parameterized import numpy as np import tensorflow as tf - +from absl import logging +from absl.testing import parameterized # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations +from tensorflow.python.distribute import combinations, strategy_combinations + # pylint: enable=g-direct-tensorflow-import -from official.nlp.nhnet import configs -from official.nlp.nhnet import models -from official.nlp.nhnet import utils +from official.nlp.nhnet import configs, models, utils def all_strategy_combinations(): diff --git a/official/nlp/nhnet/raw_data_process.py b/official/nlp/nhnet/raw_data_process.py index eccb1ed7a..216e09c71 100755 --- a/official/nlp/nhnet/raw_data_process.py +++ b/official/nlp/nhnet/raw_data_process.py @@ -17,8 +17,8 @@ import os -from absl import app -from absl import flags +from absl import app, flags + from official.nlp.nhnet import raw_data_processor FLAGS = flags.FLAGS diff --git a/official/nlp/nhnet/trainer.py b/official/nlp/nhnet/trainer.py index 3eda18510..dce214422 100755 --- a/official/nlp/nhnet/trainer.py +++ b/official/nlp/nhnet/trainer.py @@ -17,18 +17,14 @@ import os +import tensorflow as tf # Import libraries -from absl import app -from absl import flags -from absl import logging +from absl import app, flags, logging from six.moves import zip -import tensorflow as tf + from official.common import distribute_utils from official.modeling.hyperparams import params_dict -from official.nlp.nhnet import evaluation -from official.nlp.nhnet import input_pipeline -from official.nlp.nhnet import models -from official.nlp.nhnet import optimizer +from official.nlp.nhnet import evaluation, input_pipeline, models, optimizer from official.nlp.transformer import metrics as transformer_metrics from official.utils.misc import keras_utils diff --git a/official/nlp/nhnet/trainer_test.py b/official/nlp/nhnet/trainer_test.py index dba70651c..7666ae844 100755 --- a/official/nlp/nhnet/trainer_test.py +++ b/official/nlp/nhnet/trainer_test.py @@ -17,16 +17,14 @@ import os +import tensorflow as tf from absl import flags from absl.testing import parameterized -import tensorflow as tf - # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations +from tensorflow.python.distribute import combinations, strategy_combinations + # pylint: enable=g-direct-tensorflow-import -from official.nlp.nhnet import trainer -from official.nlp.nhnet import utils +from official.nlp.nhnet import trainer, utils FLAGS = flags.FLAGS trainer.define_flags() diff --git a/official/nlp/nhnet/utils.py b/official/nlp/nhnet/utils.py index a61f89035..dd6f87764 100755 --- a/official/nlp/nhnet/utils.py +++ b/official/nlp/nhnet/utils.py @@ -13,9 +13,11 @@ # limitations under the License. # ============================================================================== """Utility helpers for Bert2Bert.""" -from absl import logging -import tensorflow as tf from typing import Optional, Text + +import tensorflow as tf +from absl import logging + from official.modeling.hyperparams import params_dict from official.nlp.bert import configs from official.nlp.nhnet import configs as nhnet_configs diff --git a/official/nlp/optimization.py b/official/nlp/optimization.py index 033d942cb..71a079677 100755 --- a/official/nlp/optimization.py +++ b/official/nlp/optimization.py @@ -16,10 +16,10 @@ import re -from absl import logging import gin import tensorflow as tf import tensorflow_addons.optimizers as tfa_optimizers +from absl import logging class WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule): diff --git a/official/nlp/projects/triviaqa/dataset.py b/official/nlp/projects/triviaqa/dataset.py index 6dc57d29b..2e02b715c 100755 --- a/official/nlp/projects/triviaqa/dataset.py +++ b/official/nlp/projects/triviaqa/dataset.py @@ -16,12 +16,12 @@ import json import os -from absl import logging -import apache_beam as beam import six import tensorflow as tf import tensorflow_datasets.public_api as tfds +from absl import logging +import apache_beam as beam from official.nlp.projects.triviaqa import preprocess _CITATION = """ diff --git a/official/nlp/projects/triviaqa/download_and_prepare.py b/official/nlp/projects/triviaqa/download_and_prepare.py index 2400dee43..0346f178e 100755 --- a/official/nlp/projects/triviaqa/download_and_prepare.py +++ b/official/nlp/projects/triviaqa/download_and_prepare.py @@ -14,13 +14,12 @@ """Downloads and prepares TriviaQA dataset.""" from unittest import mock -from absl import app -from absl import flags -from absl import logging -import apache_beam as beam import tensorflow_datasets as tfds +from absl import app, flags, logging -from official.nlp.projects.triviaqa import dataset # pylint: disable=unused-import +import apache_beam as beam +from official.nlp.projects.triviaqa import \ + dataset # pylint: disable=unused-import flags.DEFINE_integer('sequence_length', 4096, 'Max number of tokens.') diff --git a/official/nlp/projects/triviaqa/evaluate.py b/official/nlp/projects/triviaqa/evaluate.py index b91bd436f..fda843afa 100755 --- a/official/nlp/projects/triviaqa/evaluate.py +++ b/official/nlp/projects/triviaqa/evaluate.py @@ -14,10 +14,8 @@ """Evalutes TriviaQA predictions.""" import json -from absl import app -from absl import flags -from absl import logging import tensorflow as tf +from absl import app, flags, logging from official.nlp.projects.triviaqa import evaluation diff --git a/official/nlp/projects/triviaqa/inputs.py b/official/nlp/projects/triviaqa/inputs.py index 15b9065f5..c4af97cfe 100755 --- a/official/nlp/projects/triviaqa/inputs.py +++ b/official/nlp/projects/triviaqa/inputs.py @@ -20,7 +20,8 @@ import tensorflow_datasets as tfds from official.modeling import tf_utils -from official.nlp.projects.triviaqa import dataset # pylint: disable=unused-import +from official.nlp.projects.triviaqa import \ + dataset # pylint: disable=unused-import def _flatten_dims(tensor: tf.Tensor, diff --git a/official/nlp/projects/triviaqa/predict.py b/official/nlp/projects/triviaqa/predict.py index 844a3103c..8b0b734ba 100755 --- a/official/nlp/projects/triviaqa/predict.py +++ b/official/nlp/projects/triviaqa/predict.py @@ -18,17 +18,13 @@ import json import operator -from absl import app -from absl import flags -from absl import logging +import sentencepiece as spm import tensorflow as tf import tensorflow_datasets as tfds +from absl import app, flags, logging -import sentencepiece as spm from official.nlp.configs import encoders # pylint: disable=unused-import -from official.nlp.projects.triviaqa import evaluation -from official.nlp.projects.triviaqa import inputs -from official.nlp.projects.triviaqa import prediction +from official.nlp.projects.triviaqa import evaluation, inputs, prediction flags.DEFINE_string('data_dir', None, 'TensorFlow Datasets directory.') diff --git a/official/nlp/projects/triviaqa/preprocess.py b/official/nlp/projects/triviaqa/preprocess.py index dfe6c3332..74ca84851 100755 --- a/official/nlp/projects/triviaqa/preprocess.py +++ b/official/nlp/projects/triviaqa/preprocess.py @@ -13,6 +13,7 @@ # limitations under the License. """Utilities for preprocessing TriviaQA data.""" import bisect +import dataclasses import json import operator import os @@ -20,17 +21,15 @@ import string from typing import Any, Dict, Generator, List, Optional, Set, Text, Tuple -from absl import logging -import apache_beam as beam -from apache_beam import metrics -import dataclasses import nltk import numpy as np +import sentencepiece as spm import tensorflow.io.gfile as gfile +from absl import logging -import sentencepiece as spm -from official.nlp.projects.triviaqa import evaluation -from official.nlp.projects.triviaqa import sentencepiece_pb2 +import apache_beam as beam +from apache_beam import metrics +from official.nlp.projects.triviaqa import evaluation, sentencepiece_pb2 @dataclasses.dataclass diff --git a/official/nlp/projects/triviaqa/sentencepiece_pb2.py b/official/nlp/projects/triviaqa/sentencepiece_pb2.py index 8ee94567c..5e53cc2de 100755 --- a/official/nlp/projects/triviaqa/sentencepiece_pb2.py +++ b/official/nlp/projects/triviaqa/sentencepiece_pb2.py @@ -20,6 +20,7 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() diff --git a/official/nlp/projects/triviaqa/train.py b/official/nlp/projects/triviaqa/train.py index 35e5f278e..77768cf88 100755 --- a/official/nlp/projects/triviaqa/train.py +++ b/official/nlp/projects/triviaqa/train.py @@ -19,20 +19,16 @@ import operator import os -from absl import app -from absl import flags -from absl import logging import gin +import sentencepiece as spm import tensorflow as tf import tensorflow_datasets as tfds +from absl import app, flags, logging -import sentencepiece as spm from official.nlp import optimization as nlp_optimization from official.nlp.configs import encoders -from official.nlp.projects.triviaqa import evaluation -from official.nlp.projects.triviaqa import inputs -from official.nlp.projects.triviaqa import modeling -from official.nlp.projects.triviaqa import prediction +from official.nlp.projects.triviaqa import (evaluation, inputs, modeling, + prediction) flags.DEFINE_string('data_dir', None, 'Data directory for TensorFlow Datasets.') diff --git a/official/nlp/tasks/electra_task.py b/official/nlp/tasks/electra_task.py index cb8d8d45f..4468d993d 100755 --- a/official/nlp/tasks/electra_task.py +++ b/official/nlp/tasks/electra_task.py @@ -16,18 +16,16 @@ """ELECTRA pretraining task (Joint Masked LM and Replaced Token Detection).""" import dataclasses + import tensorflow as tf from official.core import base_task from official.core import config_definitions as cfg from official.core import task_factory from official.modeling import tf_utils -from official.nlp.configs import bert -from official.nlp.configs import electra -from official.nlp.configs import encoders +from official.nlp.configs import bert, electra, encoders from official.nlp.data import pretrain_dataloader -from official.nlp.modeling import layers -from official.nlp.modeling import models +from official.nlp.modeling import layers, models @dataclasses.dataclass diff --git a/official/nlp/tasks/electra_task_test.py b/official/nlp/tasks/electra_task_test.py index 061ee35af..dbb018ab0 100755 --- a/official/nlp/tasks/electra_task_test.py +++ b/official/nlp/tasks/electra_task_test.py @@ -17,9 +17,7 @@ import tensorflow as tf -from official.nlp.configs import bert -from official.nlp.configs import electra -from official.nlp.configs import encoders +from official.nlp.configs import bert, electra, encoders from official.nlp.data import pretrain_dataloader from official.nlp.tasks import electra_task diff --git a/official/nlp/tasks/masked_lm.py b/official/nlp/tasks/masked_lm.py index 44fb43ca2..3fddd38af 100755 --- a/official/nlp/tasks/masked_lm.py +++ b/official/nlp/tasks/masked_lm.py @@ -16,17 +16,16 @@ """Masked language task.""" import dataclasses + import tensorflow as tf from official.core import base_task from official.core import config_definitions as cfg from official.core import task_factory from official.modeling import tf_utils -from official.nlp.configs import bert -from official.nlp.configs import encoders +from official.nlp.configs import bert, encoders from official.nlp.data import data_loader_factory -from official.nlp.modeling import layers -from official.nlp.modeling import models +from official.nlp.modeling import layers, models @dataclasses.dataclass diff --git a/official/nlp/tasks/masked_lm_test.py b/official/nlp/tasks/masked_lm_test.py index 937862da2..261c5f65b 100755 --- a/official/nlp/tasks/masked_lm_test.py +++ b/official/nlp/tasks/masked_lm_test.py @@ -17,8 +17,7 @@ import tensorflow as tf -from official.nlp.configs import bert -from official.nlp.configs import encoders +from official.nlp.configs import bert, encoders from official.nlp.data import pretrain_dataloader from official.nlp.tasks import masked_lm diff --git a/official/nlp/tasks/question_answering.py b/official/nlp/tasks/question_answering.py index 698baf820..1502394c3 100755 --- a/official/nlp/tasks/question_answering.py +++ b/official/nlp/tasks/question_answering.py @@ -14,23 +14,22 @@ # limitations under the License. # ============================================================================== """Question answering task.""" +import dataclasses import functools import json import os from typing import List, Optional -from absl import logging -import dataclasses -import orbit import tensorflow as tf +from absl import logging +import orbit from official.core import base_task from official.core import config_definitions as cfg from official.core import task_factory from official.modeling.hyperparams import base_config -from official.nlp.bert import squad_evaluate_v1_1 -from official.nlp.bert import squad_evaluate_v2_0 -from official.nlp.bert import tokenization +from official.nlp.bert import (squad_evaluate_v1_1, squad_evaluate_v2_0, + tokenization) from official.nlp.configs import encoders from official.nlp.data import data_loader_factory from official.nlp.data import squad_lib as squad_lib_wp diff --git a/official/nlp/tasks/question_answering_test.py b/official/nlp/tasks/question_answering_test.py index 114149473..556a90359 100755 --- a/official/nlp/tasks/question_answering_test.py +++ b/official/nlp/tasks/question_answering_test.py @@ -18,14 +18,12 @@ import json import os -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized -from official.nlp.configs import bert -from official.nlp.configs import encoders +from official.nlp.configs import bert, encoders from official.nlp.data import question_answering_dataloader -from official.nlp.tasks import masked_lm -from official.nlp.tasks import question_answering +from official.nlp.tasks import masked_lm, question_answering class QuestionAnsweringTaskTest(tf.test.TestCase, parameterized.TestCase): diff --git a/official/nlp/tasks/sentence_prediction.py b/official/nlp/tasks/sentence_prediction.py index 1896b67b1..61500d503 100755 --- a/official/nlp/tasks/sentence_prediction.py +++ b/official/nlp/tasks/sentence_prediction.py @@ -14,16 +14,16 @@ # limitations under the License. # ============================================================================== """Sentence prediction (classification) task.""" -from typing import List, Union, Optional - -from absl import logging import dataclasses +from typing import List, Optional, Union + import numpy as np -import orbit +import tensorflow as tf +from absl import logging from scipy import stats from sklearn import metrics as sklearn_metrics -import tensorflow as tf +import orbit from official.core import base_task from official.core import config_definitions as cfg from official.core import task_factory diff --git a/official/nlp/tasks/sentence_prediction_test.py b/official/nlp/tasks/sentence_prediction_test.py index 5341405b6..cb9504ffc 100755 --- a/official/nlp/tasks/sentence_prediction_test.py +++ b/official/nlp/tasks/sentence_prediction_test.py @@ -17,15 +17,13 @@ import functools import os -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized -from official.nlp.configs import bert -from official.nlp.configs import encoders +from official.nlp.configs import bert, encoders from official.nlp.data import sentence_prediction_dataloader -from official.nlp.tasks import masked_lm -from official.nlp.tasks import sentence_prediction +from official.nlp.tasks import masked_lm, sentence_prediction def _create_fake_dataset(output_path, seq_length, num_classes, num_examples): diff --git a/official/nlp/tasks/tagging.py b/official/nlp/tasks/tagging.py index 0ebda401f..106c38493 100755 --- a/official/nlp/tasks/tagging.py +++ b/official/nlp/tasks/tagging.py @@ -14,15 +14,12 @@ # limitations under the License. # ============================================================================== """Tagging (e.g., NER/POS) task.""" -from typing import List, Optional, Tuple - import dataclasses -import orbit - -from seqeval import metrics as seqeval_metrics +from typing import List, Optional, Tuple import tensorflow as tf +import orbit from official.core import base_task from official.core import config_definitions as cfg from official.core import task_factory @@ -31,6 +28,7 @@ from official.nlp.data import data_loader_factory from official.nlp.modeling import models from official.nlp.tasks import utils +from seqeval import metrics as seqeval_metrics @dataclasses.dataclass diff --git a/official/nlp/tasks/translation.py b/official/nlp/tasks/translation.py index d09c7b76f..b24fdd115 100755 --- a/official/nlp/tasks/translation.py +++ b/official/nlp/tasks/translation.py @@ -14,15 +14,15 @@ # limitations under the License. # ============================================================================== """Defines the translation task.""" +import dataclasses import os from typing import Optional +import tensorflow as tf from absl import logging -import dataclasses + import sacrebleu -import tensorflow as tf import tensorflow_text as tftxt - from official.core import base_task from official.core import config_definitions as cfg from official.core import task_factory diff --git a/official/nlp/tasks/translation_test.py b/official/nlp/tasks/translation_test.py index 0010c0828..20f0a302d 100755 --- a/official/nlp/tasks/translation_test.py +++ b/official/nlp/tasks/translation_test.py @@ -17,10 +17,10 @@ import functools import os -import orbit import tensorflow as tf - from sentencepiece import SentencePieceTrainer + +import orbit from official.nlp.data import wmt_dataloader from official.nlp.tasks import translation diff --git a/official/nlp/tasks/utils.py b/official/nlp/tasks/utils.py index 66228f912..e4ba9226a 100755 --- a/official/nlp/tasks/utils.py +++ b/official/nlp/tasks/utils.py @@ -16,10 +16,11 @@ """Common utils for tasks.""" from typing import Any, Callable -import orbit import tensorflow as tf import tensorflow_hub as hub +import orbit + def get_encoder_from_hub(hub_model_path: str) -> tf.keras.Model: """Gets an encoder from hub. diff --git a/official/nlp/train.py b/official/nlp/train.py index 8bb637b9b..4ddaf613a 100755 --- a/official/nlp/train.py +++ b/official/nlp/train.py @@ -15,18 +15,15 @@ # ============================================================================== """TFM common training driver.""" -from absl import app -from absl import flags import gin +from absl import app, flags -from official.common import distribute_utils -# pylint: disable=unused-import -from official.common import registry_imports # pylint: enable=unused-import +# pylint: disable=unused-import +from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib -from official.core import train_utils +from official.common import registry_imports +from official.core import task_factory, train_lib, train_utils from official.modeling import performance FLAGS = flags.FLAGS diff --git a/official/nlp/train_ctl_continuous_finetune.py b/official/nlp/train_ctl_continuous_finetune.py index b8af9a7f6..51872276d 100755 --- a/official/nlp/train_ctl_continuous_finetune.py +++ b/official/nlp/train_ctl_continuous_finetune.py @@ -14,14 +14,13 @@ # limitations under the License. # ============================================================================== """TFM continuous finetuning+eval training driver.""" -from absl import app -from absl import flags import gin +from absl import app, flags -# pylint: disable=unused-import -from official.common import registry_imports # pylint: enable=unused-import +# pylint: disable=unused-import from official.common import flags as tfm_flags +from official.common import registry_imports from official.core import train_utils from official.nlp import continuous_finetune_lib diff --git a/official/nlp/transformer/attention_layer.py b/official/nlp/transformer/attention_layer.py index 6581162f5..122a80852 100755 --- a/official/nlp/transformer/attention_layer.py +++ b/official/nlp/transformer/attention_layer.py @@ -16,6 +16,7 @@ import math import tensorflow as tf + from official.nlp.modeling import layers diff --git a/official/nlp/transformer/beam_search_v1.py b/official/nlp/transformer/beam_search_v1.py index 580402589..a2e4e6686 100755 --- a/official/nlp/transformer/beam_search_v1.py +++ b/official/nlp/transformer/beam_search_v1.py @@ -15,6 +15,7 @@ """Beam search to find the translated sequence with the highest probability.""" import tensorflow.compat.v1 as tf + from official.nlp.modeling.ops import beam_search _StateKeys = beam_search._StateKeys # pylint: disable=protected-access diff --git a/official/nlp/transformer/compute_bleu.py b/official/nlp/transformer/compute_bleu.py index 46ee23dfd..558f1bb7b 100755 --- a/official/nlp/transformer/compute_bleu.py +++ b/official/nlp/transformer/compute_bleu.py @@ -22,14 +22,12 @@ import sys import unicodedata -from absl import app -from absl import flags import six -from six.moves import range import tensorflow as tf +from absl import app, flags +from six.moves import range -from official.nlp.transformer.utils import metrics -from official.nlp.transformer.utils import tokenizer +from official.nlp.transformer.utils import metrics, tokenizer from official.utils.flags import core as flags_core diff --git a/official/nlp/transformer/data_download.py b/official/nlp/transformer/data_download.py index e3ce7fceb..9142a1f22 100755 --- a/official/nlp/transformer/data_download.py +++ b/official/nlp/transformer/data_download.py @@ -18,19 +18,17 @@ import random import tarfile -# pylint: disable=g-bad-import-order - -from absl import app -from absl import flags -from absl import logging import six -from six.moves import range -from six.moves import urllib -from six.moves import zip import tensorflow.compat.v1 as tf +from absl import app, flags, logging +from six.moves import range, urllib, zip from official.nlp.transformer.utils import tokenizer from official.utils.flags import core as flags_core + +# pylint: disable=g-bad-import-order + + # pylint: enable=g-bad-import-order # Data sources for training/evaluating the transformer translation model. diff --git a/official/nlp/transformer/data_pipeline.py b/official/nlp/transformer/data_pipeline.py index 684615cb2..751fb32cf 100755 --- a/official/nlp/transformer/data_pipeline.py +++ b/official/nlp/transformer/data_pipeline.py @@ -49,8 +49,8 @@ import os -from absl import logging import tensorflow as tf +from absl import logging from official.utils.misc import model_helpers diff --git a/official/nlp/transformer/misc.py b/official/nlp/transformer/misc.py index 424fd000d..7e47d982e 100755 --- a/official/nlp/transformer/misc.py +++ b/official/nlp/transformer/misc.py @@ -16,8 +16,8 @@ # pylint: disable=g-bad-import-order -from absl import flags import tensorflow as tf +from absl import flags from official.nlp.transformer import model_params from official.utils.flags import core as flags_core diff --git a/official/nlp/transformer/model_params.py b/official/nlp/transformer/model_params.py index e978abeaf..b4eecdea6 100755 --- a/official/nlp/transformer/model_params.py +++ b/official/nlp/transformer/model_params.py @@ -16,7 +16,6 @@ from collections import defaultdict - BASE_PARAMS = defaultdict( lambda: None, # Set default value to None. diff --git a/official/nlp/transformer/transformer.py b/official/nlp/transformer/transformer.py index c5e167a2b..077ba5966 100755 --- a/official/nlp/transformer/transformer.py +++ b/official/nlp/transformer/transformer.py @@ -19,13 +19,11 @@ """ import tensorflow as tf + from official.nlp.modeling.layers import position_embedding from official.nlp.modeling.ops import beam_search -from official.nlp.transformer import attention_layer -from official.nlp.transformer import embedding_layer -from official.nlp.transformer import ffn_layer -from official.nlp.transformer import metrics -from official.nlp.transformer import model_utils +from official.nlp.transformer import (attention_layer, embedding_layer, + ffn_layer, metrics, model_utils) from official.nlp.transformer.utils.tokenizer import EOS_ID # Disable the not-callable lint error, since it claims many objects are not diff --git a/official/nlp/transformer/transformer_forward_test.py b/official/nlp/transformer/transformer_forward_test.py index 5f76f948d..9c217e248 100755 --- a/official/nlp/transformer/transformer_forward_test.py +++ b/official/nlp/transformer/transformer_forward_test.py @@ -15,13 +15,10 @@ """Forward pass test for Transformer model refactoring.""" import numpy as np - import tensorflow as tf from official.nlp.modeling import models -from official.nlp.transformer import metrics -from official.nlp.transformer import model_params -from official.nlp.transformer import transformer +from official.nlp.transformer import metrics, model_params, transformer def _count_params(layer, trainable_only=True): diff --git a/official/nlp/transformer/transformer_layers_test.py b/official/nlp/transformer/transformer_layers_test.py index ac64c545b..a02f085e5 100755 --- a/official/nlp/transformer/transformer_layers_test.py +++ b/official/nlp/transformer/transformer_layers_test.py @@ -16,10 +16,8 @@ import tensorflow as tf -from official.nlp.transformer import attention_layer -from official.nlp.transformer import embedding_layer -from official.nlp.transformer import ffn_layer -from official.nlp.transformer import metrics +from official.nlp.transformer import (attention_layer, embedding_layer, + ffn_layer, metrics) class TransformerLayersTest(tf.test.TestCase): diff --git a/official/nlp/transformer/transformer_main.py b/official/nlp/transformer/transformer_main.py index 6b368a868..48e66b6cd 100755 --- a/official/nlp/transformer/transformer_main.py +++ b/official/nlp/transformer/transformer_main.py @@ -21,20 +21,14 @@ import os import tempfile -# Import libraries -from absl import app -from absl import flags -from absl import logging import tensorflow as tf +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils from official.modeling import performance -from official.nlp.transformer import compute_bleu -from official.nlp.transformer import data_pipeline -from official.nlp.transformer import metrics -from official.nlp.transformer import misc -from official.nlp.transformer import optimizer -from official.nlp.transformer import transformer -from official.nlp.transformer import translate +from official.nlp.transformer import (compute_bleu, data_pipeline, metrics, + misc, optimizer, transformer, translate) from official.nlp.transformer.utils import tokenizer from official.utils.flags import core as flags_core from official.utils.misc import keras_utils diff --git a/official/nlp/transformer/transformer_main_test.py b/official/nlp/transformer/transformer_main_test.py index d85549e08..16d23f0a1 100755 --- a/official/nlp/transformer/transformer_main_test.py +++ b/official/nlp/transformer/transformer_main_test.py @@ -19,12 +19,13 @@ import sys import unittest +import tensorflow as tf from absl import flags from absl.testing import flagsaver -import tensorflow as tf -from tensorflow.python.eager import context # pylint: disable=ungrouped-imports -from official.nlp.transformer import misc -from official.nlp.transformer import transformer_main +from tensorflow.python.eager import \ + context # pylint: disable=ungrouped-imports + +from official.nlp.transformer import misc, transformer_main from official.utils.misc import keras_utils FLAGS = flags.FLAGS diff --git a/official/nlp/transformer/transformer_test.py b/official/nlp/transformer/transformer_test.py index 3068c9f73..79afcf092 100755 --- a/official/nlp/transformer/transformer_test.py +++ b/official/nlp/transformer/transformer_test.py @@ -16,8 +16,7 @@ import tensorflow as tf -from official.nlp.transformer import model_params -from official.nlp.transformer import transformer +from official.nlp.transformer import model_params, transformer class TransformerV2Test(tf.test.TestCase): diff --git a/official/nlp/transformer/translate.py b/official/nlp/transformer/translate.py index 68ef6bc14..fd2bcb250 100755 --- a/official/nlp/transformer/translate.py +++ b/official/nlp/transformer/translate.py @@ -14,10 +14,10 @@ # ============================================================================== """Translate text or files using trained transformer model.""" -# Import libraries -from absl import logging import numpy as np import tensorflow as tf +# Import libraries +from absl import logging from official.nlp.transformer.utils import tokenizer diff --git a/official/nlp/transformer/utils/metrics.py b/official/nlp/transformer/utils/metrics.py index 6b8d1f7d2..1f97fbe04 100755 --- a/official/nlp/transformer/utils/metrics.py +++ b/official/nlp/transformer/utils/metrics.py @@ -23,17 +23,15 @@ https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/rouge.py """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import collections import math import numpy as np import six -from six.moves import xrange # pylint: disable=redefined-builtin import tensorflow.compat.v1 as tf +from six.moves import xrange # pylint: disable=redefined-builtin def _pad_tensors_to_same_length(x, y): diff --git a/official/nlp/transformer/utils/tokenizer.py b/official/nlp/transformer/utils/tokenizer.py index 0520c2e9f..f062d5b1c 100755 --- a/official/nlp/transformer/utils/tokenizer.py +++ b/official/nlp/transformer/utils/tokenizer.py @@ -14,21 +14,18 @@ # ============================================================================== """Defines Subtokenizer class to encode and decode strings.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import collections import re import sys import unicodedata -from absl import logging - import numpy as np import six -from six.moves import xrange # pylint: disable=redefined-builtin import tensorflow as tf +from absl import logging +from six.moves import xrange # pylint: disable=redefined-builtin # pylint: disable=g-complex-comprehension PAD = "" diff --git a/official/nlp/xlnet/__init__.py b/official/nlp/xlnet/__init__.py index 8b1378917..e69de29bb 100755 --- a/official/nlp/xlnet/__init__.py +++ b/official/nlp/xlnet/__init__.py @@ -1 +0,0 @@ - diff --git a/official/nlp/xlnet/data_utils.py b/official/nlp/xlnet/data_utils.py index ec3b1ff5c..f8e8b34a5 100755 --- a/official/nlp/xlnet/data_utils.py +++ b/official/nlp/xlnet/data_utils.py @@ -18,10 +18,9 @@ import json import os -from absl import logging - import numpy as np import tensorflow as tf +from absl import logging special_symbols = { "": 0, diff --git a/official/nlp/xlnet/optimization.py b/official/nlp/xlnet/optimization.py index aff1304c6..b082cc4d0 100755 --- a/official/nlp/xlnet/optimization.py +++ b/official/nlp/xlnet/optimization.py @@ -14,8 +14,9 @@ # ============================================================================== """Functions and classes related to optimization (weight updates).""" -from absl import logging import tensorflow as tf +from absl import logging + from official.nlp import optimization diff --git a/official/nlp/xlnet/preprocess_classification_data.py b/official/nlp/xlnet/preprocess_classification_data.py index b6cbf3057..f65fa9677 100755 --- a/official/nlp/xlnet/preprocess_classification_data.py +++ b/official/nlp/xlnet/preprocess_classification_data.py @@ -18,17 +18,13 @@ import csv import os -# Import libraries -from absl import app -from absl import flags -from absl import logging import numpy as np -import tensorflow as tf - import sentencepiece as spm -from official.nlp.xlnet import classifier_utils -from official.nlp.xlnet import preprocess_utils +import tensorflow as tf +# Import libraries +from absl import app, flags, logging +from official.nlp.xlnet import classifier_utils, preprocess_utils flags.DEFINE_bool( "overwrite_data", diff --git a/official/nlp/xlnet/preprocess_pretrain_data.py b/official/nlp/xlnet/preprocess_pretrain_data.py index a9b096eb2..13fffb3fb 100755 --- a/official/nlp/xlnet/preprocess_pretrain_data.py +++ b/official/nlp/xlnet/preprocess_pretrain_data.py @@ -19,18 +19,14 @@ import os import random -# Import libraries -from absl import app -from absl import flags import absl.logging as _logging # pylint: disable=unused-import - import numpy as np - - -import tensorflow.google as tf -from official.nlp.xlnet import preprocess_utils import sentencepiece as spm +import tensorflow.google as tf +# Import libraries +from absl import app, flags +from official.nlp.xlnet import preprocess_utils special_symbols = { "" : 0, diff --git a/official/nlp/xlnet/preprocess_squad_data.py b/official/nlp/xlnet/preprocess_squad_data.py index 18f1bb316..4759819c6 100755 --- a/official/nlp/xlnet/preprocess_squad_data.py +++ b/official/nlp/xlnet/preprocess_squad_data.py @@ -18,13 +18,11 @@ import os import random -# Import libraries -from absl import app -from absl import flags -from absl import logging +import sentencepiece as spm import tensorflow as tf +# Import libraries +from absl import app, flags, logging -import sentencepiece as spm from official.nlp.xlnet import squad_utils flags.DEFINE_integer( diff --git a/official/nlp/xlnet/run_classifier.py b/official/nlp/xlnet/run_classifier.py index 0e878430d..65b06b086 100755 --- a/official/nlp/xlnet/run_classifier.py +++ b/official/nlp/xlnet/run_classifier.py @@ -15,20 +15,16 @@ """XLNet classification finetuning runner in tf2.0.""" import functools -# Import libraries -from absl import app -from absl import flags -from absl import logging import numpy as np import tensorflow as tf +# Import libraries +from absl import app, flags, logging + # pylint: disable=unused-import from official.common import distribute_utils -from official.nlp.xlnet import common_flags -from official.nlp.xlnet import data_utils -from official.nlp.xlnet import optimization -from official.nlp.xlnet import training_utils -from official.nlp.xlnet import xlnet_config +from official.nlp.xlnet import (common_flags, data_utils, optimization, + training_utils, xlnet_config) from official.nlp.xlnet import xlnet_modeling as modeling flags.DEFINE_integer("n_class", default=2, help="Number of classes.") diff --git a/official/nlp/xlnet/run_pretrain.py b/official/nlp/xlnet/run_pretrain.py index 68710ba9b..1f017863c 100755 --- a/official/nlp/xlnet/run_pretrain.py +++ b/official/nlp/xlnet/run_pretrain.py @@ -17,18 +17,14 @@ import functools import os -# Import libraries -from absl import app -from absl import flags -from absl import logging import tensorflow as tf +# Import libraries +from absl import app, flags, logging + # pylint: disable=unused-import from official.common import distribute_utils -from official.nlp.xlnet import common_flags -from official.nlp.xlnet import data_utils -from official.nlp.xlnet import optimization -from official.nlp.xlnet import training_utils -from official.nlp.xlnet import xlnet_config +from official.nlp.xlnet import (common_flags, data_utils, optimization, + training_utils, xlnet_config) from official.nlp.xlnet import xlnet_modeling as modeling flags.DEFINE_integer( diff --git a/official/nlp/xlnet/run_squad.py b/official/nlp/xlnet/run_squad.py index be3a27079..066215f0a 100755 --- a/official/nlp/xlnet/run_squad.py +++ b/official/nlp/xlnet/run_squad.py @@ -19,21 +19,15 @@ import os import pickle -# Import libraries -from absl import app -from absl import flags -from absl import logging - -import tensorflow as tf # pylint: disable=unused-import import sentencepiece as spm +import tensorflow as tf +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils -from official.nlp.xlnet import common_flags -from official.nlp.xlnet import data_utils -from official.nlp.xlnet import optimization -from official.nlp.xlnet import squad_utils -from official.nlp.xlnet import training_utils -from official.nlp.xlnet import xlnet_config +from official.nlp.xlnet import (common_flags, data_utils, optimization, + squad_utils, training_utils, xlnet_config) from official.nlp.xlnet import xlnet_modeling as modeling flags.DEFINE_string( diff --git a/official/nlp/xlnet/squad_utils.py b/official/nlp/xlnet/squad_utils.py index efab6da6f..089c19619 100755 --- a/official/nlp/xlnet/squad_utils.py +++ b/official/nlp/xlnet/squad_utils.py @@ -14,10 +14,8 @@ # ============================================================================== # coding=utf-8 """Utilities used in SQUAD task.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import collections import gc @@ -28,13 +26,12 @@ import re import string -from absl import logging import numpy as np import six import tensorflow as tf +from absl import logging -from official.nlp.xlnet import data_utils -from official.nlp.xlnet import preprocess_utils +from official.nlp.xlnet import data_utils, preprocess_utils SPIECE_UNDERLINE = u"▁" diff --git a/official/nlp/xlnet/training_utils.py b/official/nlp/xlnet/training_utils.py index be6476fd8..ab465c0e6 100755 --- a/official/nlp/xlnet/training_utils.py +++ b/official/nlp/xlnet/training_utils.py @@ -16,13 +16,12 @@ import os import re - -from absl import logging +from typing import Any, Callable, Dict, Optional, Text # pytype: disable=attribute-error # pylint: disable=g-bare-generic,unused-import import tensorflow as tf -from typing import Any, Callable, Dict, Text, Optional +from absl import logging from official.nlp.bert import model_training_utils from official.nlp.xlnet import data_utils diff --git a/official/pip_package/setup.py b/official/pip_package/setup.py index 6d24d5ce2..d77f9c1d9 100755 --- a/official/pip_package/setup.py +++ b/official/pip_package/setup.py @@ -17,8 +17,7 @@ import os import sys -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup version = '2.4.0' diff --git a/official/recommendation/create_ncf_data.py b/official/recommendation/create_ncf_data.py index 27c2edbb9..af54f5329 100755 --- a/official/recommendation/create_ncf_data.py +++ b/official/recommendation/create_ncf_data.py @@ -16,15 +16,15 @@ import json +import tensorflow as tf # pylint: disable=g-bad-import-order # Import libraries -from absl import app -from absl import flags -import tensorflow as tf +from absl import app, flags + +from official.recommendation import data_preprocessing, movielens + # pylint: enable=g-bad-import-order -from official.recommendation import movielens -from official.recommendation import data_preprocessing flags.DEFINE_string( "data_dir", None, diff --git a/official/recommendation/data_pipeline.py b/official/recommendation/data_pipeline.py index 9cb0bf9e2..32fa61ea7 100755 --- a/official/recommendation/data_pipeline.py +++ b/official/recommendation/data_pipeline.py @@ -14,9 +14,7 @@ # ============================================================================== """Asynchronous data producer for the NCF pipeline.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import atexit import functools @@ -31,15 +29,13 @@ import numpy as np import six -from six.moves import queue import tensorflow as tf from absl import logging +from six.moves import queue +from tensorflow.python.tpu.datasets import StreamingFilesDataset from official.recommendation import constants as rconst -from official.recommendation import movielens -from official.recommendation import popen_helper -from official.recommendation import stat_utils -from tensorflow.python.tpu.datasets import StreamingFilesDataset +from official.recommendation import movielens, popen_helper, stat_utils SUMMARY_TEMPLATE = """General: {spacer}Num users: {num_users} diff --git a/official/recommendation/data_preprocessing.py b/official/recommendation/data_preprocessing.py index 4fc510ba2..184f07257 100755 --- a/official/recommendation/data_preprocessing.py +++ b/official/recommendation/data_preprocessing.py @@ -14,29 +14,28 @@ # ============================================================================== """Preprocess dataset and construct any necessary artifacts.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os import pickle import time import timeit +import typing +from typing import Dict, Text, Tuple -# pylint: disable=wrong-import-order - -from absl import logging import numpy as np import pandas as pd import tensorflow as tf -import typing -from typing import Dict, Text, Tuple -# pylint: enable=wrong-import-order +from absl import logging from official.recommendation import constants as rconst -from official.recommendation import data_pipeline -from official.recommendation import movielens +from official.recommendation import data_pipeline, movielens + +# pylint: disable=wrong-import-order + +# pylint: enable=wrong-import-order + _EXPECTED_CACHE_KEYS = (rconst.TRAIN_USER_KEY, rconst.TRAIN_ITEM_KEY, rconst.EVAL_USER_KEY, rconst.EVAL_ITEM_KEY, diff --git a/official/recommendation/data_test.py b/official/recommendation/data_test.py index 603e12732..c868992a1 100755 --- a/official/recommendation/data_test.py +++ b/official/recommendation/data_test.py @@ -14,24 +14,19 @@ # ============================================================================== """Test NCF data pipeline.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from collections import defaultdict import hashlib import os - -import mock +from collections import defaultdict import numpy as np import scipy.stats import tensorflow as tf +import mock from official.recommendation import constants as rconst -from official.recommendation import data_preprocessing -from official.recommendation import movielens -from official.recommendation import popen_helper +from official.recommendation import data_preprocessing, movielens, popen_helper DATASET = "ml-test" NUM_USERS = 1000 diff --git a/official/recommendation/movielens.py b/official/recommendation/movielens.py index f14fc6c6a..7f63fa519 100755 --- a/official/recommendation/movielens.py +++ b/official/recommendation/movielens.py @@ -17,9 +17,7 @@ Download the dataset, and perform basic preprocessing. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os import sys @@ -31,15 +29,15 @@ import numpy as np import pandas as pd import six -from six.moves import urllib # pylint: disable=redefined-builtin -from absl import app -from absl import flags -from absl import logging import tensorflow as tf -# pylint: enable=g-bad-import-order +from absl import app, flags, logging +from six.moves import urllib # pylint: disable=redefined-builtin from official.utils.flags import core as flags_core +# pylint: enable=g-bad-import-order + + ML_1M = "ml-1m" ML_20M = "ml-20m" diff --git a/official/recommendation/ncf_common.py b/official/recommendation/ncf_common.py index bb9406330..747ab3d02 100755 --- a/official/recommendation/ncf_common.py +++ b/official/recommendation/ncf_common.py @@ -14,23 +14,19 @@ # ============================================================================== """Common functionalities used by both Keras and Estimator implementations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import json import os -from absl import flags -from absl import logging import numpy as np import tensorflow as tf +from absl import flags, logging from official.common import distribute_utils from official.recommendation import constants as rconst -from official.recommendation import data_pipeline -from official.recommendation import data_preprocessing -from official.recommendation import movielens +from official.recommendation import (data_pipeline, data_preprocessing, + movielens) from official.utils.flags import core as flags_core from official.utils.misc import keras_utils diff --git a/official/recommendation/ncf_input_pipeline.py b/official/recommendation/ncf_input_pipeline.py index 5d092f0c7..ab57aa58a 100755 --- a/official/recommendation/ncf_input_pipeline.py +++ b/official/recommendation/ncf_input_pipeline.py @@ -18,11 +18,12 @@ # pylint: disable=g-bad-import-order import tensorflow as tf -# pylint: enable=g-bad-import-order from official.recommendation import constants as rconst -from official.recommendation import data_pipeline -from official.recommendation import movielens +from official.recommendation import data_pipeline, movielens + +# pylint: enable=g-bad-import-order + def create_dataset_from_tf_record_files(input_file_pattern, diff --git a/official/recommendation/ncf_keras_main.py b/official/recommendation/ncf_keras_main.py index 808082bcd..390d8432c 100755 --- a/official/recommendation/ncf_keras_main.py +++ b/official/recommendation/ncf_keras_main.py @@ -21,23 +21,20 @@ import json import os -# pylint: disable=g-bad-import-order - -from absl import app -from absl import flags -from absl import logging import tensorflow as tf -# pylint: enable=g-bad-import-order +from absl import app, flags, logging from official.common import distribute_utils from official.recommendation import constants as rconst -from official.recommendation import movielens -from official.recommendation import ncf_common -from official.recommendation import ncf_input_pipeline -from official.recommendation import neumf_model +from official.recommendation import ( + movielens, ncf_common, ncf_input_pipeline, neumf_model) from official.utils.flags import core as flags_core -from official.utils.misc import keras_utils -from official.utils.misc import model_helpers +from official.utils.misc import keras_utils, model_helpers + +# pylint: disable=g-bad-import-order + +# pylint: enable=g-bad-import-order + FLAGS = flags.FLAGS diff --git a/official/recommendation/ncf_test.py b/official/recommendation/ncf_test.py index 90e92bd41..ed8e7b171 100755 --- a/official/recommendation/ncf_test.py +++ b/official/recommendation/ncf_test.py @@ -14,17 +14,16 @@ # ============================================================================== """Tests NCF.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import unittest import tensorflow as tf -from tensorflow.python.eager import context # pylint: disable=ungrouped-imports +from tensorflow.python.eager import \ + context # pylint: disable=ungrouped-imports + from official.recommendation import constants as rconst -from official.recommendation import ncf_common -from official.recommendation import ncf_keras_main +from official.recommendation import ncf_common, ncf_keras_main from official.utils.testing import integration NUM_TRAIN_NEG = 4 diff --git a/official/recommendation/neumf_model.py b/official/recommendation/neumf_model.py index 546c46907..1bc88401a 100755 --- a/official/recommendation/neumf_model.py +++ b/official/recommendation/neumf_model.py @@ -29,21 +29,17 @@ In NeuMF model, it allows GMF and MLP to learn separate embeddings, and combine the two models by concatenating their last hidden layer. """ -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import sys +from typing import Any, Dict, Text -from six.moves import xrange # pylint: disable=redefined-builtin import tensorflow as tf -from typing import Any, Dict, Text +from six.moves import xrange # pylint: disable=redefined-builtin from official.recommendation import constants as rconst -from official.recommendation import movielens -from official.recommendation import ncf_common -from official.recommendation import stat_utils +from official.recommendation import movielens, ncf_common, stat_utils def sparse_to_dense_grads(grads_and_vars): diff --git a/official/recommendation/stat_utils.py b/official/recommendation/stat_utils.py index 526db0fbb..b51744f49 100755 --- a/official/recommendation/stat_utils.py +++ b/official/recommendation/stat_utils.py @@ -14,9 +14,7 @@ # ============================================================================== """Statistics utility functions of NCF.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os diff --git a/official/staging/training/grad_utils.py b/official/staging/training/grad_utils.py index 004ee275d..17c57b9a3 100755 --- a/official/staging/training/grad_utils.py +++ b/official/staging/training/grad_utils.py @@ -14,9 +14,8 @@ # ============================================================================== """Some gradient util functions to help users writing custom training loop.""" -from absl import logging - import tensorflow as tf +from absl import logging def _filter_grads(grads_and_vars): diff --git a/official/utils/docs/build_docs.py b/official/utils/docs/build_docs.py index 904052b26..f1e9c710f 100755 --- a/official/utils/docs/build_docs.py +++ b/official/utils/docs/build_docs.py @@ -26,16 +26,12 @@ import os -from absl import app -from absl import flags -from absl import logging - import tensorflow as tf -from tensorflow_docs.api_generator import doc_controls -from tensorflow_docs.api_generator import generate_lib -from tensorflow_docs.api_generator import public_api +from absl import app, flags, logging from official.nlp import modeling as tfnlp +from tensorflow_docs.api_generator import (doc_controls, generate_lib, + public_api) FLAGS = flags.FLAGS diff --git a/official/utils/flags/_base.py b/official/utils/flags/_base.py index cbe7c1ec2..73959550d 100755 --- a/official/utils/flags/_base.py +++ b/official/utils/flags/_base.py @@ -14,12 +14,11 @@ # ============================================================================== """Flags which will be nearly universal across models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from absl import flags import tensorflow as tf +from absl import flags + from official.utils.flags._conventions import help_wrap diff --git a/official/utils/flags/_benchmark.py b/official/utils/flags/_benchmark.py index acaec19a8..b8a545c85 100755 --- a/official/utils/flags/_benchmark.py +++ b/official/utils/flags/_benchmark.py @@ -14,9 +14,7 @@ # ============================================================================== """Flags for benchmarking models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from absl import flags diff --git a/official/utils/flags/_conventions.py b/official/utils/flags/_conventions.py index b6f248b44..db271264a 100755 --- a/official/utils/flags/_conventions.py +++ b/official/utils/flags/_conventions.py @@ -14,13 +14,11 @@ # ============================================================================== """Central location for shared argparse convention definitions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -import sys import codecs import functools +import sys from absl import app as absl_app from absl import flags diff --git a/official/utils/flags/_device.py b/official/utils/flags/_device.py index 2ac2c859e..5ac6d6819 100755 --- a/official/utils/flags/_device.py +++ b/official/utils/flags/_device.py @@ -14,12 +14,9 @@ # ============================================================================== """Flags for managing compute devices. Currently only contains TPU flags.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from absl import flags -from absl import logging +from absl import flags, logging from official.utils.flags._conventions import help_wrap diff --git a/official/utils/flags/_distribution.py b/official/utils/flags/_distribution.py index 04edecb48..3a6510ce5 100755 --- a/official/utils/flags/_distribution.py +++ b/official/utils/flags/_distribution.py @@ -14,12 +14,10 @@ # ============================================================================== """Flags related to distributed execution.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from absl import flags import tensorflow as tf +from absl import flags from official.utils.flags._conventions import help_wrap diff --git a/official/utils/flags/_misc.py b/official/utils/flags/_misc.py index b8b88f873..a426d09af 100755 --- a/official/utils/flags/_misc.py +++ b/official/utils/flags/_misc.py @@ -14,9 +14,7 @@ # ============================================================================== """Misc flags.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from absl import flags diff --git a/official/utils/flags/_performance.py b/official/utils/flags/_performance.py index 99b0b572b..4b0dfdd19 100755 --- a/official/utils/flags/_performance.py +++ b/official/utils/flags/_performance.py @@ -14,14 +14,12 @@ # ============================================================================== """Register flags for optimizing performance.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import multiprocessing -from absl import flags # pylint: disable=g-bad-import-order import tensorflow as tf # pylint: disable=g-bad-import-order +from absl import flags # pylint: disable=g-bad-import-order from official.utils.flags._conventions import help_wrap diff --git a/official/utils/flags/core.py b/official/utils/flags/core.py index d6eebbd76..46608d15f 100755 --- a/official/utils/flags/core.py +++ b/official/utils/flags/core.py @@ -17,24 +17,16 @@ See _example.py for detailed instructions on defining flags. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import sys -from six.moves import shlex_quote - from absl import app as absl_app from absl import flags +from six.moves import shlex_quote -from official.utils.flags import _base -from official.utils.flags import _benchmark -from official.utils.flags import _conventions -from official.utils.flags import _device -from official.utils.flags import _distribution -from official.utils.flags import _misc -from official.utils.flags import _performance +from official.utils.flags import (_base, _benchmark, _conventions, _device, + _distribution, _misc, _performance) def set_defaults(**kwargs): diff --git a/official/utils/flags/flags_test.py b/official/utils/flags/flags_test.py index 64fc2f9eb..ef0766022 100755 --- a/official/utils/flags/flags_test.py +++ b/official/utils/flags/flags_test.py @@ -15,8 +15,8 @@ import unittest -from absl import flags import tensorflow as tf +from absl import flags from official.utils.flags import core as flags_core # pylint: disable=g-bad-import-order diff --git a/official/utils/hyperparams_flags.py b/official/utils/hyperparams_flags.py index 4b8150677..3c5a0d76b 100755 --- a/official/utils/hyperparams_flags.py +++ b/official/utils/hyperparams_flags.py @@ -14,12 +14,11 @@ # ============================================================================== """Common flags for importing hyperparameters.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function from absl import flags + from official.utils.flags import core as flags_core FLAGS = flags.FLAGS diff --git a/official/utils/misc/distribution_utils.py b/official/utils/misc/distribution_utils.py index 92e540c18..77b9152ce 100755 --- a/official/utils/misc/distribution_utils.py +++ b/official/utils/misc/distribution_utils.py @@ -13,5 +13,4 @@ # limitations under the License. # ============================================================================== """Helper functions for running models in a distributed setting.""" -# pylint: disable=wildcard-import from official.common.distribute_utils import * diff --git a/official/utils/misc/keras_utils.py b/official/utils/misc/keras_utils.py index 6a4227486..087ca4536 100755 --- a/official/utils/misc/keras_utils.py +++ b/official/utils/misc/keras_utils.py @@ -18,9 +18,8 @@ import os import time -from absl import logging import tensorflow as tf - +from absl import logging from tensorflow.python.eager import monitoring global_batch_size_gauge = monitoring.IntGauge( diff --git a/official/utils/misc/model_helpers.py b/official/utils/misc/model_helpers.py index ae5ab4bb3..6eaed3098 100755 --- a/official/utils/misc/model_helpers.py +++ b/official/utils/misc/model_helpers.py @@ -14,16 +14,14 @@ # ============================================================================== """Miscellaneous functions that can be called by models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numbers -from absl import logging import tensorflow as tf - +from absl import logging from tensorflow.python.util import nest + # pylint:disable=logging-format-interpolation diff --git a/official/utils/misc/model_helpers_test.py b/official/utils/misc/model_helpers_test.py index 388243301..dc2f54c0e 100755 --- a/official/utils/misc/model_helpers_test.py +++ b/official/utils/misc/model_helpers_test.py @@ -14,9 +14,7 @@ # ============================================================================== """Tests for Model Helper functions.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf # pylint: disable=g-bad-import-order diff --git a/official/utils/testing/integration.py b/official/utils/testing/integration.py index fde2a3332..b70cc4a49 100755 --- a/official/utils/testing/integration.py +++ b/official/utils/testing/integration.py @@ -14,9 +14,7 @@ # ============================================================================== """Helper code to run complete models from within python.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os import shutil diff --git a/official/utils/testing/mock_task.py b/official/utils/testing/mock_task.py index 17bf16043..0400cedf4 100755 --- a/official/utils/testing/mock_task.py +++ b/official/utils/testing/mock_task.py @@ -16,13 +16,13 @@ """Mock task for testing.""" import dataclasses + import numpy as np import tensorflow as tf from official.core import base_task from official.core import config_definitions as cfg -from official.core import exp_factory -from official.core import task_factory +from official.core import exp_factory, task_factory class MockModel(tf.keras.Model): diff --git a/official/vision/beta/__init__.py b/official/vision/beta/__init__.py index 146a44e28..f371adaf2 100755 --- a/official/vision/beta/__init__.py +++ b/official/vision/beta/__init__.py @@ -15,5 +15,4 @@ """Vision package definition.""" # Lint as: python3 # pylint: disable=unused-import -from official.vision.beta import configs -from official.vision.beta import tasks +from official.vision.beta import configs, tasks diff --git a/official/vision/beta/configs/__init__.py b/official/vision/beta/configs/__init__.py index 091beea62..b579f181a 100755 --- a/official/vision/beta/configs/__init__.py +++ b/official/vision/beta/configs/__init__.py @@ -15,8 +15,6 @@ # ============================================================================== """Configs package definition.""" -from official.vision.beta.configs import image_classification -from official.vision.beta.configs import maskrcnn -from official.vision.beta.configs import retinanet -from official.vision.beta.configs import semantic_segmentation -from official.vision.beta.configs import video_classification +from official.vision.beta.configs import (image_classification, maskrcnn, + retinanet, semantic_segmentation, + video_classification) diff --git a/official/vision/beta/configs/backbones.py b/official/vision/beta/configs/backbones.py index 93740818d..70b3a2e64 100755 --- a/official/vision/beta/configs/backbones.py +++ b/official/vision/beta/configs/backbones.py @@ -14,10 +14,9 @@ # limitations under the License. # ============================================================================== """Backbones configurations.""" -from typing import Optional, List - # Import libraries import dataclasses +from typing import List, Optional from official.modeling import hyperparams diff --git a/official/vision/beta/configs/backbones_3d.py b/official/vision/beta/configs/backbones_3d.py index 37d01d4b7..f41dd6f45 100755 --- a/official/vision/beta/configs/backbones_3d.py +++ b/official/vision/beta/configs/backbones_3d.py @@ -14,10 +14,9 @@ # limitations under the License. # ============================================================================== """3D Backbones configurations.""" -from typing import Optional, Tuple - # Import libraries import dataclasses +from typing import Optional, Tuple from official.modeling import hyperparams diff --git a/official/vision/beta/configs/decoders.py b/official/vision/beta/configs/decoders.py index ec974f48e..08fb36bf6 100755 --- a/official/vision/beta/configs/decoders.py +++ b/official/vision/beta/configs/decoders.py @@ -14,10 +14,9 @@ # limitations under the License. # ============================================================================== """Decoders configurations.""" -from typing import Optional, List - # Import libraries import dataclasses +from typing import List, Optional from official.modeling import hyperparams diff --git a/official/vision/beta/configs/image_classification.py b/official/vision/beta/configs/image_classification.py index baa456cc5..bfaab9a60 100755 --- a/official/vision/beta/configs/image_classification.py +++ b/official/vision/beta/configs/image_classification.py @@ -14,15 +14,14 @@ # limitations under the License. # ============================================================================== """Image classification configuration definition.""" +import dataclasses import os from typing import List, Optional -import dataclasses + from official.core import config_definitions as cfg from official.core import exp_factory -from official.modeling import hyperparams -from official.modeling import optimization -from official.vision.beta.configs import backbones -from official.vision.beta.configs import common +from official.modeling import hyperparams, optimization +from official.vision.beta.configs import backbones, common @dataclasses.dataclass diff --git a/official/vision/beta/configs/image_classification_test.py b/official/vision/beta/configs/image_classification_test.py index 8ad55d124..c5bc236b7 100755 --- a/official/vision/beta/configs/image_classification_test.py +++ b/official/vision/beta/configs/image_classification_test.py @@ -14,9 +14,9 @@ # limitations under the License. # ============================================================================== """Tests for image_classification.""" +import tensorflow as tf # pylint: disable=unused-import from absl.testing import parameterized -import tensorflow as tf from official.core import config_definitions as cfg from official.core import exp_factory diff --git a/official/vision/beta/configs/maskrcnn.py b/official/vision/beta/configs/maskrcnn.py index a1ec46c1f..d70a9920c 100755 --- a/official/vision/beta/configs/maskrcnn.py +++ b/official/vision/beta/configs/maskrcnn.py @@ -15,16 +15,14 @@ # ============================================================================== """Mask R-CNN configuration definition.""" +import dataclasses import os from typing import List, Optional -import dataclasses + from official.core import config_definitions as cfg from official.core import exp_factory -from official.modeling import hyperparams -from official.modeling import optimization -from official.vision.beta.configs import backbones -from official.vision.beta.configs import common -from official.vision.beta.configs import decoders +from official.modeling import hyperparams, optimization +from official.vision.beta.configs import backbones, common, decoders # pylint: disable=missing-class-docstring diff --git a/official/vision/beta/configs/retinanet.py b/official/vision/beta/configs/retinanet.py index df27b202b..e117d5ac7 100755 --- a/official/vision/beta/configs/retinanet.py +++ b/official/vision/beta/configs/retinanet.py @@ -15,16 +15,14 @@ # ============================================================================== """RetinaNet configuration definition.""" +import dataclasses import os from typing import List, Optional -import dataclasses + from official.core import config_definitions as cfg from official.core import exp_factory -from official.modeling import hyperparams -from official.modeling import optimization -from official.vision.beta.configs import backbones -from official.vision.beta.configs import common -from official.vision.beta.configs import decoders +from official.modeling import hyperparams, optimization +from official.vision.beta.configs import backbones, common, decoders # pylint: disable=missing-class-docstring diff --git a/official/vision/beta/configs/semantic_segmentation.py b/official/vision/beta/configs/semantic_segmentation.py index 7e192e14c..2ea227b6b 100755 --- a/official/vision/beta/configs/semantic_segmentation.py +++ b/official/vision/beta/configs/semantic_segmentation.py @@ -14,19 +14,16 @@ # limitations under the License. # ============================================================================== """Semantic segmentation configuration definition.""" +import dataclasses import os from typing import List, Optional, Union -import dataclasses import numpy as np from official.core import exp_factory -from official.modeling import hyperparams -from official.modeling import optimization +from official.modeling import hyperparams, optimization from official.modeling.hyperparams import config_definitions as cfg -from official.vision.beta.configs import backbones -from official.vision.beta.configs import common -from official.vision.beta.configs import decoders +from official.vision.beta.configs import backbones, common, decoders @dataclasses.dataclass diff --git a/official/vision/beta/configs/semantic_segmentation_test.py b/official/vision/beta/configs/semantic_segmentation_test.py index 0cb5a7c81..fe4f7386a 100755 --- a/official/vision/beta/configs/semantic_segmentation_test.py +++ b/official/vision/beta/configs/semantic_segmentation_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for semantic_segmentation.""" +import tensorflow as tf # pylint: disable=unused-import from absl.testing import parameterized -import tensorflow as tf from official.core import exp_factory from official.modeling.hyperparams import config_definitions as cfg diff --git a/official/vision/beta/configs/video_classification.py b/official/vision/beta/configs/video_classification.py index 06897d2a1..ad40781a8 100755 --- a/official/vision/beta/configs/video_classification.py +++ b/official/vision/beta/configs/video_classification.py @@ -14,14 +14,13 @@ # limitations under the License. # ============================================================================== """Video classification configuration definition.""" -from typing import Optional, Tuple import dataclasses +from typing import Optional, Tuple + from official.core import config_definitions as cfg from official.core import exp_factory -from official.modeling import hyperparams -from official.modeling import optimization -from official.vision.beta.configs import backbones_3d -from official.vision.beta.configs import common +from official.modeling import hyperparams, optimization +from official.vision.beta.configs import backbones_3d, common @dataclasses.dataclass diff --git a/official/vision/beta/configs/video_classification_test.py b/official/vision/beta/configs/video_classification_test.py index bb65acacf..b387fb69a 100755 --- a/official/vision/beta/configs/video_classification_test.py +++ b/official/vision/beta/configs/video_classification_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for video_classification.""" +import tensorflow as tf # pylint: disable=unused-import from absl.testing import parameterized -import tensorflow as tf from official.core import config_definitions as cfg from official.core import exp_factory diff --git a/official/vision/beta/data/create_coco_tf_record.py b/official/vision/beta/data/create_coco_tf_record.py index 64317cc72..12d3f4b83 100755 --- a/official/vision/beta/data/create_coco_tf_record.py +++ b/official/vision/beta/data/create_coco_tf_record.py @@ -30,19 +30,17 @@ import collections import json import logging +import multiprocessing as mp import os +import numpy as np +import tensorflow as tf from absl import app # pylint:disable=unused-import from absl import flags -import numpy as np - from pycocotools import mask -import tensorflow as tf -import multiprocessing as mp from official.vision.beta.data import tfrecord_lib - flags.DEFINE_boolean( 'include_masks', False, 'Whether to include instance segmentations masks ' '(PNG encoded) in the result. default: False.') diff --git a/official/vision/beta/data/tfrecord_lib.py b/official/vision/beta/data/tfrecord_lib.py index bef18eb3b..c9bdc9370 100755 --- a/official/vision/beta/data/tfrecord_lib.py +++ b/official/vision/beta/data/tfrecord_lib.py @@ -17,13 +17,12 @@ import hashlib import io import itertools +import multiprocessing as mp -from absl import logging import numpy as np -from PIL import Image import tensorflow as tf - -import multiprocessing as mp +from absl import logging +from PIL import Image def convert_to_feature(value, value_type=None): @@ -172,4 +171,3 @@ def check_and_make_dir(directory): """Creates the directory if it doesn't exist.""" if not tf.io.gfile.isdir(directory): tf.io.gfile.makedirs(directory) - diff --git a/official/vision/beta/data/tfrecord_lib_test.py b/official/vision/beta/data/tfrecord_lib_test.py index 23d4b9173..66f8b0fc8 100755 --- a/official/vision/beta/data/tfrecord_lib_test.py +++ b/official/vision/beta/data/tfrecord_lib_test.py @@ -16,13 +16,12 @@ import os +import tensorflow as tf from absl import flags from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.data import tfrecord_lib - FLAGS = flags.FLAGS diff --git a/official/vision/beta/dataloaders/classification_input.py b/official/vision/beta/dataloaders/classification_input.py index 5d9acbd6f..42c084d2c 100755 --- a/official/vision/beta/dataloaders/classification_input.py +++ b/official/vision/beta/dataloaders/classification_input.py @@ -14,13 +14,12 @@ # ============================================================================== """Classification decoder and parser.""" from typing import List, Optional + # Import libraries import tensorflow as tf -from official.vision.beta.dataloaders import decoder -from official.vision.beta.dataloaders import parser -from official.vision.beta.ops import augment -from official.vision.beta.ops import preprocess_ops +from official.vision.beta.dataloaders import decoder, parser +from official.vision.beta.ops import augment, preprocess_ops MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255) STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255) diff --git a/official/vision/beta/dataloaders/maskrcnn_input.py b/official/vision/beta/dataloaders/maskrcnn_input.py index 8b789c4ed..923e0a52e 100755 --- a/official/vision/beta/dataloaders/maskrcnn_input.py +++ b/official/vision/beta/dataloaders/maskrcnn_input.py @@ -18,11 +18,8 @@ import tensorflow as tf -from official.vision.beta.dataloaders import parser -from official.vision.beta.dataloaders import utils -from official.vision.beta.ops import anchor -from official.vision.beta.ops import box_ops -from official.vision.beta.ops import preprocess_ops +from official.vision.beta.dataloaders import parser, utils +from official.vision.beta.ops import anchor, box_ops, preprocess_ops class Parser(parser.Parser): diff --git a/official/vision/beta/dataloaders/retinanet_input.py b/official/vision/beta/dataloaders/retinanet_input.py index 235376a89..770cb546f 100755 --- a/official/vision/beta/dataloaders/retinanet_input.py +++ b/official/vision/beta/dataloaders/retinanet_input.py @@ -21,11 +21,8 @@ # Import libraries import tensorflow as tf -from official.vision.beta.dataloaders import parser -from official.vision.beta.dataloaders import utils -from official.vision.beta.ops import anchor -from official.vision.beta.ops import box_ops -from official.vision.beta.ops import preprocess_ops +from official.vision.beta.dataloaders import parser, utils +from official.vision.beta.ops import anchor, box_ops, preprocess_ops class Parser(parser.Parser): diff --git a/official/vision/beta/dataloaders/segmentation_input.py b/official/vision/beta/dataloaders/segmentation_input.py index 394c63416..ceb12260d 100755 --- a/official/vision/beta/dataloaders/segmentation_input.py +++ b/official/vision/beta/dataloaders/segmentation_input.py @@ -15,8 +15,8 @@ """Data parser and processing for segmentation datasets.""" import tensorflow as tf -from official.vision.beta.dataloaders import decoder -from official.vision.beta.dataloaders import parser + +from official.vision.beta.dataloaders import decoder, parser from official.vision.beta.ops import preprocess_ops diff --git a/official/vision/beta/dataloaders/tf_example_decoder_test.py b/official/vision/beta/dataloaders/tf_example_decoder_test.py index 91bcb6857..8c18f9201 100755 --- a/official/vision/beta/dataloaders/tf_example_decoder_test.py +++ b/official/vision/beta/dataloaders/tf_example_decoder_test.py @@ -15,15 +15,15 @@ """Tests for tf_example_decoder.py.""" import io + +import numpy as np +import tensorflow as tf # Import libraries from absl.testing import parameterized -import numpy as np from PIL import Image -import tensorflow as tf from official.vision.beta.dataloaders import tf_example_decoder - DUMP_SOURCE_ID = b'123' diff --git a/official/vision/beta/dataloaders/tf_example_label_map_decoder.py b/official/vision/beta/dataloaders/tf_example_label_map_decoder.py index e9d7fa416..4c7dc4e91 100755 --- a/official/vision/beta/dataloaders/tf_example_label_map_decoder.py +++ b/official/vision/beta/dataloaders/tf_example_label_map_decoder.py @@ -18,6 +18,7 @@ protos for object detection. """ import csv + # Import libraries import tensorflow as tf diff --git a/official/vision/beta/dataloaders/tf_example_label_map_decoder_test.py b/official/vision/beta/dataloaders/tf_example_label_map_decoder_test.py index 8dedabc28..88c9aa4da 100755 --- a/official/vision/beta/dataloaders/tf_example_label_map_decoder_test.py +++ b/official/vision/beta/dataloaders/tf_example_label_map_decoder_test.py @@ -16,15 +16,15 @@ import io import os + +import numpy as np +import tensorflow as tf # Import libraries from absl.testing import parameterized -import numpy as np from PIL import Image -import tensorflow as tf from official.vision.beta.dataloaders import tf_example_label_map_decoder - DUMP_SOURCE_ID = b'123' LABEL_MAP_CSV_CONTENT = '0,class_0\n1,class_1\n2,class_2' diff --git a/official/vision/beta/dataloaders/tfds_classification_decoders.py b/official/vision/beta/dataloaders/tfds_classification_decoders.py index 0a6f1cf5b..223c0bb02 100755 --- a/official/vision/beta/dataloaders/tfds_classification_decoders.py +++ b/official/vision/beta/dataloaders/tfds_classification_decoders.py @@ -15,6 +15,7 @@ """TFDS Classification decoders.""" import tensorflow as tf + from official.vision.beta.dataloaders import decoder diff --git a/official/vision/beta/dataloaders/tfds_detection_decoders.py b/official/vision/beta/dataloaders/tfds_detection_decoders.py index 6b8e9deab..d0c7ccc8f 100755 --- a/official/vision/beta/dataloaders/tfds_detection_decoders.py +++ b/official/vision/beta/dataloaders/tfds_detection_decoders.py @@ -15,6 +15,7 @@ """TFDS detection decoders.""" import tensorflow as tf + from official.vision.beta.dataloaders import decoder diff --git a/official/vision/beta/dataloaders/tfds_segmentation_decoders.py b/official/vision/beta/dataloaders/tfds_segmentation_decoders.py index 8ee9962d1..6c960bcdc 100755 --- a/official/vision/beta/dataloaders/tfds_segmentation_decoders.py +++ b/official/vision/beta/dataloaders/tfds_segmentation_decoders.py @@ -15,6 +15,7 @@ """TFDS Semantic Segmentation decoders.""" import tensorflow as tf + from official.vision.beta.dataloaders import decoder diff --git a/official/vision/beta/dataloaders/video_input.py b/official/vision/beta/dataloaders/video_input.py index 3045591ef..bf7abfdf8 100755 --- a/official/vision/beta/dataloaders/video_input.py +++ b/official/vision/beta/dataloaders/video_input.py @@ -17,12 +17,11 @@ from typing import Dict, Optional, Tuple, Union -from absl import logging import tensorflow as tf +from absl import logging from official.vision.beta.configs import video_classification as exp_cfg -from official.vision.beta.dataloaders import decoder -from official.vision.beta.dataloaders import parser +from official.vision.beta.dataloaders import decoder, parser from official.vision.beta.ops import preprocess_ops_3d IMAGE_KEY = 'image/encoded' diff --git a/official/vision/beta/dataloaders/video_input_test.py b/official/vision/beta/dataloaders/video_input_test.py index ddaba79a4..a19a3f965 100755 --- a/official/vision/beta/dataloaders/video_input_test.py +++ b/official/vision/beta/dataloaders/video_input_test.py @@ -18,13 +18,12 @@ # Import libraries import numpy as np -from PIL import Image import tensorflow as tf +from PIL import Image from official.vision.beta.configs import video_classification as exp_cfg from official.vision.beta.dataloaders import video_input - AUDIO_KEY = 'features/audio' diff --git a/official/vision/beta/evaluation/coco_evaluator.py b/official/vision/beta/evaluation/coco_evaluator.py index 84ec57b6f..2376be6f1 100755 --- a/official/vision/beta/evaluation/coco_evaluator.py +++ b/official/vision/beta/evaluation/coco_evaluator.py @@ -28,12 +28,13 @@ import atexit import tempfile -# Import libraries -from absl import logging + import numpy as np -from pycocotools import cocoeval import six import tensorflow as tf +# Import libraries +from absl import logging +from pycocotools import cocoeval from official.vision.beta.evaluation import coco_utils diff --git a/official/vision/beta/evaluation/coco_utils.py b/official/vision/beta/evaluation/coco_utils.py index 3bbecf0d6..c8c6ed2a4 100755 --- a/official/vision/beta/evaluation/coco_utils.py +++ b/official/vision/beta/evaluation/coco_utils.py @@ -17,18 +17,17 @@ import copy import json +import numpy as np +import six +import tensorflow as tf # Import libraries from absl import logging -import numpy as np from PIL import Image from pycocotools import coco from pycocotools import mask as mask_api -import six -import tensorflow as tf from official.vision.beta.dataloaders import tf_example_decoder -from official.vision.beta.ops import box_ops -from official.vision.beta.ops import mask_ops +from official.vision.beta.ops import box_ops, mask_ops class COCOWrapper(coco.COCO): diff --git a/official/vision/beta/losses/maskrcnn_losses.py b/official/vision/beta/losses/maskrcnn_losses.py index 3b0aeffef..cf9f25b65 100755 --- a/official/vision/beta/losses/maskrcnn_losses.py +++ b/official/vision/beta/losses/maskrcnn_losses.py @@ -299,4 +299,3 @@ def __call__(self, mask_outputs, mask_targets, select_class_targets): # The loss is normalized by the number of 1's in weights and # + 0.01 is used to avoid division by zero. return mask_loss / (tf.reduce_sum(weights) + 0.01) - diff --git a/official/vision/beta/modeling/backbones/__init__.py b/official/vision/beta/modeling/backbones/__init__.py index 49a0f595a..351996af4 100755 --- a/official/vision/beta/modeling/backbones/__init__.py +++ b/official/vision/beta/modeling/backbones/__init__.py @@ -19,6 +19,7 @@ from official.vision.beta.modeling.backbones.mobilenet import MobileNet from official.vision.beta.modeling.backbones.resnet import ResNet from official.vision.beta.modeling.backbones.resnet_3d import ResNet3D -from official.vision.beta.modeling.backbones.resnet_deeplab import DilatedResNet +from official.vision.beta.modeling.backbones.resnet_deeplab import \ + DilatedResNet from official.vision.beta.modeling.backbones.revnet import RevNet from official.vision.beta.modeling.backbones.spinenet import SpineNet diff --git a/official/vision/beta/modeling/backbones/efficientnet.py b/official/vision/beta/modeling/backbones/efficientnet.py index d39a698ce..6219af823 100755 --- a/official/vision/beta/modeling/backbones/efficientnet.py +++ b/official/vision/beta/modeling/backbones/efficientnet.py @@ -15,12 +15,13 @@ """Contains definitions of EfficientNet Networks.""" import math + # Import libraries import tensorflow as tf + from official.modeling import tf_utils from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import nn_blocks -from official.vision.beta.modeling.layers import nn_layers +from official.vision.beta.modeling.layers import nn_blocks, nn_layers layers = tf.keras.layers diff --git a/official/vision/beta/modeling/backbones/efficientnet_test.py b/official/vision/beta/modeling/backbones/efficientnet_test.py index 5869a301d..0c3a7f314 100755 --- a/official/vision/beta/modeling/backbones/efficientnet_test.py +++ b/official/vision/beta/modeling/backbones/efficientnet_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for EfficientNet.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.backbones import efficientnet diff --git a/official/vision/beta/modeling/backbones/factory.py b/official/vision/beta/modeling/backbones/factory.py index 174e6f05e..c467d7415 100755 --- a/official/vision/beta/modeling/backbones/factory.py +++ b/official/vision/beta/modeling/backbones/factory.py @@ -47,7 +47,6 @@ def build_my_backbone(): from official.core import registry - _REGISTERED_BACKBONE_CLS = {} diff --git a/official/vision/beta/modeling/backbones/factory_test.py b/official/vision/beta/modeling/backbones/factory_test.py index 5914ddeaf..8df4d13e5 100755 --- a/official/vision/beta/modeling/backbones/factory_test.py +++ b/official/vision/beta/modeling/backbones/factory_test.py @@ -14,11 +14,11 @@ # limitations under the License. # ============================================================================== """Tests for factory functions.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf - from tensorflow.python.distribute import combinations + from official.vision.beta.configs import backbones as backbones_cfg from official.vision.beta.configs import backbones_3d as backbones_3d_cfg from official.vision.beta.configs import common as common_cfg diff --git a/official/vision/beta/modeling/backbones/mobilenet.py b/official/vision/beta/modeling/backbones/mobilenet.py index 2ad8c4e22..1b7284d64 100755 --- a/official/vision/beta/modeling/backbones/mobilenet.py +++ b/official/vision/beta/modeling/backbones/mobilenet.py @@ -14,16 +14,15 @@ # ============================================================================== """Contains definitions of Mobilenet Networks.""" -from typing import Text, Optional, Dict, Any, Tuple - # Import libraries import dataclasses +from typing import Any, Dict, Optional, Text, Tuple + import tensorflow as tf -from official.modeling import hyperparams -from official.modeling import tf_utils + +from official.modeling import hyperparams, tf_utils from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import nn_blocks -from official.vision.beta.modeling.layers import nn_layers +from official.vision.beta.modeling.layers import nn_blocks, nn_layers layers = tf.keras.layers regularizers = tf.keras.regularizers diff --git a/official/vision/beta/modeling/backbones/mobilenet_test.py b/official/vision/beta/modeling/backbones/mobilenet_test.py index 79296f335..469c70430 100755 --- a/official/vision/beta/modeling/backbones/mobilenet_test.py +++ b/official/vision/beta/modeling/backbones/mobilenet_test.py @@ -16,13 +16,16 @@ """Tests for MobileNet.""" import itertools -# Import libraries -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized from official.vision.beta.modeling.backbones import mobilenet +# Import libraries + + + class MobileNetTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/modeling/backbones/resnet.py b/official/vision/beta/modeling/backbones/resnet.py index 48eb6f973..1c4d4f193 100755 --- a/official/vision/beta/modeling/backbones/resnet.py +++ b/official/vision/beta/modeling/backbones/resnet.py @@ -21,10 +21,10 @@ # Import libraries import tensorflow as tf + from official.modeling import tf_utils from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import nn_blocks -from official.vision.beta.modeling.layers import nn_layers +from official.vision.beta.modeling.layers import nn_blocks, nn_layers layers = tf.keras.layers diff --git a/official/vision/beta/modeling/backbones/resnet_3d.py b/official/vision/beta/modeling/backbones/resnet_3d.py index 77db96bbc..de5139dfb 100755 --- a/official/vision/beta/modeling/backbones/resnet_3d.py +++ b/official/vision/beta/modeling/backbones/resnet_3d.py @@ -17,6 +17,7 @@ # Import libraries import tensorflow as tf + from official.modeling import tf_utils from official.vision.beta.modeling.backbones import factory from official.vision.beta.modeling.layers import nn_blocks_3d diff --git a/official/vision/beta/modeling/backbones/resnet_3d_test.py b/official/vision/beta/modeling/backbones/resnet_3d_test.py index d7adf94a0..974be4854 100755 --- a/official/vision/beta/modeling/backbones/resnet_3d_test.py +++ b/official/vision/beta/modeling/backbones/resnet_3d_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for resnet.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.backbones import resnet_3d diff --git a/official/vision/beta/modeling/backbones/resnet_deeplab.py b/official/vision/beta/modeling/backbones/resnet_deeplab.py index 2d7b64c16..436c381df 100755 --- a/official/vision/beta/modeling/backbones/resnet_deeplab.py +++ b/official/vision/beta/modeling/backbones/resnet_deeplab.py @@ -16,10 +16,10 @@ import numpy as np import tensorflow as tf + from official.modeling import tf_utils from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import nn_blocks -from official.vision.beta.modeling.layers import nn_layers +from official.vision.beta.modeling.layers import nn_blocks, nn_layers layers = tf.keras.layers diff --git a/official/vision/beta/modeling/backbones/resnet_deeplab_test.py b/official/vision/beta/modeling/backbones/resnet_deeplab_test.py index de9a23e83..9abb2b73c 100755 --- a/official/vision/beta/modeling/backbones/resnet_deeplab_test.py +++ b/official/vision/beta/modeling/backbones/resnet_deeplab_test.py @@ -15,13 +15,12 @@ # ============================================================================== """Tests for resnet_deeplab models.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.vision.beta.modeling.backbones import resnet_deeplab diff --git a/official/vision/beta/modeling/backbones/resnet_test.py b/official/vision/beta/modeling/backbones/resnet_test.py index e4635bd68..3eab4d8d4 100755 --- a/official/vision/beta/modeling/backbones/resnet_test.py +++ b/official/vision/beta/modeling/backbones/resnet_test.py @@ -15,13 +15,12 @@ # ============================================================================== """Tests for resnet.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.vision.beta.modeling.backbones import resnet diff --git a/official/vision/beta/modeling/backbones/revnet.py b/official/vision/beta/modeling/backbones/revnet.py index 276639910..f4b5c0ef7 100755 --- a/official/vision/beta/modeling/backbones/revnet.py +++ b/official/vision/beta/modeling/backbones/revnet.py @@ -21,13 +21,14 @@ """ from typing import Any, Callable, Dict, Optional + # Import libraries import tensorflow as tf + from official.modeling import tf_utils from official.vision.beta.modeling.backbones import factory from official.vision.beta.modeling.layers import nn_blocks - # Specifications for different RevNet variants. # Each entry specifies block configurations of the particular RevNet variant. # Each element in the block configuration is in the following format: diff --git a/official/vision/beta/modeling/backbones/revnet_test.py b/official/vision/beta/modeling/backbones/revnet_test.py index 1e9d4525c..4a957b8ed 100755 --- a/official/vision/beta/modeling/backbones/revnet_test.py +++ b/official/vision/beta/modeling/backbones/revnet_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for RevNet.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.backbones import revnet diff --git a/official/vision/beta/modeling/backbones/spinenet.py b/official/vision/beta/modeling/backbones/spinenet.py index 85f2eb6b9..2f1d88737 100755 --- a/official/vision/beta/modeling/backbones/spinenet.py +++ b/official/vision/beta/modeling/backbones/spinenet.py @@ -21,13 +21,13 @@ """ import math +import tensorflow as tf # Import libraries from absl import logging -import tensorflow as tf + from official.modeling import tf_utils from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import nn_blocks -from official.vision.beta.modeling.layers import nn_layers +from official.vision.beta.modeling.layers import nn_blocks, nn_layers from official.vision.beta.ops import spatial_transform_ops layers = tf.keras.layers diff --git a/official/vision/beta/modeling/backbones/spinenet_test.py b/official/vision/beta/modeling/backbones/spinenet_test.py index b062095d0..cac31a50c 100755 --- a/official/vision/beta/modeling/backbones/spinenet_test.py +++ b/official/vision/beta/modeling/backbones/spinenet_test.py @@ -14,9 +14,9 @@ # limitations under the License. # ============================================================================== """Tests for SpineNet.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.backbones import spinenet diff --git a/official/vision/beta/modeling/classification_model_test.py b/official/vision/beta/modeling/classification_model_test.py index ea3d1bcc3..be3fc0152 100755 --- a/official/vision/beta/modeling/classification_model_test.py +++ b/official/vision/beta/modeling/classification_model_test.py @@ -15,15 +15,13 @@ # ============================================================================== """Tests for classification network.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations -from official.vision.beta.modeling import backbones -from official.vision.beta.modeling import classification_model +from official.vision.beta.modeling import backbones, classification_model class ClassificationNetworkTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/modeling/decoders/aspp_test.py b/official/vision/beta/modeling/decoders/aspp_test.py index 145df33be..9c5800c26 100755 --- a/official/vision/beta/modeling/decoders/aspp_test.py +++ b/official/vision/beta/modeling/decoders/aspp_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for aspp.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.backbones import resnet from official.vision.beta.modeling.decoders import aspp diff --git a/official/vision/beta/modeling/decoders/fpn_test.py b/official/vision/beta/modeling/decoders/fpn_test.py index c4e09e7a8..1f494c01b 100755 --- a/official/vision/beta/modeling/decoders/fpn_test.py +++ b/official/vision/beta/modeling/decoders/fpn_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for FPN.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.backbones import resnet from official.vision.beta.modeling.decoders import fpn diff --git a/official/vision/beta/modeling/decoders/nasfpn.py b/official/vision/beta/modeling/decoders/nasfpn.py index 735257741..67224b288 100755 --- a/official/vision/beta/modeling/decoders/nasfpn.py +++ b/official/vision/beta/modeling/decoders/nasfpn.py @@ -19,13 +19,12 @@ https://arxiv.org/abs/1904.07392. CVPR 2019. """ +import tensorflow as tf # Import libraries from absl import logging -import tensorflow as tf from official.vision.beta.ops import spatial_transform_ops - # The fixed NAS-FPN architecture discovered by NAS. # Each element represents a specification of a building block: # (block_level, combine_fn, (input_offset0, input_offset1), is_output). diff --git a/official/vision/beta/modeling/decoders/nasfpn_test.py b/official/vision/beta/modeling/decoders/nasfpn_test.py index 4cebc2b55..ef4025af6 100755 --- a/official/vision/beta/modeling/decoders/nasfpn_test.py +++ b/official/vision/beta/modeling/decoders/nasfpn_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for NAS-FPN.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.backbones import resnet from official.vision.beta.modeling.decoders import nasfpn diff --git a/official/vision/beta/modeling/factory.py b/official/vision/beta/modeling/factory.py index 6877dcab8..40ef93f7a 100755 --- a/official/vision/beta/modeling/factory.py +++ b/official/vision/beta/modeling/factory.py @@ -18,24 +18,22 @@ import tensorflow as tf -from official.vision.beta.configs import image_classification as classification_cfg +from official.vision.beta.configs import \ + image_classification as classification_cfg from official.vision.beta.configs import maskrcnn as maskrcnn_cfg from official.vision.beta.configs import retinanet as retinanet_cfg -from official.vision.beta.configs import semantic_segmentation as segmentation_cfg -from official.vision.beta.modeling import backbones -from official.vision.beta.modeling import classification_model -from official.vision.beta.modeling import maskrcnn_model -from official.vision.beta.modeling import retinanet_model -from official.vision.beta.modeling import segmentation_model +from official.vision.beta.configs import \ + semantic_segmentation as segmentation_cfg +from official.vision.beta.modeling import (backbones, classification_model, + maskrcnn_model, retinanet_model, + segmentation_model) from official.vision.beta.modeling.decoders import factory as decoder_factory -from official.vision.beta.modeling.heads import dense_prediction_heads -from official.vision.beta.modeling.heads import instance_heads -from official.vision.beta.modeling.heads import segmentation_heads -from official.vision.beta.modeling.layers import detection_generator -from official.vision.beta.modeling.layers import mask_sampler -from official.vision.beta.modeling.layers import roi_aligner -from official.vision.beta.modeling.layers import roi_generator -from official.vision.beta.modeling.layers import roi_sampler +from official.vision.beta.modeling.heads import (dense_prediction_heads, + instance_heads, + segmentation_heads) +from official.vision.beta.modeling.layers import (detection_generator, + mask_sampler, roi_aligner, + roi_generator, roi_sampler) def build_classification_model( diff --git a/official/vision/beta/modeling/factory_3d.py b/official/vision/beta/modeling/factory_3d.py index fd5253803..527362834 100755 --- a/official/vision/beta/modeling/factory_3d.py +++ b/official/vision/beta/modeling/factory_3d.py @@ -18,9 +18,9 @@ import tensorflow as tf from official.core import registry -from official.vision.beta.configs import video_classification as video_classification_cfg -from official.vision.beta.modeling import video_classification_model -from official.vision.beta.modeling import backbones +from official.vision.beta.configs import \ + video_classification as video_classification_cfg +from official.vision.beta.modeling import backbones, video_classification_model _REGISTERED_MODEL_CLS = {} diff --git a/official/vision/beta/modeling/factory_test.py b/official/vision/beta/modeling/factory_test.py index 7aa240fd3..7047e0025 100755 --- a/official/vision/beta/modeling/factory_test.py +++ b/official/vision/beta/modeling/factory_test.py @@ -15,18 +15,18 @@ # ============================================================================== """Tests for factory.py.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf -from official.vision.beta.configs import backbones -from official.vision.beta.configs import backbones_3d -from official.vision.beta.configs import image_classification as classification_cfg +from official.vision.beta.configs import backbones, backbones_3d +from official.vision.beta.configs import \ + image_classification as classification_cfg from official.vision.beta.configs import maskrcnn as maskrcnn_cfg from official.vision.beta.configs import retinanet as retinanet_cfg -from official.vision.beta.configs import video_classification as video_classification_cfg -from official.vision.beta.modeling import factory -from official.vision.beta.modeling import factory_3d +from official.vision.beta.configs import \ + video_classification as video_classification_cfg +from official.vision.beta.modeling import factory, factory_3d class ClassificationModelBuilderTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/modeling/heads/dense_prediction_heads_test.py b/official/vision/beta/modeling/heads/dense_prediction_heads_test.py index b701dbe6c..c474e3eaa 100755 --- a/official/vision/beta/modeling/heads/dense_prediction_heads_test.py +++ b/official/vision/beta/modeling/heads/dense_prediction_heads_test.py @@ -15,10 +15,10 @@ # ============================================================================== """Tests for dense_prediction_heads.py.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized from official.vision.beta.modeling.heads import dense_prediction_heads diff --git a/official/vision/beta/modeling/heads/instance_heads_test.py b/official/vision/beta/modeling/heads/instance_heads_test.py index 6939875d4..71d3fb2b4 100755 --- a/official/vision/beta/modeling/heads/instance_heads_test.py +++ b/official/vision/beta/modeling/heads/instance_heads_test.py @@ -15,10 +15,10 @@ # ============================================================================== """Tests for instance_heads.py.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized from official.vision.beta.modeling.heads import instance_heads diff --git a/official/vision/beta/modeling/heads/segmentation_heads_test.py b/official/vision/beta/modeling/heads/segmentation_heads_test.py index 18894bfde..b27bd183c 100755 --- a/official/vision/beta/modeling/heads/segmentation_heads_test.py +++ b/official/vision/beta/modeling/heads/segmentation_heads_test.py @@ -15,10 +15,10 @@ # ============================================================================== """Tests for segmentation_heads.py.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized from official.vision.beta.modeling.heads import segmentation_heads diff --git a/official/vision/beta/modeling/layers/detection_generator.py b/official/vision/beta/modeling/layers/detection_generator.py index 4e0eff2a1..919ee384d 100755 --- a/official/vision/beta/modeling/layers/detection_generator.py +++ b/official/vision/beta/modeling/layers/detection_generator.py @@ -18,8 +18,7 @@ import tensorflow as tf -from official.vision.beta.ops import box_ops -from official.vision.beta.ops import nms +from official.vision.beta.ops import box_ops, nms def _generate_detections_v1(boxes, diff --git a/official/vision/beta/modeling/layers/detection_generator_test.py b/official/vision/beta/modeling/layers/detection_generator_test.py index ccf1058b9..c2d7204fe 100755 --- a/official/vision/beta/modeling/layers/detection_generator_test.py +++ b/official/vision/beta/modeling/layers/detection_generator_test.py @@ -15,9 +15,9 @@ """Tests for detection_generator.py.""" # Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.vision.beta.modeling.layers import detection_generator from official.vision.beta.ops import anchor diff --git a/official/vision/beta/modeling/layers/nn_blocks.py b/official/vision/beta/modeling/layers/nn_blocks.py index 673cde8d4..9ad55699f 100755 --- a/official/vision/beta/modeling/layers/nn_blocks.py +++ b/official/vision/beta/modeling/layers/nn_blocks.py @@ -14,11 +14,11 @@ # ============================================================================== """Contains common building blocks for neural networks.""" -from typing import Any, Callable, Dict, List, Optional, Tuple, Union, Text +from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Union +import tensorflow as tf # Import libraries from absl import logging -import tensorflow as tf from official.modeling import tf_utils from official.vision.beta.modeling.layers import nn_layers diff --git a/official/vision/beta/modeling/layers/nn_blocks_3d_test.py b/official/vision/beta/modeling/layers/nn_blocks_3d_test.py index 80a5caa37..edfd0c27e 100755 --- a/official/vision/beta/modeling/layers/nn_blocks_3d_test.py +++ b/official/vision/beta/modeling/layers/nn_blocks_3d_test.py @@ -15,9 +15,9 @@ # ============================================================================== """Tests for resnet.""" +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf from official.vision.beta.modeling.layers import nn_blocks_3d diff --git a/official/vision/beta/modeling/layers/nn_blocks_test.py b/official/vision/beta/modeling/layers/nn_blocks_test.py index 15228e29b..697acc9c2 100755 --- a/official/vision/beta/modeling/layers/nn_blocks_test.py +++ b/official/vision/beta/modeling/layers/nn_blocks_test.py @@ -16,12 +16,12 @@ """Tests for nn_blocks.""" from typing import Any, Iterable, Tuple + +import tensorflow as tf # Import libraries from absl.testing import parameterized -import tensorflow as tf +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.vision.beta.modeling.layers import nn_blocks diff --git a/official/vision/beta/modeling/layers/nn_layers.py b/official/vision/beta/modeling/layers/nn_layers.py index 4f8f65974..548c6690d 100755 --- a/official/vision/beta/modeling/layers/nn_layers.py +++ b/official/vision/beta/modeling/layers/nn_layers.py @@ -16,13 +16,15 @@ from typing import Optional -# Import libraries - -from absl import logging import tensorflow as tf +from absl import logging from official.modeling import tf_utils +# Import libraries + + + def make_divisible(value: float, divisor: int, diff --git a/official/vision/beta/modeling/layers/roi_generator.py b/official/vision/beta/modeling/layers/roi_generator.py index e122f3880..6f7518de1 100755 --- a/official/vision/beta/modeling/layers/roi_generator.py +++ b/official/vision/beta/modeling/layers/roi_generator.py @@ -17,8 +17,7 @@ # Import libraries import tensorflow as tf -from official.vision.beta.ops import box_ops -from official.vision.beta.ops import nms +from official.vision.beta.ops import box_ops, nms def _multilevel_propose_rois(raw_boxes, diff --git a/official/vision/beta/modeling/maskrcnn_model_test.py b/official/vision/beta/modeling/maskrcnn_model_test.py index 9723941a5..e08c31f16 100755 --- a/official/vision/beta/modeling/maskrcnn_model_test.py +++ b/official/vision/beta/modeling/maskrcnn_model_test.py @@ -15,21 +15,19 @@ # ============================================================================== """Tests for maskrcnn_model.py.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized from official.vision.beta.modeling import maskrcnn_model from official.vision.beta.modeling.backbones import resnet from official.vision.beta.modeling.decoders import fpn -from official.vision.beta.modeling.heads import dense_prediction_heads -from official.vision.beta.modeling.heads import instance_heads -from official.vision.beta.modeling.layers import detection_generator -from official.vision.beta.modeling.layers import mask_sampler -from official.vision.beta.modeling.layers import roi_aligner -from official.vision.beta.modeling.layers import roi_generator -from official.vision.beta.modeling.layers import roi_sampler +from official.vision.beta.modeling.heads import (dense_prediction_heads, + instance_heads) +from official.vision.beta.modeling.layers import (detection_generator, + mask_sampler, roi_aligner, + roi_generator, roi_sampler) from official.vision.beta.ops import anchor diff --git a/official/vision/beta/modeling/retinanet_model_test.py b/official/vision/beta/modeling/retinanet_model_test.py index 28c8555c3..645a1620b 100755 --- a/official/vision/beta/modeling/retinanet_model_test.py +++ b/official/vision/beta/modeling/retinanet_model_test.py @@ -15,13 +15,12 @@ # ============================================================================== """Tests for RetinaNet models.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.vision.beta.modeling import retinanet_model from official.vision.beta.modeling.backbones import resnet from official.vision.beta.modeling.decoders import fpn @@ -220,4 +219,3 @@ def test_serialize_deserialize(self): if __name__ == '__main__': tf.test.main() - diff --git a/official/vision/beta/modeling/segmentation_model_test.py b/official/vision/beta/modeling/segmentation_model_test.py index 41eb41dfb..d5406cc6a 100755 --- a/official/vision/beta/modeling/segmentation_model_test.py +++ b/official/vision/beta/modeling/segmentation_model_test.py @@ -15,12 +15,11 @@ # ============================================================================== """Tests for segmentation network.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized -from official.vision.beta.modeling import backbones -from official.vision.beta.modeling import segmentation_model +from official.vision.beta.modeling import backbones, segmentation_model from official.vision.beta.modeling.decoders import fpn from official.vision.beta.modeling.heads import segmentation_heads diff --git a/official/vision/beta/modeling/video_classification_model.py b/official/vision/beta/modeling/video_classification_model.py index 16935e681..6bfb0f4ab 100755 --- a/official/vision/beta/modeling/video_classification_model.py +++ b/official/vision/beta/modeling/video_classification_model.py @@ -14,6 +14,7 @@ # ============================================================================== """Build video classification models.""" from typing import Mapping + import tensorflow as tf layers = tf.keras.layers diff --git a/official/vision/beta/modeling/video_classification_model_test.py b/official/vision/beta/modeling/video_classification_model_test.py index 92e75db13..510092fe7 100755 --- a/official/vision/beta/modeling/video_classification_model_test.py +++ b/official/vision/beta/modeling/video_classification_model_test.py @@ -15,13 +15,12 @@ # ============================================================================== """Tests for video classification network.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized -from official.vision.beta.modeling import backbones -from official.vision.beta.modeling import video_classification_model +from official.vision.beta.modeling import backbones, video_classification_model class VideoClassificationNetworkTest(parameterized.TestCase, tf.test.TestCase): diff --git a/official/vision/beta/ops/anchor.py b/official/vision/beta/ops/anchor.py index a69f9ac94..88234355f 100755 --- a/official/vision/beta/ops/anchor.py +++ b/official/vision/beta/ops/anchor.py @@ -15,12 +15,13 @@ """Anchor box and labeler definition.""" import collections + # Import libraries import tensorflow as tf + from official.vision import keras_cv -from official.vision.detection.utils.object_detection import balanced_positive_negative_sampler -from official.vision.detection.utils.object_detection import box_list -from official.vision.detection.utils.object_detection import faster_rcnn_box_coder +from official.vision.detection.utils.object_detection import ( + balanced_positive_negative_sampler, box_list, faster_rcnn_box_coder) class Anchor(object): diff --git a/official/vision/beta/ops/anchor_test.py b/official/vision/beta/ops/anchor_test.py index 7c60dde21..0dfdce55d 100755 --- a/official/vision/beta/ops/anchor_test.py +++ b/official/vision/beta/ops/anchor_test.py @@ -14,10 +14,11 @@ # ============================================================================== """Tests for anchor.py.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized + from official.vision.beta.ops import anchor diff --git a/official/vision/beta/ops/augment.py b/official/vision/beta/ops/augment.py index 0c3d976d0..24bce4e4e 100755 --- a/official/vision/beta/ops/augment.py +++ b/official/vision/beta/ops/augment.py @@ -18,13 +18,12 @@ RandAugment Reference: https://arxiv.org/abs/1909.13719 """ import math -from typing import Any, List, Optional, Text, Tuple, Iterable +from typing import Any, Iterable, List, Optional, Text, Tuple import numpy as np import tensorflow as tf - -from tensorflow.python.keras.layers.preprocessing import image_preprocessing as image_ops - +from tensorflow.python.keras.layers.preprocessing import \ + image_preprocessing as image_ops # This signifies the max integer that the controller RNN could predict for the # augmentation scheme. diff --git a/official/vision/beta/ops/augment_test.py b/official/vision/beta/ops/augment_test.py index 670845ae2..01c9b5ac6 100755 --- a/official/vision/beta/ops/augment_test.py +++ b/official/vision/beta/ops/augment_test.py @@ -14,15 +14,13 @@ # ============================================================================== """Tests for autoaugment.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import random -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized from official.vision.beta.ops import augment diff --git a/official/vision/beta/ops/box_ops.py b/official/vision/beta/ops/box_ops.py index 4fe7600cb..28b0a7c5c 100755 --- a/official/vision/beta/ops/box_ops.py +++ b/official/vision/beta/ops/box_ops.py @@ -18,7 +18,6 @@ import numpy as np import tensorflow as tf - EPSILON = 1e-8 BBOX_XFORM_CLIP = np.log(1000. / 16.) diff --git a/official/vision/beta/ops/mask_ops.py b/official/vision/beta/ops/mask_ops.py index bc25b1514..3f95c5d0c 100755 --- a/official/vision/beta/ops/mask_ops.py +++ b/official/vision/beta/ops/mask_ops.py @@ -15,6 +15,7 @@ """Utility functions for segmentations.""" import math + # Import libraries import cv2 import numpy as np @@ -187,4 +188,3 @@ def paste_instance_masks_v2(masks, segms = np.array(segms) return segms - diff --git a/official/vision/beta/ops/mask_ops_test.py b/official/vision/beta/ops/mask_ops_test.py index d07335036..5f6332b89 100755 --- a/official/vision/beta/ops/mask_ops_test.py +++ b/official/vision/beta/ops/mask_ops_test.py @@ -18,6 +18,7 @@ # Import libraries import numpy as np import tensorflow as tf + from official.vision.beta.ops import mask_ops diff --git a/official/vision/beta/ops/nms.py b/official/vision/beta/ops/nms.py index 645e2a4bc..ad01969db 100755 --- a/official/vision/beta/ops/nms.py +++ b/official/vision/beta/ops/nms.py @@ -19,7 +19,6 @@ from official.vision.beta.ops import box_ops - NMS_TILE_SIZE = 512 diff --git a/official/vision/beta/ops/preprocess_ops.py b/official/vision/beta/ops/preprocess_ops.py index 0d47ce601..f7aff0b77 100755 --- a/official/vision/beta/ops/preprocess_ops.py +++ b/official/vision/beta/ops/preprocess_ops.py @@ -15,12 +15,12 @@ """Preprocessing ops.""" import math -from six.moves import range + import tensorflow as tf +from six.moves import range from official.vision.beta.ops import box_ops - CENTER_CROP_FRACTION = 0.875 diff --git a/official/vision/beta/ops/preprocess_ops_3d.py b/official/vision/beta/ops/preprocess_ops_3d.py index fabe1f711..411d5ad8a 100755 --- a/official/vision/beta/ops/preprocess_ops_3d.py +++ b/official/vision/beta/ops/preprocess_ops_3d.py @@ -16,6 +16,7 @@ """Utils for processing video dataset features.""" from typing import Optional, Tuple + import tensorflow as tf diff --git a/official/vision/beta/ops/preprocess_ops_3d_test.py b/official/vision/beta/ops/preprocess_ops_3d_test.py index 9f5f0359e..5c27f0886 100755 --- a/official/vision/beta/ops/preprocess_ops_3d_test.py +++ b/official/vision/beta/ops/preprocess_ops_3d_test.py @@ -16,9 +16,10 @@ import io import itertools + import numpy as np -from PIL import Image import tensorflow as tf +from PIL import Image from official.vision.beta.ops import preprocess_ops_3d diff --git a/official/vision/beta/ops/preprocess_ops_test.py b/official/vision/beta/ops/preprocess_ops_test.py index 280f0f387..ed708927a 100755 --- a/official/vision/beta/ops/preprocess_ops_test.py +++ b/official/vision/beta/ops/preprocess_ops_test.py @@ -16,11 +16,12 @@ """Tests for preprocess_ops.py.""" import io + +import numpy as np +import tensorflow as tf # Import libraries from absl.testing import parameterized -import numpy as np from PIL import Image -import tensorflow as tf from official.vision.beta.ops import preprocess_ops diff --git a/official/vision/beta/projects/yolo/configs/backbones.py b/official/vision/beta/projects/yolo/configs/backbones.py index 31444687c..4693432ad 100755 --- a/official/vision/beta/projects/yolo/configs/backbones.py +++ b/official/vision/beta/projects/yolo/configs/backbones.py @@ -19,7 +19,6 @@ import dataclasses from official.modeling import hyperparams - from official.vision.beta.configs import backbones diff --git a/official/vision/beta/projects/yolo/configs/darknet_classification.py b/official/vision/beta/projects/yolo/configs/darknet_classification.py index f3061df5c..b709adaa9 100755 --- a/official/vision/beta/projects/yolo/configs/darknet_classification.py +++ b/official/vision/beta/projects/yolo/configs/darknet_classification.py @@ -15,9 +15,8 @@ # ============================================================================== """Image classification with darknet configs.""" -from typing import List, Optional - import dataclasses +from typing import List, Optional from official.core import config_definitions as cfg from official.core import exp_factory diff --git a/official/vision/beta/projects/yolo/dataloaders/classification_tfds_decoder.py b/official/vision/beta/projects/yolo/dataloaders/classification_tfds_decoder.py index e0549aac9..123ca70a2 100755 --- a/official/vision/beta/projects/yolo/dataloaders/classification_tfds_decoder.py +++ b/official/vision/beta/projects/yolo/dataloaders/classification_tfds_decoder.py @@ -32,5 +32,3 @@ def decode(self, serialized_example): 'image/class/label': serialized_example['label'], } return sample_dict - - diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py index ad63aeaca..3ca806ae7 100755 --- a/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py +++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py @@ -15,12 +15,11 @@ # ============================================================================== """Tests for resnet.""" -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.vision.beta.projects.yolo.modeling.backbones import darknet diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py index 3cdf16086..373b8392b 100755 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py @@ -17,7 +17,9 @@ """Contains common building blocks for yolo neural networks.""" from typing import Callable, List + import tensorflow as tf + from official.modeling import tf_utils diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py index 3d5ec2b97..99bb30ba6 100755 --- a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py +++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py @@ -14,9 +14,9 @@ # limitations under the License. # ============================================================================== -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized from official.vision.beta.projects.yolo.modeling.layers import nn_blocks diff --git a/official/vision/beta/projects/yolo/tasks/image_classification.py b/official/vision/beta/projects/yolo/tasks/image_classification.py index 75abc2959..cba0afdb3 100755 --- a/official/vision/beta/projects/yolo/tasks/image_classification.py +++ b/official/vision/beta/projects/yolo/tasks/image_classification.py @@ -16,11 +16,12 @@ """Image classification task definition.""" import tensorflow as tf -from official.core import input_reader -from official.core import task_factory +from official.core import input_reader, task_factory from official.vision.beta.dataloaders import classification_input -from official.vision.beta.projects.yolo.configs import darknet_classification as exp_cfg -from official.vision.beta.projects.yolo.dataloaders import classification_tfds_decoder as cli +from official.vision.beta.projects.yolo.configs import \ + darknet_classification as exp_cfg +from official.vision.beta.projects.yolo.dataloaders import \ + classification_tfds_decoder as cli from official.vision.beta.tasks import image_classification @@ -112,5 +113,3 @@ def train_step(self, inputs, model, optimizer, metrics=None): self.process_compiled_metrics(model.compiled_metrics, labels, outputs) logs.update({m.name: m.result() for m in model.metrics}) return logs - - diff --git a/official/vision/beta/projects/yolo/train.py b/official/vision/beta/projects/yolo/train.py index ba59837fb..33eab8e0e 100755 --- a/official/vision/beta/projects/yolo/train.py +++ b/official/vision/beta/projects/yolo/train.py @@ -15,17 +15,15 @@ # ============================================================================== """TensorFlow Model Garden Vision training driver.""" -from absl import app -from absl import flags import gin +from absl import app, flags from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib -from official.core import train_utils +from official.core import task_factory, train_lib, train_utils from official.modeling import performance -from official.vision.beta.projects.yolo.common import registry_imports # pylint: disable=unused-import +from official.vision.beta.projects.yolo.common import \ + registry_imports # pylint: disable=unused-import FLAGS = flags.FLAGS diff --git a/official/vision/beta/serving/detection.py b/official/vision/beta/serving/detection.py index 7612ae775..afd29d770 100755 --- a/official/vision/beta/serving/detection.py +++ b/official/vision/beta/serving/detection.py @@ -19,11 +19,9 @@ from official.vision.beta import configs from official.vision.beta.modeling import factory -from official.vision.beta.ops import anchor -from official.vision.beta.ops import preprocess_ops +from official.vision.beta.ops import anchor, preprocess_ops from official.vision.beta.serving import export_base - MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255) STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255) diff --git a/official/vision/beta/serving/detection_test.py b/official/vision/beta/serving/detection_test.py index a970a6ab2..787c7f9c4 100755 --- a/official/vision/beta/serving/detection_test.py +++ b/official/vision/beta/serving/detection_test.py @@ -18,10 +18,10 @@ import io import os -from absl.testing import parameterized import numpy as np -from PIL import Image import tensorflow as tf +from absl.testing import parameterized +from PIL import Image from official.common import registry_imports # pylint: disable=unused-import from official.core import exp_factory diff --git a/official/vision/beta/serving/export_base.py b/official/vision/beta/serving/export_base.py index 85ce938be..edad0b617 100755 --- a/official/vision/beta/serving/export_base.py +++ b/official/vision/beta/serving/export_base.py @@ -16,6 +16,7 @@ """Base class for model export.""" import abc + import tensorflow as tf diff --git a/official/vision/beta/serving/export_saved_model.py b/official/vision/beta/serving/export_saved_model.py index 7f9dadaee..ee461bd2f 100755 --- a/official/vision/beta/serving/export_saved_model.py +++ b/official/vision/beta/serving/export_saved_model.py @@ -36,8 +36,7 @@ output = model_fn(input_images) """ -from absl import app -from absl import flags +from absl import app, flags from official.common import registry_imports # pylint: disable=unused-import from official.core import exp_factory diff --git a/official/vision/beta/serving/export_saved_model_lib.py b/official/vision/beta/serving/export_saved_model_lib.py index a3e3f553d..c0103aa4e 100755 --- a/official/vision/beta/serving/export_saved_model_lib.py +++ b/official/vision/beta/serving/export_saved_model_lib.py @@ -21,9 +21,8 @@ from official.core import train_utils from official.vision.beta import configs -from official.vision.beta.serving import detection -from official.vision.beta.serving import image_classification -from official.vision.beta.serving import semantic_segmentation +from official.vision.beta.serving import (detection, image_classification, + semantic_segmentation) def export_inference_graph(input_type, batch_size, input_image_size, params, diff --git a/official/vision/beta/serving/export_tfhub.py b/official/vision/beta/serving/export_tfhub.py index 241e02a05..f1f15e183 100755 --- a/official/vision/beta/serving/export_tfhub.py +++ b/official/vision/beta/serving/export_tfhub.py @@ -15,18 +15,15 @@ # ============================================================================== """A script to export the image classification as a TF-Hub SavedModel.""" -# Import libraries -from absl import app -from absl import flags - import tensorflow as tf +# Import libraries +from absl import app, flags from official.common import registry_imports # pylint: disable=unused-import from official.core import exp_factory from official.modeling import hyperparams from official.vision.beta.serving import image_classification - FLAGS = flags.FLAGS flags.DEFINE_string( diff --git a/official/vision/beta/serving/image_classification.py b/official/vision/beta/serving/image_classification.py index d7c6f4ad1..314dfa193 100755 --- a/official/vision/beta/serving/image_classification.py +++ b/official/vision/beta/serving/image_classification.py @@ -21,7 +21,6 @@ from official.vision.beta.ops import preprocess_ops from official.vision.beta.serving import export_base - MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255) STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255) diff --git a/official/vision/beta/serving/image_classification_test.py b/official/vision/beta/serving/image_classification_test.py index d9b7be7a5..79ed6d69a 100755 --- a/official/vision/beta/serving/image_classification_test.py +++ b/official/vision/beta/serving/image_classification_test.py @@ -18,10 +18,10 @@ import io import os -from absl.testing import parameterized import numpy as np -from PIL import Image import tensorflow as tf +from absl.testing import parameterized +from PIL import Image from official.common import registry_imports # pylint: disable=unused-import from official.core import exp_factory diff --git a/official/vision/beta/serving/semantic_segmentation.py b/official/vision/beta/serving/semantic_segmentation.py index 14d8a45b2..520e5d699 100755 --- a/official/vision/beta/serving/semantic_segmentation.py +++ b/official/vision/beta/serving/semantic_segmentation.py @@ -21,7 +21,6 @@ from official.vision.beta.ops import preprocess_ops from official.vision.beta.serving import export_base - MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255) STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255) diff --git a/official/vision/beta/serving/semantic_segmentation_test.py b/official/vision/beta/serving/semantic_segmentation_test.py index f9c86515b..eebc3ef2d 100755 --- a/official/vision/beta/serving/semantic_segmentation_test.py +++ b/official/vision/beta/serving/semantic_segmentation_test.py @@ -18,10 +18,10 @@ import io import os -from absl.testing import parameterized import numpy as np -from PIL import Image import tensorflow as tf +from absl.testing import parameterized +from PIL import Image from official.common import registry_imports # pylint: disable=unused-import from official.core import exp_factory diff --git a/official/vision/beta/tasks/__init__.py b/official/vision/beta/tasks/__init__.py index 90a83d37d..dc544132f 100755 --- a/official/vision/beta/tasks/__init__.py +++ b/official/vision/beta/tasks/__init__.py @@ -15,8 +15,6 @@ # ============================================================================== """Tasks package definition.""" -from official.vision.beta.tasks import image_classification -from official.vision.beta.tasks import maskrcnn -from official.vision.beta.tasks import retinanet -from official.vision.beta.tasks import semantic_segmentation -from official.vision.beta.tasks import video_classification +from official.vision.beta.tasks import (image_classification, maskrcnn, + retinanet, semantic_segmentation, + video_classification) diff --git a/official/vision/beta/tasks/image_classification.py b/official/vision/beta/tasks/image_classification.py index afe62927d..cdb2c9ddf 100755 --- a/official/vision/beta/tasks/image_classification.py +++ b/official/vision/beta/tasks/image_classification.py @@ -14,16 +14,15 @@ # limitations under the License. # ============================================================================== """Image classification task definition.""" -from absl import logging import tensorflow as tf +from absl import logging + from official.common import dataset_fn -from official.core import base_task -from official.core import input_reader -from official.core import task_factory +from official.core import base_task, input_reader, task_factory from official.modeling import tf_utils from official.vision.beta.configs import image_classification as exp_cfg -from official.vision.beta.dataloaders import classification_input -from official.vision.beta.dataloaders import tfds_classification_decoders +from official.vision.beta.dataloaders import (classification_input, + tfds_classification_decoders) from official.vision.beta.modeling import factory diff --git a/official/vision/beta/tasks/maskrcnn.py b/official/vision/beta/tasks/maskrcnn.py index a48612fca..356496b17 100755 --- a/official/vision/beta/tasks/maskrcnn.py +++ b/official/vision/beta/tasks/maskrcnn.py @@ -15,16 +15,15 @@ # ============================================================================== """RetinaNet task definition.""" -from absl import logging import tensorflow as tf +from absl import logging + from official.common import dataset_fn -from official.core import base_task -from official.core import input_reader -from official.core import task_factory +from official.core import base_task, input_reader, task_factory from official.vision.beta.configs import maskrcnn as exp_cfg -from official.vision.beta.dataloaders import maskrcnn_input -from official.vision.beta.dataloaders import tf_example_decoder -from official.vision.beta.dataloaders import tf_example_label_map_decoder +from official.vision.beta.dataloaders import (maskrcnn_input, + tf_example_decoder, + tf_example_label_map_decoder) from official.vision.beta.evaluation import coco_evaluator from official.vision.beta.losses import maskrcnn_losses from official.vision.beta.modeling import factory diff --git a/official/vision/beta/tasks/retinanet.py b/official/vision/beta/tasks/retinanet.py index 4e323a2e7..f06dea1df 100755 --- a/official/vision/beta/tasks/retinanet.py +++ b/official/vision/beta/tasks/retinanet.py @@ -15,18 +15,17 @@ # ============================================================================== """RetinaNet task definition.""" -from absl import logging import tensorflow as tf +from absl import logging + from official.common import dataset_fn -from official.core import base_task -from official.core import input_reader -from official.core import task_factory +from official.core import base_task, input_reader, task_factory from official.vision import keras_cv from official.vision.beta.configs import retinanet as exp_cfg -from official.vision.beta.dataloaders import retinanet_input -from official.vision.beta.dataloaders import tf_example_decoder -from official.vision.beta.dataloaders import tfds_detection_decoders -from official.vision.beta.dataloaders import tf_example_label_map_decoder +from official.vision.beta.dataloaders import (retinanet_input, + tf_example_decoder, + tf_example_label_map_decoder, + tfds_detection_decoders) from official.vision.beta.evaluation import coco_evaluator from official.vision.beta.modeling import factory diff --git a/official/vision/beta/tasks/semantic_segmentation.py b/official/vision/beta/tasks/semantic_segmentation.py index b4b7e64f2..62fb787b4 100755 --- a/official/vision/beta/tasks/semantic_segmentation.py +++ b/official/vision/beta/tasks/semantic_segmentation.py @@ -15,15 +15,14 @@ # ============================================================================== """Image segmentation task definition.""" -from absl import logging import tensorflow as tf +from absl import logging + from official.common import dataset_fn -from official.core import base_task -from official.core import input_reader -from official.core import task_factory +from official.core import base_task, input_reader, task_factory from official.vision.beta.configs import semantic_segmentation as exp_cfg -from official.vision.beta.dataloaders import segmentation_input -from official.vision.beta.dataloaders import tfds_segmentation_decoders +from official.vision.beta.dataloaders import (segmentation_input, + tfds_segmentation_decoders) from official.vision.beta.evaluation import segmentation_metrics from official.vision.beta.losses import segmentation_losses from official.vision.beta.modeling import factory diff --git a/official/vision/beta/tasks/video_classification.py b/official/vision/beta/tasks/video_classification.py index 33c6601e2..3fca0596b 100755 --- a/official/vision/beta/tasks/video_classification.py +++ b/official/vision/beta/tasks/video_classification.py @@ -14,11 +14,10 @@ # limitations under the License. # ============================================================================== """Video classification task definition.""" -from absl import logging import tensorflow as tf -from official.core import base_task -from official.core import input_reader -from official.core import task_factory +from absl import logging + +from official.core import base_task, input_reader, task_factory from official.modeling import tf_utils from official.vision.beta.configs import video_classification as exp_cfg from official.vision.beta.dataloaders import video_input diff --git a/official/vision/beta/train.py b/official/vision/beta/train.py index 0b4b617d8..31aa26cc8 100755 --- a/official/vision/beta/train.py +++ b/official/vision/beta/train.py @@ -15,18 +15,15 @@ # ============================================================================== """TensorFlow Model Garden Vision training driver.""" -from absl import app -from absl import flags import gin +from absl import app, flags -# pylint: disable=unused-import -from official.common import registry_imports # pylint: enable=unused-import +# pylint: disable=unused-import from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib -from official.core import train_utils +from official.common import registry_imports +from official.core import task_factory, train_lib, train_utils from official.modeling import performance FLAGS = flags.FLAGS diff --git a/official/vision/beta/train_spatial_partitioning.py b/official/vision/beta/train_spatial_partitioning.py index 4f0e19d52..b9491b872 100755 --- a/official/vision/beta/train_spatial_partitioning.py +++ b/official/vision/beta/train_spatial_partitioning.py @@ -15,21 +15,17 @@ # ============================================================================== """TensorFlow Model Garden Vision training driver with spatial partitioning.""" -from absl import app -from absl import flags import gin import numpy as np import tensorflow as tf +from absl import app, flags from official.common import registry_imports # pylint: disable=unused-import from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib -from official.core import train_utils +from official.core import task_factory, train_lib, train_utils from official.modeling import performance - FLAGS = flags.FLAGS diff --git a/official/vision/detection/configs/factory.py b/official/vision/detection/configs/factory.py index 2bd48fec4..61b3421f5 100755 --- a/official/vision/detection/configs/factory.py +++ b/official/vision/detection/configs/factory.py @@ -15,10 +15,8 @@ """Factory to provide model configs.""" from official.modeling.hyperparams import params_dict -from official.vision.detection.configs import maskrcnn_config -from official.vision.detection.configs import olnmask_config -from official.vision.detection.configs import retinanet_config -from official.vision.detection.configs import shapemask_config +from official.vision.detection.configs import ( + maskrcnn_config, olnmask_config, retinanet_config, shapemask_config) def config_generator(model): diff --git a/official/vision/detection/configs/maskrcnn_config.py b/official/vision/detection/configs/maskrcnn_config.py index c62756274..bf8d052e6 100755 --- a/official/vision/detection/configs/maskrcnn_config.py +++ b/official/vision/detection/configs/maskrcnn_config.py @@ -17,7 +17,6 @@ from official.modeling.hyperparams import params_dict from official.vision.detection.configs import base_config - # pylint: disable=line-too-long MASKRCNN_CFG = params_dict.ParamsDict(base_config.BASE_CFG) MASKRCNN_CFG.override({ diff --git a/official/vision/detection/configs/olnmask_config.py b/official/vision/detection/configs/olnmask_config.py index d1fd24f20..db49e1973 100755 --- a/official/vision/detection/configs/olnmask_config.py +++ b/official/vision/detection/configs/olnmask_config.py @@ -17,7 +17,6 @@ from official.modeling.hyperparams import params_dict from official.vision.detection.configs import base_config - # pylint: disable=line-too-long OLNMASK_CFG = params_dict.ParamsDict(base_config.BASE_CFG) OLNMASK_CFG.override({ diff --git a/official/vision/detection/configs/retinanet_config.py b/official/vision/detection/configs/retinanet_config.py index 7e89163b4..e229e6243 100755 --- a/official/vision/detection/configs/retinanet_config.py +++ b/official/vision/detection/configs/retinanet_config.py @@ -17,7 +17,6 @@ from official.modeling.hyperparams import params_dict from official.vision.detection.configs import base_config - # pylint: disable=line-too-long RETINANET_CFG = params_dict.ParamsDict(base_config.BASE_CFG) RETINANET_CFG.override({ diff --git a/official/vision/detection/dataloader/anchor.py b/official/vision/detection/dataloader/anchor.py index 8ef9f4bd1..f790dd304 100755 --- a/official/vision/detection/dataloader/anchor.py +++ b/official/vision/detection/dataloader/anchor.py @@ -14,20 +14,17 @@ # ============================================================================== """Anchor box and labeler definition.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import collections import tensorflow as tf + from official.vision import keras_cv from official.vision.detection.utils import box_utils -from official.vision.detection.utils.object_detection import argmax_matcher -from official.vision.detection.utils.object_detection import balanced_positive_negative_sampler -from official.vision.detection.utils.object_detection import box_list -from official.vision.detection.utils.object_detection import faster_rcnn_box_coder -from official.vision.detection.utils.object_detection import target_assigner +from official.vision.detection.utils.object_detection import ( + argmax_matcher, balanced_positive_negative_sampler, box_list, + faster_rcnn_box_coder, target_assigner) class Anchor(object): diff --git a/official/vision/detection/dataloader/factory.py b/official/vision/detection/dataloader/factory.py index 414b5b964..76f0c18ae 100755 --- a/official/vision/detection/dataloader/factory.py +++ b/official/vision/detection/dataloader/factory.py @@ -14,14 +14,12 @@ # ============================================================================== """Model architecture factory.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from official.vision.detection.dataloader import maskrcnn_parser -from official.vision.detection.dataloader import olnmask_parser -from official.vision.detection.dataloader import retinanet_parser -from official.vision.detection.dataloader import shapemask_parser +from official.vision.detection.dataloader import (maskrcnn_parser, + olnmask_parser, + retinanet_parser, + shapemask_parser) def parser_generator(params, mode): diff --git a/official/vision/detection/dataloader/input_reader.py b/official/vision/detection/dataloader/input_reader.py index 2e405cf0e..53c88c6dd 100755 --- a/official/vision/detection/dataloader/input_reader.py +++ b/official/vision/detection/dataloader/input_reader.py @@ -14,14 +14,13 @@ # ============================================================================== """Data loader and input processing.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function + +from typing import Optional, Text import tensorflow as tf -from typing import Text, Optional from official.modeling.hyperparams import params_dict from official.vision.detection.dataloader import factory from official.vision.detection.dataloader import mode_keys as ModeKeys diff --git a/official/vision/detection/dataloader/maskrcnn_parser.py b/official/vision/detection/dataloader/maskrcnn_parser.py index 35db6f147..54f4926ff 100755 --- a/official/vision/detection/dataloader/maskrcnn_parser.py +++ b/official/vision/detection/dataloader/maskrcnn_parser.py @@ -19,9 +19,8 @@ from official.vision.detection.dataloader import anchor from official.vision.detection.dataloader import mode_keys as ModeKeys from official.vision.detection.dataloader import tf_example_decoder -from official.vision.detection.utils import box_utils -from official.vision.detection.utils import dataloader_utils -from official.vision.detection.utils import input_utils +from official.vision.detection.utils import (box_utils, dataloader_utils, + input_utils) class Parser(object): diff --git a/official/vision/detection/dataloader/mode_keys.py b/official/vision/detection/dataloader/mode_keys.py index 020382b24..eb725075d 100755 --- a/official/vision/detection/dataloader/mode_keys.py +++ b/official/vision/detection/dataloader/mode_keys.py @@ -22,10 +22,7 @@ * `PREDICT_WITH_GT`: prediction mode with groundtruths in returned variables. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - +from __future__ import absolute_import, division, print_function TRAIN = 'train' EVAL = 'eval' diff --git a/official/vision/detection/dataloader/olnmask_parser.py b/official/vision/detection/dataloader/olnmask_parser.py index cd1236a1a..58290dad6 100755 --- a/official/vision/detection/dataloader/olnmask_parser.py +++ b/official/vision/detection/dataloader/olnmask_parser.py @@ -17,10 +17,9 @@ import tensorflow as tf from official.vision.detection.dataloader import anchor -from official.vision.detection.dataloader.maskrcnn_parser import Parser as MaskrcnnParser -from official.vision.detection.utils import box_utils -from official.vision.detection.utils import class_utils -from official.vision.detection.utils import input_utils +from official.vision.detection.dataloader.maskrcnn_parser import \ + Parser as MaskrcnnParser +from official.vision.detection.utils import box_utils, class_utils, input_utils class Parser(MaskrcnnParser): diff --git a/official/vision/detection/dataloader/retinanet_parser.py b/official/vision/detection/dataloader/retinanet_parser.py index f4badbde6..62d92a74c 100755 --- a/official/vision/detection/dataloader/retinanet_parser.py +++ b/official/vision/detection/dataloader/retinanet_parser.py @@ -26,8 +26,7 @@ from official.vision.detection.dataloader import anchor from official.vision.detection.dataloader import mode_keys as ModeKeys from official.vision.detection.dataloader import tf_example_decoder -from official.vision.detection.utils import box_utils -from official.vision.detection.utils import input_utils +from official.vision.detection.utils import box_utils, input_utils def process_source_id(source_id): diff --git a/official/vision/detection/dataloader/shapemask_parser.py b/official/vision/detection/dataloader/shapemask_parser.py index 3fd50dbd8..9df030a86 100755 --- a/official/vision/detection/dataloader/shapemask_parser.py +++ b/official/vision/detection/dataloader/shapemask_parser.py @@ -26,10 +26,8 @@ from official.vision.detection.dataloader import anchor from official.vision.detection.dataloader import mode_keys as ModeKeys from official.vision.detection.dataloader import tf_example_decoder -from official.vision.detection.utils import box_utils -from official.vision.detection.utils import class_utils -from official.vision.detection.utils import dataloader_utils -from official.vision.detection.utils import input_utils +from official.vision.detection.utils import (box_utils, class_utils, + dataloader_utils, input_utils) def pad_to_size(input_tensor, size): diff --git a/official/vision/detection/evaluation/coco_evaluator.py b/official/vision/detection/evaluation/coco_evaluator.py index fdc5bbef2..9b89c4c94 100755 --- a/official/vision/detection/evaluation/coco_evaluator.py +++ b/official/vision/detection/evaluation/coco_evaluator.py @@ -26,19 +26,17 @@ See also: https://github.com/cocodataset/cocoapi/ """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import atexit import copy import tempfile -from absl import logging import numpy as np -from pycocotools import cocoeval import six import tensorflow as tf +from absl import logging +from pycocotools import cocoeval from official.vision.detection.evaluation import coco_utils from official.vision.detection.utils import class_utils diff --git a/official/vision/detection/evaluation/coco_utils.py b/official/vision/detection/evaluation/coco_utils.py index a4f366850..532621565 100755 --- a/official/vision/detection/evaluation/coco_utils.py +++ b/official/vision/detection/evaluation/coco_utils.py @@ -14,24 +14,21 @@ # ============================================================================== """Util functions related to pycocotools and COCO eval.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import copy import json -from absl import logging import numpy as np +import six +import tensorflow as tf +from absl import logging from PIL import Image from pycocotools import coco from pycocotools import mask as mask_api -import six -import tensorflow as tf from official.vision.detection.dataloader import tf_example_decoder -from official.vision.detection.utils import box_utils -from official.vision.detection.utils import mask_utils +from official.vision.detection.utils import box_utils, mask_utils class COCOWrapper(coco.COCO): diff --git a/official/vision/detection/evaluation/factory.py b/official/vision/detection/evaluation/factory.py index b24f48454..00b70fcbd 100755 --- a/official/vision/detection/evaluation/factory.py +++ b/official/vision/detection/evaluation/factory.py @@ -14,9 +14,7 @@ # ============================================================================== """Evaluator factory.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from official.vision.detection.evaluation import coco_evaluator diff --git a/official/vision/detection/executor/detection_executor.py b/official/vision/detection/executor/detection_executor.py index 0e76d7d16..6ffe7274f 100755 --- a/official/vision/detection/executor/detection_executor.py +++ b/official/vision/detection/executor/detection_executor.py @@ -14,16 +14,15 @@ # ============================================================================== """An executor class for running model on TensorFlow 2.0.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function +import tensorflow as tf from absl import logging -import tensorflow as tf from official.vision.detection.executor import distributed_executor as executor -from official.vision.detection.utils.object_detection import visualization_utils +from official.vision.detection.utils.object_detection import \ + visualization_utils class DetectionDistributedExecutor(executor.DistributedExecutor): diff --git a/official/vision/detection/executor/distributed_executor.py b/official/vision/detection/executor/distributed_executor.py index 75a214865..38902e06e 100755 --- a/official/vision/detection/executor/distributed_executor.py +++ b/official/vision/detection/executor/distributed_executor.py @@ -14,24 +14,20 @@ # ============================================================================== """Custom training loop for running TensorFlow 2.0 models.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os - -from absl import flags -from absl import logging +# pylint: disable=unused-import,g-import-not-at-top,redefined-outer-name,reimported +from typing import Any, Callable, Dict, Iterator, List, Optional, Text, Union import numpy as np import tensorflow as tf +from absl import flags, logging -# pylint: disable=unused-import,g-import-not-at-top,redefined-outer-name,reimported -from typing import Optional, Dict, List, Text, Callable, Union, Iterator, Any +from official.common import distribute_utils from official.modeling.hyperparams import params_dict from official.utils import hyperparams_flags -from official.common import distribute_utils from official.utils.misc import keras_utils FLAGS = flags.FLAGS diff --git a/official/vision/detection/main.py b/official/vision/detection/main.py index 4f28658f4..6cc283ab1 100755 --- a/official/vision/detection/main.py +++ b/official/vision/detection/main.py @@ -17,10 +17,8 @@ import functools import pprint -from absl import app -from absl import flags -from absl import logging import tensorflow as tf +from absl import app, flags, logging from official.common import distribute_utils from official.modeling.hyperparams import params_dict @@ -31,7 +29,8 @@ from official.vision.detection.dataloader import input_reader from official.vision.detection.dataloader import mode_keys as ModeKeys from official.vision.detection.executor import distributed_executor as executor -from official.vision.detection.executor.detection_executor import DetectionDistributedExecutor +from official.vision.detection.executor.detection_executor import \ + DetectionDistributedExecutor from official.vision.detection.modeling import factory as model_factory hyperparams_flags.initialize_common_flags() diff --git a/official/vision/detection/modeling/architecture/factory.py b/official/vision/detection/modeling/architecture/factory.py index 8d7cf7ad6..3f48392bc 100755 --- a/official/vision/detection/modeling/architecture/factory.py +++ b/official/vision/detection/modeling/architecture/factory.py @@ -14,16 +14,11 @@ # ============================================================================== """Model architecture factory.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from official.vision.detection.modeling.architecture import fpn -from official.vision.detection.modeling.architecture import heads -from official.vision.detection.modeling.architecture import identity -from official.vision.detection.modeling.architecture import nn_ops -from official.vision.detection.modeling.architecture import resnet -from official.vision.detection.modeling.architecture import spinenet +from __future__ import absolute_import, division, print_function + +from official.vision.detection.modeling.architecture import (fpn, heads, + identity, nn_ops, + resnet, spinenet) def norm_activation_generator(params): diff --git a/official/vision/detection/modeling/architecture/fpn.py b/official/vision/detection/modeling/architecture/fpn.py index f531b49e3..9526846b9 100755 --- a/official/vision/detection/modeling/architecture/fpn.py +++ b/official/vision/detection/modeling/architecture/fpn.py @@ -20,16 +20,13 @@ Feature Pyramid Networks for Object Detection. CVPR 2017. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import tensorflow as tf -from official.vision.detection.modeling.architecture import keras_utils -from official.vision.detection.modeling.architecture import nn_ops +from official.vision.detection.modeling.architecture import keras_utils, nn_ops from official.vision.detection.ops import spatial_transform_ops diff --git a/official/vision/detection/modeling/architecture/heads.py b/official/vision/detection/modeling/architecture/heads.py index c6084bd97..2e97fe6e3 100755 --- a/official/vision/detection/modeling/architecture/heads.py +++ b/official/vision/detection/modeling/architecture/heads.py @@ -14,17 +14,14 @@ # ============================================================================== """Classes to build various prediction heads in all supported models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import numpy as np import tensorflow as tf -from official.vision.detection.modeling.architecture import keras_utils -from official.vision.detection.modeling.architecture import nn_ops +from official.vision.detection.modeling.architecture import keras_utils, nn_ops from official.vision.detection.ops import spatial_transform_ops diff --git a/official/vision/detection/modeling/architecture/identity.py b/official/vision/detection/modeling/architecture/identity.py index acc90c4d5..181f57003 100755 --- a/official/vision/detection/modeling/architecture/identity.py +++ b/official/vision/detection/modeling/architecture/identity.py @@ -14,9 +14,7 @@ # ============================================================================== """Identity Fn that forwards the input features.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function class Identity(object): @@ -25,4 +23,3 @@ class Identity(object): def __call__(self, features, is_training=False): """Only forwards the input features.""" return features - diff --git a/official/vision/detection/modeling/architecture/keras_utils.py b/official/vision/detection/modeling/architecture/keras_utils.py index 530f8f1e2..5fb4d6ee7 100755 --- a/official/vision/detection/modeling/architecture/keras_utils.py +++ b/official/vision/detection/modeling/architecture/keras_utils.py @@ -14,10 +14,7 @@ # ============================================================================== """Util functions to integrate with Keras internals.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - +from __future__ import absolute_import, division, print_function from tensorflow.python.keras import backend diff --git a/official/vision/detection/modeling/architecture/nn_blocks.py b/official/vision/detection/modeling/architecture/nn_blocks.py index c94a079f9..f9b548ddb 100755 --- a/official/vision/detection/modeling/architecture/nn_blocks.py +++ b/official/vision/detection/modeling/architecture/nn_blocks.py @@ -14,9 +14,7 @@ # ============================================================================== """Contains common building blocks for neural networks.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf diff --git a/official/vision/detection/modeling/architecture/nn_ops.py b/official/vision/detection/modeling/architecture/nn_ops.py index 4ce14e8ff..4c6038a9f 100755 --- a/official/vision/detection/modeling/architecture/nn_ops.py +++ b/official/vision/detection/modeling/architecture/nn_ops.py @@ -14,9 +14,7 @@ # ============================================================================== """Neural network operations commonly shared by the architectures.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools diff --git a/official/vision/detection/modeling/architecture/resnet.py b/official/vision/detection/modeling/architecture/resnet.py index bdcec6961..57fc6d411 100755 --- a/official/vision/detection/modeling/architecture/resnet.py +++ b/official/vision/detection/modeling/architecture/resnet.py @@ -19,14 +19,12 @@ Deep Residual Learning for Image Recognition. arXiv:1512.03385 """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from absl import logging import tensorflow as tf -from official.vision.detection.modeling.architecture import keras_utils -from official.vision.detection.modeling.architecture import nn_ops +from absl import logging + +from official.vision.detection.modeling.architecture import keras_utils, nn_ops # TODO(b/140112644): Refactor the code with Keras style, i.e. build and call. diff --git a/official/vision/detection/modeling/architecture/spinenet.py b/official/vision/detection/modeling/architecture/spinenet.py index 9c1094cc7..740f89b8f 100755 --- a/official/vision/detection/modeling/architecture/spinenet.py +++ b/official/vision/detection/modeling/architecture/spinenet.py @@ -21,12 +21,12 @@ """ import math -from absl import logging import tensorflow as tf +from absl import logging from official.modeling import tf_utils -from official.vision.detection.modeling.architecture import keras_utils -from official.vision.detection.modeling.architecture import nn_blocks +from official.vision.detection.modeling.architecture import (keras_utils, + nn_blocks) layers = tf.keras.layers diff --git a/official/vision/detection/modeling/base_model.py b/official/vision/detection/modeling/base_model.py index e9dd5c6d1..bb5c7cff0 100755 --- a/official/vision/detection/modeling/base_model.py +++ b/official/vision/detection/modeling/base_model.py @@ -14,18 +14,16 @@ # ============================================================================== """Base Model definition.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import abc import functools import re import tensorflow as tf -from official.vision.detection.modeling import checkpoint_utils -from official.vision.detection.modeling import learning_rates -from official.vision.detection.modeling import optimizers + +from official.vision.detection.modeling import (checkpoint_utils, + learning_rates, optimizers) def _make_filter_trainable_variables_fn(frozen_variable_prefix): diff --git a/official/vision/detection/modeling/checkpoint_utils.py b/official/vision/detection/modeling/checkpoint_utils.py index 73760e832..c4d39beb5 100755 --- a/official/vision/detection/modeling/checkpoint_utils.py +++ b/official/vision/detection/modeling/checkpoint_utils.py @@ -18,15 +18,12 @@ checkpoint to Tensorflow 2.x (keras) model. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import re -from absl import logging - import tensorflow as tf +from absl import logging def _build_assignment_map(keras_model, diff --git a/official/vision/detection/modeling/factory.py b/official/vision/detection/modeling/factory.py index 1c25ec97b..e90f6d10d 100755 --- a/official/vision/detection/modeling/factory.py +++ b/official/vision/detection/modeling/factory.py @@ -15,10 +15,9 @@ """Factory to build detection model.""" -from official.vision.detection.modeling import maskrcnn_model -from official.vision.detection.modeling import olnmask_model -from official.vision.detection.modeling import retinanet_model -from official.vision.detection.modeling import shapemask_model +from official.vision.detection.modeling import (maskrcnn_model, olnmask_model, + retinanet_model, + shapemask_model) def model_generator(params): diff --git a/official/vision/detection/modeling/learning_rates.py b/official/vision/detection/modeling/learning_rates.py index 6756ce5ba..8243df64c 100755 --- a/official/vision/detection/modeling/learning_rates.py +++ b/official/vision/detection/modeling/learning_rates.py @@ -14,14 +14,13 @@ # ============================================================================== """Learning rate schedule.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import numpy as np import tensorflow as tf + from official.modeling.hyperparams import params_dict diff --git a/official/vision/detection/modeling/losses.py b/official/vision/detection/modeling/losses.py index 2cfe11d2b..bdde7e1bf 100755 --- a/official/vision/detection/modeling/losses.py +++ b/official/vision/detection/modeling/losses.py @@ -14,12 +14,10 @@ # ============================================================================== """Losses used for detection models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from absl import logging import tensorflow as tf +from absl import logging def focal_loss(logits, targets, alpha, gamma, normalizer): diff --git a/official/vision/detection/modeling/maskrcnn_model.py b/official/vision/detection/modeling/maskrcnn_model.py index aa347ffb4..2233f465d 100755 --- a/official/vision/detection/modeling/maskrcnn_model.py +++ b/official/vision/detection/modeling/maskrcnn_model.py @@ -14,23 +14,17 @@ # ============================================================================== """Model defination for the Mask R-CNN Model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf -from official.vision.detection.dataloader import anchor -from official.vision.detection.dataloader import mode_keys +from official.vision.detection.dataloader import anchor, mode_keys from official.vision.detection.evaluation import factory as eval_factory -from official.vision.detection.modeling import base_model -from official.vision.detection.modeling import losses -from official.vision.detection.modeling.architecture import factory -from official.vision.detection.modeling.architecture import keras_utils -from official.vision.detection.ops import postprocess_ops -from official.vision.detection.ops import roi_ops -from official.vision.detection.ops import spatial_transform_ops -from official.vision.detection.ops import target_ops +from official.vision.detection.modeling import base_model, losses +from official.vision.detection.modeling.architecture import (factory, + keras_utils) +from official.vision.detection.ops import (postprocess_ops, roi_ops, + spatial_transform_ops, target_ops) from official.vision.detection.utils import box_utils diff --git a/official/vision/detection/modeling/olnmask_model.py b/official/vision/detection/modeling/olnmask_model.py index b405c41cf..63a6ad4ba 100755 --- a/official/vision/detection/modeling/olnmask_model.py +++ b/official/vision/detection/modeling/olnmask_model.py @@ -14,22 +14,17 @@ # ============================================================================== """Model defination for the Object Localization Network (OLN) Model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf -from official.vision.detection.dataloader import anchor -from official.vision.detection.dataloader import mode_keys +from official.vision.detection.dataloader import anchor, mode_keys from official.vision.detection.modeling import losses -from official.vision.detection.modeling.architecture import factory -from official.vision.detection.modeling.architecture import keras_utils +from official.vision.detection.modeling.architecture import (factory, + keras_utils) from official.vision.detection.modeling.maskrcnn_model import MaskrcnnModel -from official.vision.detection.ops import postprocess_ops -from official.vision.detection.ops import roi_ops -from official.vision.detection.ops import spatial_transform_ops -from official.vision.detection.ops import target_ops +from official.vision.detection.ops import (postprocess_ops, roi_ops, + spatial_transform_ops, target_ops) from official.vision.detection.utils import box_utils diff --git a/official/vision/detection/modeling/optimizers.py b/official/vision/detection/modeling/optimizers.py index fd51bb59f..1a573a935 100755 --- a/official/vision/detection/modeling/optimizers.py +++ b/official/vision/detection/modeling/optimizers.py @@ -14,9 +14,7 @@ # ============================================================================== """Optimizers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools diff --git a/official/vision/detection/modeling/retinanet_model.py b/official/vision/detection/modeling/retinanet_model.py index 0714d9784..6f7fc23c3 100755 --- a/official/vision/detection/modeling/retinanet_model.py +++ b/official/vision/detection/modeling/retinanet_model.py @@ -14,18 +14,15 @@ # ============================================================================== """Model defination for the RetinaNet Model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf from official.vision.detection.dataloader import mode_keys from official.vision.detection.evaluation import factory as eval_factory -from official.vision.detection.modeling import base_model -from official.vision.detection.modeling import losses -from official.vision.detection.modeling.architecture import factory -from official.vision.detection.modeling.architecture import keras_utils +from official.vision.detection.modeling import base_model, losses +from official.vision.detection.modeling.architecture import (factory, + keras_utils) from official.vision.detection.ops import postprocess_ops diff --git a/official/vision/detection/modeling/shapemask_model.py b/official/vision/detection/modeling/shapemask_model.py index 8e3085422..d1af48505 100755 --- a/official/vision/detection/modeling/shapemask_model.py +++ b/official/vision/detection/modeling/shapemask_model.py @@ -14,19 +14,15 @@ # ============================================================================== """Model definition for the ShapeMask Model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf -from official.vision.detection.dataloader import anchor -from official.vision.detection.dataloader import mode_keys +from official.vision.detection.dataloader import anchor, mode_keys from official.vision.detection.evaluation import factory as eval_factory -from official.vision.detection.modeling import base_model -from official.vision.detection.modeling import losses -from official.vision.detection.modeling.architecture import factory -from official.vision.detection.modeling.architecture import keras_utils +from official.vision.detection.modeling import base_model, losses +from official.vision.detection.modeling.architecture import (factory, + keras_utils) from official.vision.detection.ops import postprocess_ops from official.vision.detection.utils import box_utils diff --git a/official/vision/detection/ops/nms.py b/official/vision/detection/ops/nms.py index 7f4468992..ec9cd2f16 100755 --- a/official/vision/detection/ops/nms.py +++ b/official/vision/detection/ops/nms.py @@ -14,9 +14,7 @@ # ============================================================================== """Tensorflow implementation of non max suppression.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf diff --git a/official/vision/detection/ops/postprocess_ops.py b/official/vision/detection/ops/postprocess_ops.py index 802f9072e..397e86704 100755 --- a/official/vision/detection/ops/postprocess_ops.py +++ b/official/vision/detection/ops/postprocess_ops.py @@ -14,9 +14,7 @@ # ============================================================================== """Post-processing model outputs to generate detection.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools diff --git a/official/vision/detection/ops/roi_ops.py b/official/vision/detection/ops/roi_ops.py index ee67bb0ad..86f98eee6 100755 --- a/official/vision/detection/ops/roi_ops.py +++ b/official/vision/detection/ops/roi_ops.py @@ -14,9 +14,7 @@ # ============================================================================== """ROI-related ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf diff --git a/official/vision/detection/ops/spatial_transform_ops.py b/official/vision/detection/ops/spatial_transform_ops.py index c80677646..b4677c5eb 100755 --- a/official/vision/detection/ops/spatial_transform_ops.py +++ b/official/vision/detection/ops/spatial_transform_ops.py @@ -14,9 +14,7 @@ # ============================================================================== """Functions to performa spatial transformation for Tensor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf diff --git a/official/vision/detection/ops/target_ops.py b/official/vision/detection/ops/target_ops.py index a2b54b207..3099be47c 100755 --- a/official/vision/detection/ops/target_ops.py +++ b/official/vision/detection/ops/target_ops.py @@ -14,15 +14,14 @@ # ============================================================================== """Target and sampling related ops.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf from official.vision.detection.ops import spatial_transform_ops from official.vision.detection.utils import box_utils -from official.vision.detection.utils.object_detection import balanced_positive_negative_sampler +from official.vision.detection.utils.object_detection import \ + balanced_positive_negative_sampler def box_matching(boxes, gt_boxes, gt_classes): diff --git a/official/vision/detection/utils/box_utils.py b/official/vision/detection/utils/box_utils.py index d8ad9f13d..a53da30fe 100755 --- a/official/vision/detection/utils/box_utils.py +++ b/official/vision/detection/utils/box_utils.py @@ -14,10 +14,8 @@ # ============================================================================== """Utility functions for bounding box processing.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np import tensorflow as tf diff --git a/official/vision/detection/utils/mask_utils.py b/official/vision/detection/utils/mask_utils.py index 773aced60..c37782fc4 100755 --- a/official/vision/detection/utils/mask_utils.py +++ b/official/vision/detection/utils/mask_utils.py @@ -13,14 +13,12 @@ # limitations under the License. # ============================================================================== """Utility functions for segmentations.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import math -import numpy as np import cv2 +import numpy as np def paste_instance_masks(masks, detected_boxes, image_height, image_width): diff --git a/official/vision/detection/utils/object_detection/argmax_matcher.py b/official/vision/detection/utils/object_detection/argmax_matcher.py index 3f8b051bf..58c8a860d 100755 --- a/official/vision/detection/utils/object_detection/argmax_matcher.py +++ b/official/vision/detection/utils/object_detection/argmax_matcher.py @@ -28,8 +28,8 @@ """ import tensorflow as tf -from official.vision.detection.utils.object_detection import matcher -from official.vision.detection.utils.object_detection import shape_utils +from official.vision.detection.utils.object_detection import (matcher, + shape_utils) class ArgMaxMatcher(matcher.Matcher): diff --git a/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py b/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py index 66b4b1a66..8d3a9278d 100755 --- a/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py +++ b/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py @@ -33,8 +33,8 @@ import tensorflow as tf -from official.vision.detection.utils.object_detection import minibatch_sampler -from official.vision.detection.utils.object_detection import ops +from official.vision.detection.utils.object_detection import ( + minibatch_sampler, ops) class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler): diff --git a/official/vision/detection/utils/object_detection/box_coder.py b/official/vision/detection/utils/object_detection/box_coder.py index 1ccb347f1..b1aeefe1f 100755 --- a/official/vision/detection/utils/object_detection/box_coder.py +++ b/official/vision/detection/utils/object_detection/box_coder.py @@ -25,9 +25,7 @@ In both cases, the arguments are assumed to be in 1-1 correspondence already; it is not the job of a BoxCoder to perform matching. """ -from abc import ABCMeta -from abc import abstractmethod -from abc import abstractproperty +from abc import ABCMeta, abstractmethod, abstractproperty import tensorflow as tf diff --git a/official/vision/detection/utils/object_detection/box_list_ops.py b/official/vision/detection/utils/object_detection/box_list_ops.py index 0420d8aa3..6aad38585 100755 --- a/official/vision/detection/utils/object_detection/box_list_ops.py +++ b/official/vision/detection/utils/object_detection/box_list_ops.py @@ -22,15 +22,12 @@ Whenever box_list_ops functions output a BoxList, the fields of the incoming BoxList are retained unless documented otherwise. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from six.moves import range import tensorflow as tf +from six.moves import range -from official.vision.detection.utils.object_detection import box_list -from official.vision.detection.utils.object_detection import ops +from official.vision.detection.utils.object_detection import box_list, ops class SortOrder(object): diff --git a/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py b/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py index 11bd50c38..6e7aacf19 100755 --- a/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py +++ b/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py @@ -29,8 +29,8 @@ import tensorflow as tf -from official.vision.detection.utils.object_detection import box_coder -from official.vision.detection.utils.object_detection import box_list +from official.vision.detection.utils.object_detection import (box_coder, + box_list) EPSILON = 1e-8 diff --git a/official/vision/detection/utils/object_detection/matcher.py b/official/vision/detection/utils/object_detection/matcher.py index 1ae806f57..0b48dbd1e 100755 --- a/official/vision/detection/utils/object_detection/matcher.py +++ b/official/vision/detection/utils/object_detection/matcher.py @@ -30,8 +30,7 @@ The Match class is used to store the match results and it provides simple apis to query the results. """ -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod import tensorflow as tf diff --git a/official/vision/detection/utils/object_detection/minibatch_sampler.py b/official/vision/detection/utils/object_detection/minibatch_sampler.py index 81acb4f2d..3b937952e 100755 --- a/official/vision/detection/utils/object_detection/minibatch_sampler.py +++ b/official/vision/detection/utils/object_detection/minibatch_sampler.py @@ -29,8 +29,7 @@ This is originally implemented in TensorFlow Object Detection API. """ -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod import tensorflow as tf diff --git a/official/vision/detection/utils/object_detection/preprocessor.py b/official/vision/detection/utils/object_detection/preprocessor.py index 8a9da9004..40a533b41 100755 --- a/official/vision/detection/utils/object_detection/preprocessor.py +++ b/official/vision/detection/utils/object_detection/preprocessor.py @@ -39,9 +39,8 @@ back to rank 4. """ -import tensorflow as tf - import numpy as np +import tensorflow as tf from official.vision.detection.utils.object_detection import box_list diff --git a/official/vision/detection/utils/object_detection/region_similarity_calculator.py b/official/vision/detection/utils/object_detection/region_similarity_calculator.py index e7ae375e6..b66a3336b 100755 --- a/official/vision/detection/utils/object_detection/region_similarity_calculator.py +++ b/official/vision/detection/utils/object_detection/region_similarity_calculator.py @@ -17,8 +17,7 @@ Region Similarity Calculators compare a pairwise measure of similarity between the boxes in two BoxLists. """ -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod import tensorflow as tf diff --git a/official/vision/detection/utils/object_detection/target_assigner.py b/official/vision/detection/utils/object_detection/target_assigner.py index 828097cf4..a5dffa2fa 100755 --- a/official/vision/detection/utils/object_detection/target_assigner.py +++ b/official/vision/detection/utils/object_detection/target_assigner.py @@ -33,8 +33,8 @@ import tensorflow as tf -from official.vision.detection.utils.object_detection import box_list -from official.vision.detection.utils.object_detection import shape_utils +from official.vision.detection.utils.object_detection import (box_list, + shape_utils) KEYPOINTS_FIELD_NAME = 'keypoints' diff --git a/official/vision/detection/utils/object_detection/visualization_utils.py b/official/vision/detection/utils/object_detection/visualization_utils.py index ada70af18..53f6fbc58 100755 --- a/official/vision/detection/utils/object_detection/visualization_utils.py +++ b/official/vision/detection/utils/object_detection/visualization_utils.py @@ -21,10 +21,8 @@ import collections import functools -from absl import logging # Set headless-friendly backend. import matplotlib -matplotlib.use('Agg') # pylint: disable=multiple-statements import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top import numpy as np import PIL.Image as Image @@ -33,10 +31,14 @@ import PIL.ImageFont as ImageFont import six import tensorflow as tf +from absl import logging from official.vision.detection.utils import box_utils from official.vision.detection.utils.object_detection import shape_utils +matplotlib.use('Agg') # pylint: disable=multiple-statements + + _TITLE_LEFT_MARGIN = 10 _TITLE_TOP_MARGIN = 10 STANDARD_COLORS = [ diff --git a/official/vision/image_classification/augment.py b/official/vision/image_classification/augment.py index 3a96a648d..88fcc2291 100755 --- a/official/vision/image_classification/augment.py +++ b/official/vision/image_classification/augment.py @@ -18,17 +18,15 @@ RandAugment Reference: https://arxiv.org/abs/1909.13719 """ -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import math - -import tensorflow as tf from typing import Any, Dict, List, Optional, Text, Tuple -from tensorflow.python.keras.layers.preprocessing import image_preprocessing as image_ops +import tensorflow as tf +from tensorflow.python.keras.layers.preprocessing import \ + image_preprocessing as image_ops # This signifies the max integer that the controller RNN could predict for the # augmentation scheme. diff --git a/official/vision/image_classification/augment_test.py b/official/vision/image_classification/augment_test.py index 27db5eb51..27a138e29 100755 --- a/official/vision/image_classification/augment_test.py +++ b/official/vision/image_classification/augment_test.py @@ -14,14 +14,11 @@ # ============================================================================== """Tests for autoaugment.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function - -from absl.testing import parameterized +from __future__ import absolute_import, division, print_function import tensorflow as tf +from absl.testing import parameterized from official.vision.image_classification import augment diff --git a/official/vision/image_classification/callbacks.py b/official/vision/image_classification/callbacks.py index 3d07f1e2a..6b4ec1abb 100755 --- a/official/vision/image_classification/callbacks.py +++ b/official/vision/image_classification/callbacks.py @@ -14,16 +14,14 @@ # limitations under the License. # ============================================================================== """Common modules for callbacks.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os from typing import Any, List, MutableMapping, Text -from absl import logging import tensorflow as tf +from absl import logging from official.modeling import optimization from official.utils.misc import keras_utils diff --git a/official/vision/image_classification/classifier_trainer.py b/official/vision/image_classification/classifier_trainer.py index 415225243..a651bd16a 100755 --- a/official/vision/image_classification/classifier_trainer.py +++ b/official/vision/image_classification/classifier_trainer.py @@ -17,25 +17,22 @@ import os import pprint -from typing import Any, Tuple, Text, Optional, Mapping +from typing import Any, Mapping, Optional, Text, Tuple -from absl import app -from absl import flags -from absl import logging import tensorflow as tf +from absl import app, flags, logging + from official.common import distribute_utils -from official.modeling import hyperparams -from official.modeling import performance +from official.modeling import hyperparams, performance from official.utils import hyperparams_flags from official.utils.misc import keras_utils from official.vision.image_classification import callbacks as custom_callbacks -from official.vision.image_classification import dataset_factory -from official.vision.image_classification import optimizer_factory -from official.vision.image_classification.configs import base_configs -from official.vision.image_classification.configs import configs -from official.vision.image_classification.efficientnet import efficientnet_model -from official.vision.image_classification.resnet import common -from official.vision.image_classification.resnet import resnet_model +from official.vision.image_classification import (dataset_factory, + optimizer_factory) +from official.vision.image_classification.configs import base_configs, configs +from official.vision.image_classification.efficientnet import \ + efficientnet_model +from official.vision.image_classification.resnet import common, resnet_model def get_models() -> Mapping[str, tf.keras.Model]: diff --git a/official/vision/image_classification/classifier_trainer_test.py b/official/vision/image_classification/classifier_trainer_test.py index cc61bb84c..fa8135719 100755 --- a/official/vision/image_classification/classifier_trainer_test.py +++ b/official/vision/image_classification/classifier_trainer_test.py @@ -15,29 +15,23 @@ # ============================================================================== """Unit tests for the classifier trainer models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools import json - import os import sys +from typing import ( + Any, Callable, Iterable, Mapping, MutableMapping, Optional, Tuple) -from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional, Tuple - -from absl import flags -from absl.testing import flagsaver -from absl.testing import parameterized import tensorflow as tf +from absl import flags +from absl.testing import flagsaver, parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.utils.flags import core as flags_core from official.vision.image_classification import classifier_trainer - classifier_trainer.define_classifier_flags() diff --git a/official/vision/image_classification/classifier_trainer_util_test.py b/official/vision/image_classification/classifier_trainer_util_test.py index bbb7dfb66..cc50239cf 100755 --- a/official/vision/image_classification/classifier_trainer_util_test.py +++ b/official/vision/image_classification/classifier_trainer_util_test.py @@ -15,19 +15,16 @@ # ============================================================================== """Unit tests for the classifier trainer models.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import copy import os -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized -from official.vision.image_classification import classifier_trainer -from official.vision.image_classification import dataset_factory -from official.vision.image_classification import test_utils +from official.vision.image_classification import (classifier_trainer, + dataset_factory, test_utils) from official.vision.image_classification.configs import base_configs diff --git a/official/vision/image_classification/configs/base_configs.py b/official/vision/image_classification/configs/base_configs.py index 316e8f2c4..8408f5614 100755 --- a/official/vision/image_classification/configs/base_configs.py +++ b/official/vision/image_classification/configs/base_configs.py @@ -14,9 +14,8 @@ # limitations under the License. # ============================================================================== """Definitions for high level configuration groups..""" -from typing import Any, List, Mapping, Optional - import dataclasses +from typing import Any, List, Mapping, Optional from official.core import config_definitions from official.modeling import hyperparams diff --git a/official/vision/image_classification/configs/configs.py b/official/vision/image_classification/configs/configs.py index 20b332143..a81d6e4fa 100755 --- a/official/vision/image_classification/configs/configs.py +++ b/official/vision/image_classification/configs/configs.py @@ -14,15 +14,14 @@ # limitations under the License. # ============================================================================== """Configuration utils for image classification experiments.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import dataclasses from official.vision.image_classification import dataset_factory from official.vision.image_classification.configs import base_configs -from official.vision.image_classification.efficientnet import efficientnet_config +from official.vision.image_classification.efficientnet import \ + efficientnet_config from official.vision.image_classification.resnet import resnet_config diff --git a/official/vision/image_classification/dataset_factory.py b/official/vision/image_classification/dataset_factory.py index d452f22bf..c93d9b2b3 100755 --- a/official/vision/image_classification/dataset_factory.py +++ b/official/vision/image_classification/dataset_factory.py @@ -14,22 +14,19 @@ # limitations under the License. # ============================================================================== """Dataset utilities for vision tasks using TFDS and tf.data.Dataset.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -from typing import Any, List, Optional, Tuple, Mapping, Union - -from absl import logging from dataclasses import dataclass +from typing import Any, List, Mapping, Optional, Tuple, Union + import tensorflow as tf import tensorflow_datasets as tfds +from absl import logging from official.modeling.hyperparams import base_config -from official.vision.image_classification import augment -from official.vision.image_classification import preprocessing +from official.vision.image_classification import augment, preprocessing AUGMENTERS = { 'autoaugment': augment.AutoAugment, diff --git a/official/vision/image_classification/efficientnet/common_modules.py b/official/vision/image_classification/efficientnet/common_modules.py index f656dc6d0..36a582104 100755 --- a/official/vision/image_classification/efficientnet/common_modules.py +++ b/official/vision/image_classification/efficientnet/common_modules.py @@ -13,16 +13,14 @@ # limitations under the License. # ============================================================================== """Common modeling utilities.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function + +from typing import Optional, Text import numpy as np import tensorflow as tf import tensorflow.compat.v1 as tf1 -from typing import Text, Optional - from tensorflow.python.tpu import tpu_function diff --git a/official/vision/image_classification/efficientnet/efficientnet_config.py b/official/vision/image_classification/efficientnet/efficientnet_config.py index a758cc63c..9705ae46d 100755 --- a/official/vision/image_classification/efficientnet/efficientnet_config.py +++ b/official/vision/image_classification/efficientnet/efficientnet_config.py @@ -14,13 +14,10 @@ # limitations under the License. # ============================================================================== """Configuration definitions for EfficientNet losses, learning rates, and optimizers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from typing import Any, Mapping +from __future__ import absolute_import, division, print_function import dataclasses +from typing import Any, Mapping from official.modeling.hyperparams import base_config from official.vision.image_classification.configs import base_configs diff --git a/official/vision/image_classification/efficientnet/efficientnet_model.py b/official/vision/image_classification/efficientnet/efficientnet_model.py index 01c0bc091..fa6157604 100755 --- a/official/vision/image_classification/efficientnet/efficientnet_model.py +++ b/official/vision/image_classification/efficientnet/efficientnet_model.py @@ -20,17 +20,15 @@ ICML'19, https://arxiv.org/abs/1905.11946 """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import math import os +from dataclasses import dataclass from typing import Any, Dict, Optional, Text, Tuple -from absl import logging -from dataclasses import dataclass import tensorflow as tf +from absl import logging from official.modeling import tf_utils from official.modeling.hyperparams import base_config diff --git a/official/vision/image_classification/efficientnet/tfhub_export.py b/official/vision/image_classification/efficientnet/tfhub_export.py index 71589920f..f171b6b46 100755 --- a/official/vision/image_classification/efficientnet/tfhub_export.py +++ b/official/vision/image_classification/efficientnet/tfhub_export.py @@ -14,19 +14,16 @@ # ============================================================================== """A script to export TF-Hub SavedModel.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -from absl import app -from absl import flags - import tensorflow as tf +from absl import app, flags -from official.vision.image_classification.efficientnet import efficientnet_model +from official.vision.image_classification.efficientnet import \ + efficientnet_model FLAGS = flags.FLAGS diff --git a/official/vision/image_classification/learning_rate.py b/official/vision/image_classification/learning_rate.py index becb118d7..3d5fa19f4 100755 --- a/official/vision/image_classification/learning_rate.py +++ b/official/vision/image_classification/learning_rate.py @@ -14,9 +14,7 @@ # limitations under the License. # ============================================================================== """Learning rate utilities for vision tasks.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function from typing import Any, Mapping, Optional diff --git a/official/vision/image_classification/learning_rate_test.py b/official/vision/image_classification/learning_rate_test.py index 52809abd1..ecfa8d2ad 100755 --- a/official/vision/image_classification/learning_rate_test.py +++ b/official/vision/image_classification/learning_rate_test.py @@ -14,9 +14,7 @@ # ============================================================================== """Tests for learning_rate.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf diff --git a/official/vision/image_classification/mnist_main.py b/official/vision/image_classification/mnist_main.py index 0d85853c3..9fd6174a4 100755 --- a/official/vision/image_classification/mnist_main.py +++ b/official/vision/image_classification/mnist_main.py @@ -13,18 +13,15 @@ # limitations under the License. # ============================================================================== """Runs a simple model on the MNIST dataset.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -# Import libraries -from absl import app -from absl import flags -from absl import logging import tensorflow as tf import tensorflow_datasets as tfds +# Import libraries +from absl import app, flags, logging + from official.common import distribute_utils from official.utils.flags import core as flags_core from official.utils.misc import model_helpers diff --git a/official/vision/image_classification/mnist_test.py b/official/vision/image_classification/mnist_test.py index fb97fb945..b0120d622 100755 --- a/official/vision/image_classification/mnist_test.py +++ b/official/vision/image_classification/mnist_test.py @@ -14,21 +14,17 @@ # ============================================================================== """Test the Keras MNIST model on GPU.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import functools -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from official.utils.testing import integration from official.vision.image_classification import mnist_main - mnist_main.define_mnist_flags() diff --git a/official/vision/image_classification/optimizer_factory.py b/official/vision/image_classification/optimizer_factory.py index 4e138d617..8dc492c3f 100755 --- a/official/vision/image_classification/optimizer_factory.py +++ b/official/vision/image_classification/optimizer_factory.py @@ -13,16 +13,14 @@ # limitations under the License. # ============================================================================== """Optimizer factory for vision tasks.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function from typing import Any, Dict, Text -from absl import logging import tensorflow as tf import tensorflow_addons as tfa +from absl import logging from official.modeling import optimization from official.vision.image_classification import learning_rate diff --git a/official/vision/image_classification/optimizer_factory_test.py b/official/vision/image_classification/optimizer_factory_test.py index 538605574..9b160302c 100755 --- a/official/vision/image_classification/optimizer_factory_test.py +++ b/official/vision/image_classification/optimizer_factory_test.py @@ -14,14 +14,12 @@ # ============================================================================== """Tests for optimizer_factory.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function +import tensorflow as tf from absl.testing import parameterized -import tensorflow as tf from official.vision.image_classification import optimizer_factory from official.vision.image_classification.configs import base_configs diff --git a/official/vision/image_classification/preprocessing.py b/official/vision/image_classification/preprocessing.py index 87ddeffdb..23548eef1 100755 --- a/official/vision/image_classification/preprocessing.py +++ b/official/vision/image_classification/preprocessing.py @@ -14,16 +14,14 @@ # ============================================================================== """Preprocessing functions for images.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function -import tensorflow as tf from typing import List, Optional, Text, Tuple -from official.vision.image_classification import augment +import tensorflow as tf +from official.vision.image_classification import augment # Calculated from the ImageNet training set MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255) diff --git a/official/vision/image_classification/resnet/common.py b/official/vision/image_classification/resnet/common.py index b9ea044cf..fe5a408ec 100755 --- a/official/vision/image_classification/resnet/common.py +++ b/official/vision/image_classification/resnet/common.py @@ -13,16 +13,15 @@ # limitations under the License. # ============================================================================== """Common util functions and classes used by both keras cifar and imagenet.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -from absl import flags import tensorflow as tf +from absl import flags +from tensorflow.python.keras.optimizer_v2 import \ + gradient_descent as gradient_descent_v2 -from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2 import tensorflow_model_optimization as tfmot from official.utils.flags import core as flags_core from official.utils.misc import keras_utils diff --git a/official/vision/image_classification/resnet/imagenet_preprocessing.py b/official/vision/image_classification/resnet/imagenet_preprocessing.py index 9f88471a8..d2f560ebf 100755 --- a/official/vision/image_classification/resnet/imagenet_preprocessing.py +++ b/official/vision/image_classification/resnet/imagenet_preprocessing.py @@ -31,14 +31,12 @@ """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -from absl import logging import tensorflow as tf +from absl import logging DEFAULT_IMAGE_SIZE = 224 NUM_CHANNELS = 3 diff --git a/official/vision/image_classification/resnet/resnet_config.py b/official/vision/image_classification/resnet/resnet_config.py index 787c8fd9f..8e09129d3 100755 --- a/official/vision/image_classification/resnet/resnet_config.py +++ b/official/vision/image_classification/resnet/resnet_config.py @@ -14,9 +14,7 @@ # limitations under the License. # ============================================================================== """Configuration definitions for ResNet losses, learning rates, and optimizers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import dataclasses diff --git a/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py b/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py index 400c2ac0c..0868b86fa 100755 --- a/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py +++ b/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py @@ -17,20 +17,17 @@ import math import os +import tensorflow as tf # Import libraries -from absl import app -from absl import flags -from absl import logging +from absl import app, flags, logging + import orbit -import tensorflow as tf from official.common import distribute_utils from official.modeling import performance from official.utils.flags import core as flags_core -from official.utils.misc import keras_utils -from official.utils.misc import model_helpers -from official.vision.image_classification.resnet import common -from official.vision.image_classification.resnet import imagenet_preprocessing -from official.vision.image_classification.resnet import resnet_runnable +from official.utils.misc import keras_utils, model_helpers +from official.vision.image_classification.resnet import ( + common, imagenet_preprocessing, resnet_runnable) flags.DEFINE_boolean(name='use_tf_function', default=True, help='Wrap the train and test step inside a ' diff --git a/official/vision/image_classification/resnet/resnet_model.py b/official/vision/image_classification/resnet/resnet_model.py index 10f123335..2cdd184b9 100755 --- a/official/vision/image_classification/resnet/resnet_model.py +++ b/official/vision/image_classification/resnet/resnet_model.py @@ -23,16 +23,11 @@ - http://torch.ch/blog/2016/02/04/resnets.html """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf +from tensorflow.python.keras import backend, initializers, models, regularizers -from tensorflow.python.keras import backend -from tensorflow.python.keras import initializers -from tensorflow.python.keras import models -from tensorflow.python.keras import regularizers from official.vision.image_classification.resnet import imagenet_preprocessing layers = tf.keras.layers diff --git a/official/vision/image_classification/resnet/resnet_runnable.py b/official/vision/image_classification/resnet/resnet_runnable.py index cd63f1f86..ba06d4f3f 100755 --- a/official/vision/image_classification/resnet/resnet_runnable.py +++ b/official/vision/image_classification/resnet/resnet_runnable.py @@ -14,15 +14,14 @@ # ============================================================================== """Runs a ResNet model on the ImageNet dataset using custom training loops.""" -import orbit import tensorflow as tf +import orbit from official.modeling import performance from official.staging.training import grad_utils from official.utils.flags import core as flags_core -from official.vision.image_classification.resnet import common -from official.vision.image_classification.resnet import imagenet_preprocessing -from official.vision.image_classification.resnet import resnet_model +from official.vision.image_classification.resnet import ( + common, imagenet_preprocessing, resnet_model) class ResnetRunnable(orbit.StandardTrainer, orbit.StandardEvaluator): diff --git a/official/vision/image_classification/resnet/tfhub_export.py b/official/vision/image_classification/resnet/tfhub_export.py index 2f49f5523..838b7d25b 100755 --- a/official/vision/image_classification/resnet/tfhub_export.py +++ b/official/vision/image_classification/resnet/tfhub_export.py @@ -14,21 +14,17 @@ # ============================================================================== """A script to export TF-Hub SavedModel.""" -from __future__ import absolute_import -from __future__ import division # from __future__ import google_type_annotations -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os -# Import libraries -from absl import app -from absl import flags - import tensorflow as tf +# Import libraries +from absl import app, flags -from official.vision.image_classification.resnet import imagenet_preprocessing -from official.vision.image_classification.resnet import resnet_model +from official.vision.image_classification.resnet import ( + imagenet_preprocessing, resnet_model) FLAGS = flags.FLAGS diff --git a/official/vision/image_classification/test_utils.py b/official/vision/image_classification/test_utils.py index a6dc91dc7..98bf69ce6 100755 --- a/official/vision/image_classification/test_utils.py +++ b/official/vision/image_classification/test_utils.py @@ -14,13 +14,9 @@ # ============================================================================== """Test utilities for image classification tasks.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from tensorflow.python.keras import backend -from tensorflow.python.keras import layers -from tensorflow.python.keras import models +from tensorflow.python.keras import backend, layers, models def trivial_model(num_classes): diff --git a/official/vision/keras_cv/__init__.py b/official/vision/keras_cv/__init__.py index d0f087e73..b99c3af42 100755 --- a/official/vision/keras_cv/__init__.py +++ b/official/vision/keras_cv/__init__.py @@ -14,7 +14,4 @@ # ============================================================================== """Keras-CV package definition.""" # pylint: disable=wildcard-import -from official.vision.keras_cv import layers -from official.vision.keras_cv import losses -from official.vision.keras_cv import metrics -from official.vision.keras_cv import ops +from official.vision.keras_cv import layers, losses, metrics, ops diff --git a/official/vision/keras_cv/layers/deeplab_test.py b/official/vision/keras_cv/layers/deeplab_test.py index 7f5a750c5..80b0f5bf7 100755 --- a/official/vision/keras_cv/layers/deeplab_test.py +++ b/official/vision/keras_cv/layers/deeplab_test.py @@ -15,8 +15,8 @@ """Tests for ASPP.""" import tensorflow as tf - from tensorflow.python.keras import keras_parameterized + from official.vision.keras_cv.layers import deeplab diff --git a/official/vision/keras_cv/ops/anchor_generator_test.py b/official/vision/keras_cv/ops/anchor_generator_test.py index 7b95d4032..a3050f691 100755 --- a/official/vision/keras_cv/ops/anchor_generator_test.py +++ b/official/vision/keras_cv/ops/anchor_generator_test.py @@ -14,8 +14,9 @@ # ============================================================================== """Tests for anchor_generator.py.""" -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized + from official.vision.keras_cv.ops import anchor_generator diff --git a/official/vision/keras_cv/setup.py b/official/vision/keras_cv/setup.py index e9e8ca233..1c14b8dd5 100755 --- a/official/vision/keras_cv/setup.py +++ b/official/vision/keras_cv/setup.py @@ -16,8 +16,7 @@ import os -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup version = '0.0.1' diff --git a/orbit/__init__.py b/orbit/__init__.py index a97bb719d..cdd006c33 100755 --- a/orbit/__init__.py +++ b/orbit/__init__.py @@ -15,13 +15,8 @@ """Defines exported symbols for the `orbit` package.""" from orbit import utils - from orbit.controller import Controller - -from orbit.runner import AbstractEvaluator -from orbit.runner import AbstractTrainer - -from orbit.standard_runner import StandardEvaluator -from orbit.standard_runner import StandardEvaluatorOptions -from orbit.standard_runner import StandardTrainer -from orbit.standard_runner import StandardTrainerOptions +from orbit.runner import AbstractEvaluator, AbstractTrainer +from orbit.standard_runner import ( + StandardEvaluator, StandardEvaluatorOptions, StandardTrainer, + StandardTrainerOptions) diff --git a/orbit/controller.py b/orbit/controller.py index 9ecb2c544..03764e997 100755 --- a/orbit/controller.py +++ b/orbit/controller.py @@ -16,15 +16,12 @@ import pprint import time - from typing import Callable, Optional, Union +import tensorflow as tf from absl import logging -from orbit import runner -from orbit import utils - -import tensorflow as tf +from orbit import runner, utils def _log(message: str): diff --git a/orbit/controller_test.py b/orbit/controller_test.py index b4620b83b..35f71bbab 100755 --- a/orbit/controller_test.py +++ b/orbit/controller_test.py @@ -16,16 +16,12 @@ import os +import numpy as np +import tensorflow as tf from absl import logging from absl.testing import parameterized -import numpy as np - -from orbit import controller -from orbit import runner -from orbit import standard_runner - -import tensorflow as tf +from orbit import controller, runner, standard_runner def create_model(): diff --git a/orbit/runner.py b/orbit/runner.py index b0377c521..fbaed04d6 100755 --- a/orbit/runner.py +++ b/orbit/runner.py @@ -15,13 +15,11 @@ """Provides AbstractTrainer/Evaluator base classes, defining train/eval APIs.""" import abc - from typing import Dict, Optional, Union import numpy as np import tensorflow as tf - Output = Dict[str, Union[tf.Tensor, float, np.number, np.ndarray, 'Output']] # pytype: disable=not-supported-yet diff --git a/orbit/standard_runner.py b/orbit/standard_runner.py index a89e3ed2c..6da458920 100755 --- a/orbit/standard_runner.py +++ b/orbit/standard_runner.py @@ -34,16 +34,14 @@ """ import abc - +import dataclasses from typing import Any, Optional -import dataclasses +import tensorflow as tf from orbit import runner from orbit.utils import loop_fns -import tensorflow as tf - @dataclasses.dataclass(frozen=True) class StandardTrainerOptions: diff --git a/orbit/standard_runner_test.py b/orbit/standard_runner_test.py index e84ecb38e..20972d325 100755 --- a/orbit/standard_runner_test.py +++ b/orbit/standard_runner_test.py @@ -14,12 +14,10 @@ """Tests for orbit.standard_runner.""" +import tensorflow as tf from absl.testing import parameterized -from orbit import standard_runner -from orbit import utils - -import tensorflow as tf +from orbit import standard_runner, utils def dataset_fn(input_context=None): diff --git a/orbit/utils/__init__.py b/orbit/utils/__init__.py index 3eeb67c4a..197575cdb 100755 --- a/orbit/utils/__init__.py +++ b/orbit/utils/__init__.py @@ -14,16 +14,10 @@ """Defines exported symbols for the `orbit.utils` package.""" -from orbit.utils.common import create_global_step -from orbit.utils.common import get_value -from orbit.utils.common import make_distributed_dataset - +from orbit.utils.common import (create_global_step, get_value, + make_distributed_dataset) from orbit.utils.epoch_helper import EpochHelper - -from orbit.utils.loop_fns import create_loop_fn -from orbit.utils.loop_fns import create_tf_while_loop_fn -from orbit.utils.loop_fns import LoopFnWithSummaries - +from orbit.utils.loop_fns import (LoopFnWithSummaries, create_loop_fn, + create_tf_while_loop_fn) from orbit.utils.summary_manager import SummaryManager - from orbit.utils.tpu_summaries import OptionalSummariesFunction diff --git a/orbit/utils/common_test.py b/orbit/utils/common_test.py index 1a68e7c66..570d76b4c 100755 --- a/orbit/utils/common_test.py +++ b/orbit/utils/common_test.py @@ -14,10 +14,10 @@ """Tests for orbit.utils.common.""" -from orbit.utils import common - import tensorflow as tf +from orbit.utils import common + class UtilsTest(tf.test.TestCase): diff --git a/orbit/utils/loop_fns.py b/orbit/utils/loop_fns.py index 6e3262469..559485f37 100755 --- a/orbit/utils/loop_fns.py +++ b/orbit/utils/loop_fns.py @@ -14,10 +14,10 @@ """Utilities for creating loop functions.""" -from orbit.utils import tpu_summaries - import tensorflow as tf +from orbit.utils import tpu_summaries + def create_loop_fn(step_fn): """Creates a loop function driven by a Python `while` loop. diff --git a/orbit/utils/tpu_summaries_test.py b/orbit/utils/tpu_summaries_test.py index 4aa0d0820..43b352257 100755 --- a/orbit/utils/tpu_summaries_test.py +++ b/orbit/utils/tpu_summaries_test.py @@ -17,11 +17,10 @@ import functools import os -from orbit.utils import common -from orbit.utils import tpu_summaries - import tensorflow as tf +from orbit.utils import common, tpu_summaries + class TrainFunctionWithSummaries(tpu_summaries.OptionalSummariesFunction): """Implements a two-program approach for summaries on TPU.""" diff --git a/panoptic/data/create_cityscaped_tf_record.py b/panoptic/data/create_cityscaped_tf_record.py index 593f3557c..212cc5943 100755 --- a/panoptic/data/create_cityscaped_tf_record.py +++ b/panoptic/data/create_cityscaped_tf_record.py @@ -1,9 +1,11 @@ -import tensorflow as tf +import json import os + import cv2 -import json -from PIL import Image, ImageDraw import numpy as np +import pycocotools.mask as mask_utils +import tensorflow as tf +from PIL import Image, ImageDraw dataset_folder = "/media/vbanna/DATA_SHARE/CityScapes_raw" @@ -54,7 +56,6 @@ def _get_file_generator(image_folder, split): samples.append(get_file_lists(folder, file)) return samples -import pycocotools.mask as mask_utils def draw_polygons(): # mask_utils. return @@ -96,4 +97,4 @@ def get_instance_list(json_file): # print(polygon["label"]) # instance = load_instance(polygon["polygon"], json_polygons["imgWidth"], json_polygons["imgHeight"]) # plt.imshow(image_) -# plt.show() \ No newline at end of file +# plt.show() diff --git a/panoptic/dataloaders/decoders/coco_panoptic.py b/panoptic/dataloaders/decoders/coco_panoptic.py index d6d3b8599..15aec5eed 100755 --- a/panoptic/dataloaders/decoders/coco_panoptic.py +++ b/panoptic/dataloaders/decoders/coco_panoptic.py @@ -1,8 +1,8 @@ -import tensorflow_datasets as tfds -import tensorflow as tf import matplotlib.pyplot as plt -import utils.demos.utils as utils +import tensorflow as tf +import tensorflow_datasets as tfds +import utils.demos.utils as utils from panoptic.dataloaders.decoders import tfds_panoptic_coco_decoder path = "/media/vbanna/DATA_SHARE/tfds" @@ -30,4 +30,4 @@ plt.tight_layout() plt.show() if i > (lim + 1): - break \ No newline at end of file + break diff --git a/panoptic/dataloaders/decoders/tfds_panoptic_coco_decoder.py b/panoptic/dataloaders/decoders/tfds_panoptic_coco_decoder.py index dc9bf0ec9..2ad23d9dd 100755 --- a/panoptic/dataloaders/decoders/tfds_panoptic_coco_decoder.py +++ b/panoptic/dataloaders/decoders/tfds_panoptic_coco_decoder.py @@ -3,11 +3,13 @@ protos for object detection. """ import csv + # Import libraries import tensorflow as tf from official.vision.beta.dataloaders import decoder + def _generate_source_id(image_bytes): return tf.strings.as_string( tf.strings.to_hash_bucket_fast(image_bytes, 2**63 - 1)) diff --git a/panoptic/dataloaders/encoders/coco.py b/panoptic/dataloaders/encoders/coco.py index 87ae840f2..657b3d4b9 100755 --- a/panoptic/dataloaders/encoders/coco.py +++ b/panoptic/dataloaders/encoders/coco.py @@ -19,10 +19,9 @@ import json import os -from absl import logging import tensorflow.compat.v2 as tf - import tensorflow_datasets.public_api as tfds +from absl import logging _CITATION = """\ @article{DBLP:journals/corr/LinMBHPRDZ14, @@ -476,4 +475,4 @@ def get_annotations(self, img_id): AnnotationType.NONE: CocoAnnotation, AnnotationType.BBOXES: CocoAnnotationBBoxes, AnnotationType.PANOPTIC: CocoAnnotationPanoptic, -} \ No newline at end of file +} diff --git a/panoptic/dataloaders/encoders/coco_record_writer.py b/panoptic/dataloaders/encoders/coco_record_writer.py index 37dac3524..c0fb6fe66 100755 --- a/panoptic/dataloaders/encoders/coco_record_writer.py +++ b/panoptic/dataloaders/encoders/coco_record_writer.py @@ -4,13 +4,12 @@ import json import os -from absl import logging +import matplotlib.pyplot as plt +import numpy as np +import pycocotools.mask as mask_utils import tensorflow.compat.v2 as tf import tensorflow_datasets as tfds -import pycocotools.mask as mask_utils -import json -import matplotlib.pyplot as plt -import numpy as np +from absl import logging def reformat_dictionary(things_file, stuff_file, panoptic_file): diff --git a/panoptic/dataloaders/specs/tfds_load.py b/panoptic/dataloaders/specs/tfds_load.py index 539b1a2e3..02d47335c 100755 --- a/panoptic/dataloaders/specs/tfds_load.py +++ b/panoptic/dataloaders/specs/tfds_load.py @@ -4,4 +4,4 @@ dataset = "coco/2017_panoptic" train = tfds.load(name = dataset, download = True, data_dir = path, shuffle_files = True, split = "train") -print(train) \ No newline at end of file +print(train) diff --git a/panoptic/dataloaders/visual_test.py b/panoptic/dataloaders/visual_test.py index 636e5dc26..d2bc4282b 100755 --- a/panoptic/dataloaders/visual_test.py +++ b/panoptic/dataloaders/visual_test.py @@ -1,7 +1,9 @@ -import tensorflow_datasets as tfds import matplotlib.pyplot as plt +import tensorflow_datasets as tfds + from utils.demos import utils + """{image: (None, None, 3), image/filename: (), image/id: (), @@ -82,8 +84,3 @@ def __call__(self, sample): draw(sample) if i > 10: break - - - - - diff --git a/panoptic/train.py b/panoptic/train.py index b00fc5986..111646212 100755 --- a/panoptic/train.py +++ b/panoptic/train.py @@ -14,25 +14,24 @@ # limitations under the License. # ============================================================================== """TensorFlow Model Garden Vision training driver.""" +import gin +from absl import app, flags + +# pylint: enable=unused-import +from official.common import distribute_utils +from official.common import flags as tfm_flags +from official.core import task_factory, train_lib, train_utils +from official.modeling import performance +# pylint: disable=unused-import +from yolo.common import registry_imports from yolo.utils.run_utils import prep_gpu + try: prep_gpu() except BaseException: print('GPUs ready') -from absl import app -from absl import flags -import gin -from official.core import train_utils -# pylint: disable=unused-import -from yolo.common import registry_imports -# pylint: enable=unused-import -from official.common import distribute_utils -from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib -from official.modeling import performance FLAGS = flags.FLAGS """ diff --git a/pylint.sh b/pylint.sh new file mode 100644 index 000000000..bab33732a --- /dev/null +++ b/pylint.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# Pylint wrapper extracted from main TensorFlow, sharing same exceptions. +# As this is meant for smaller repos, drops "modified files" checking in favor +# of full-repo checking. + +set -euo pipefail + +# Download latest configs from main TensorFlow repo. +wget -q -O /tmp/pylintrc https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylintrc +wget -q -O /tmp/pylint_allowlist https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylint_allowlist + +SCRIPT_DIR=/tmp + +num_cpus() { + # Get the number of CPUs + if [[ -f /proc/cpuinfo ]]; then + N_CPUS=$(grep -c ^processor /proc/cpuinfo) + else + # Fallback method + N_CPUS=`getconf _NPROCESSORS_ONLN` + fi + if [[ -z ${N_CPUS} ]]; then + die "ERROR: Unable to determine the number of CPUs" + fi + + echo ${N_CPUS} +} + +get_py_files_to_check() { + find . -name '*.py' +} + +do_pylint() { + # Get all Python files, regardless of mode. + PYTHON_SRC_FILES=$(get_py_files_to_check) + + # Something happened. TF no longer has Python code if this branch is taken + if [[ -z ${PYTHON_SRC_FILES} ]]; then + echo "do_pylint found no Python files to check. Returning." + return 0 + fi + + # Now that we know we have to do work, check if `pylint` is installed + # PYLINT_BIN="python3.8 -m pylint" + PYLINT_BIN="python3 -m pylint" + + echo "" + echo "check whether pylint is available or not." + echo "" + ${PYLINT_BIN} --version + if [[ $? -eq 0 ]] + then + echo "" + echo "pylint available, proceeding with pylint sanity check." + echo "" + else + echo "" + echo "pylint not available." + echo "" + return 1 + fi + + # Configure pylint using the following file + PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc" + + if [[ ! -f "${PYLINTRC_FILE}" ]]; then + die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}" + fi + + # Run pylint in parallel, after some disk setup + NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w) + NUM_CPUS=$(num_cpus) + + echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\ +"parallel jobs..." + echo "" + + PYLINT_START_TIME=$(date +'%s') + OUTPUT_FILE="$(mktemp)_pylint_output.log" + ERRORS_FILE="$(mktemp)_pylint_errors.log" + PERMIT_FILE="$(mktemp)_pylint_permit.log" + FORBID_FILE="$(mktemp)_pylint_forbid.log" + + rm -rf ${OUTPUT_FILE} + rm -rf ${ERRORS_FILE} + rm -rf ${PERMIT_FILE} + rm -rf ${FORBID_FILE} + + set +e + # When running, filter to only contain the error code lines. Removes module + # header, removes lines of context that show up from some lines. + # Also, don't redirect stderr as this would hide pylint fatal errors. + ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \ + --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} | grep '\[[CEFW]' > ${OUTPUT_FILE} + PYLINT_END_TIME=$(date +'%s') + + echo "" + echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s" + echo "" + + # Report only what we care about + # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html + # E: all errors + # W0311 bad-indentation + # W0312 mixed-indentation + # C0330 bad-continuation + # C0301 line-too-long + # C0326 bad-whitespace + # W0611 unused-import + # W0622 redefined-builtin + grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE} + + # Split the pylint reported errors into permitted ones and those we want to + # block submit on until fixed. + # We use `${ALLOW_LIST_FILE}` to record the errors we temporarily accept. Goal + # is to make that file only contain errors caused by difference between + # internal and external versions. + ALLOW_LIST_FILE="${SCRIPT_DIR}/pylint_allowlist" + + if [[ ! -f "${ALLOW_LIST_FILE}" ]]; then + die "ERROR: Cannot find pylint allowlist file at ${ALLOW_LIST_FILE}" + fi + + # We can split with just 2 grep invocations + grep -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${PERMIT_FILE} + grep -v -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${FORBID_FILE} + + # Determine counts of errors + N_PERMIT_ERRORS=$(wc -l ${PERMIT_FILE} | cut -d' ' -f1) + N_FORBID_ERRORS=$(wc -l ${FORBID_FILE} | cut -d' ' -f1) + set -e + + # First print all allowed errors + echo "" + if [[ ${N_PERMIT_ERRORS} != 0 ]]; then + echo "Found ${N_PERMIT_ERRORS} allowlisted pylint errors:" + cat ${PERMIT_FILE} + fi + + # Now, print the errors we should fix + echo "" + if [[ ${N_FORBID_ERRORS} != 0 ]]; then + echo "Found ${N_FORBID_ERRORS} non-allowlisted pylint errors:" + cat ${FORBID_FILE} + fi + + echo "" + if [[ ${N_FORBID_ERRORS} != 0 ]]; then + echo "FAIL: Found ${N_FORBID_ERRORS} non-allowlisted errors and ${N_PERMIT_ERRORS} allowlisted errors" + return 1 + else + echo "PASS: Found only ${N_PERMIT_ERRORS} allowlisted errors" + fi +} + +do_pylint \ No newline at end of file diff --git a/pylintrc b/pylintrc index 88750048c..5e643542f 100755 --- a/pylintrc +++ b/pylintrc @@ -1,3 +1,12 @@ +# NOTE: this pylintrc file is a combination of 3 pylint files. The source +# files are denoted in the comments as the following: +# { From [1] } : TensorFlow +# https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylintrc +# { From [2] } : TensorFlow Model Garden +# https://raw.githubusercontent.com/tensorflow/models/master/official/utils/testing/pylint.rcfile +# { From [3] } : Google Style Guide +# https://google.github.io/styleguide/pylintrc + # This Pylint rcfile contains a best-effort configuration to uphold the # best-practices and style described in the Google Python style guide: # https://google.github.io/styleguide/pyguide.html @@ -7,20 +16,24 @@ [MASTER] -# Add files or directories to the blacklist. They should be base names, not -# paths. +# Files or directories to be skipped. They should be base names, not paths. +# { From [1] } +# ignore=CVS ignore=third_party -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. ignore-patterns= # Pickle collected data for later comparisons. -persistent=no +persistent=yes # { From [1] } +# persistent=no # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. -load-plugins= +# { From [1] } +load-plugins=pylint.extensions.docparams +accept-no-param-doc=no # Use multiple processes to speed up Pylint. jobs=4 @@ -29,10 +42,8 @@ jobs=4 # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= +# Profiled execution. { From [1] } +profile=no [MESSAGES CONTROL] @@ -45,7 +56,8 @@ confidence= # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. -#enable= +# { From [1] } +enable=indexing-exception,old-raise-syntax # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this @@ -56,103 +68,113 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=abstract-method, - apply-builtin, - arguments-differ, - attribute-defined-outside-init, - backtick, - bad-option-value, - basestring-builtin, - buffer-builtin, - c-extension-no-member, - consider-using-enumerate, - cmp-builtin, - cmp-method, - coerce-builtin, - coerce-method, - delslice-method, - div-method, - duplicate-code, - eq-without-hash, - execfile-builtin, - file-builtin, - filter-builtin-not-iterating, - fixme, - getslice-method, - global-statement, - hex-method, - idiv-method, - implicit-str-concat-in-sequence, - import-error, - import-self, - import-star-module-level, - inconsistent-return-statements, - input-builtin, - intern-builtin, - invalid-str-codec, - locally-disabled, - long-builtin, - long-suffix, - map-builtin-not-iterating, - misplaced-comparison-constant, - missing-function-docstring, - metaclass-assignment, - next-method-called, - next-method-defined, - no-absolute-import, - no-else-break, - no-else-continue, - no-else-raise, - no-else-return, - no-init, # added - no-member, - no-name-in-module, - no-self-use, - nonzero-method, - oct-method, - old-division, - old-ne-operator, - old-octal-literal, - old-raise-syntax, - parameter-unpacking, - print-statement, - raising-string, - range-builtin-not-iterating, - raw_input-builtin, - rdiv-method, - reduce-builtin, - relative-import, - reload-builtin, - round-builtin, - setslice-method, - signature-differs, - standarderror-builtin, - suppressed-message, - sys-max-int, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-boolean-expressions, - too-many-branches, - too-many-instance-attributes, - too-many-locals, - too-many-nested-blocks, - too-many-public-methods, - too-many-return-statements, - too-many-statements, - trailing-newlines, - unichr-builtin, - unicode-builtin, - unnecessary-pass, - unpacking-in-except, - useless-else-on-loop, - useless-object-inheritance, - useless-suppression, - using-cmp-argument, - wrong-import-order, - xrange-builtin, - zip-builtin-not-iterating, +# { From [1] } +disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,no-member,no-name-in-module,import-error,unsubscriptable-object,unbalanced-tuple-unpacking,undefined-variable,not-context-manager,invalid-sequence-index +# { From [2] } +#disable=R,W,bad-option-value,trailing-newlines,no-name-in-module + +# { From [3] } +#disable=abstract-method, +# apply-builtin, +# arguments-differ, +# attribute-defined-outside-init, +# backtick, +# bad-option-value, +# basestring-builtin, +# buffer-builtin, +# c-extension-no-member, +# consider-using-enumerate, +# cmp-builtin, +# cmp-method, +# coerce-builtin, +# coerce-method, +# delslice-method, +# div-method, +# duplicate-code, +# eq-without-hash, +# execfile-builtin, +# file-builtin, +# filter-builtin-not-iterating, +# fixme, +# getslice-method, +# global-statement, +# hex-method, +# idiv-method, +# implicit-str-concat-in-sequence, +# import-error, +# import-self, +# import-star-module-level, +# inconsistent-return-statements, +# input-builtin, +# intern-builtin, +# invalid-str-codec, +# locally-disabled, +# long-builtin, +# long-suffix, +# map-builtin-not-iterating, +# misplaced-comparison-constant, +# missing-function-docstring, +# metaclass-assignment, +# next-method-called, +# next-method-defined, +# no-absolute-import, +# no-else-break, +# no-else-continue, +# no-else-raise, +# no-else-return, +# no-init, # added +# no-member, +# no-name-in-module, +# no-self-use, +# nonzero-method, +# oct-method, +# old-division, +# old-ne-operator, +# old-octal-literal, +# old-raise-syntax, +# parameter-unpacking, +# print-statement, +# raising-string, +# range-builtin-not-iterating, +# raw_input-builtin, +# rdiv-method, +# reduce-builtin, +# relative-import, +# reload-builtin, +# round-builtin, +# setslice-method, +# signature-differs, +# standarderror-builtin, +# suppressed-message, +# sys-max-int, +# too-few-public-methods, +# too-many-ancestors, +# too-many-arguments, +# too-many-boolean-expressions, +# too-many-branches, +# too-many-instance-attributes, +# too-many-locals, +# too-many-nested-blocks, +# too-many-public-methods, +# too-many-return-statements, +# too-many-statements, +# trailing-newlines, +# unichr-builtin, +# unicode-builtin, +# unnecessary-pass, +# unpacking-in-except, +# useless-else-on-loop, +# useless-object-inheritance, +# useless-suppression, +# using-cmp-argument, +# wrong-import-order, +# xrange-builtin, +# zip-builtin-not-iterating, + +# Set the cache size for astng objects. +# { From [1] } +cache-size=500 [REPORTS] @@ -177,13 +199,29 @@ reports=no # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +# { From [1] } +comment=no + # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details #msg-template= +# Activate the evaluation score. +score=no + [BASIC] +# List of builtins function names that should not be used, separated by a comma { Frpm tensorflow/master/tensorflow/tools } +bad-functions=apply,input,reduce + +# Disable the report(s) with the given id(s). +# All non-Google reports are disabled by default. +# { From [1] } +disable-report=R0001,R0002,R0003,R0004,R0101,R0102,R0201,R0202,R0220,R0401,R0402,R0701,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,R0923 + # Good variable names which should always be accepted, separated by a comma good-names=main,_ @@ -202,7 +240,12 @@ include-naming-hint=no property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl # Regular expression matching correct function names -function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ +# { From [1] } +# function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ +# { From [2] } +function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ +# { From [3] } +#function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ # Regular expression matching correct variable names variable-rgx=^[a-z][a-z0-9_]*$ @@ -226,19 +269,37 @@ inlinevar-rgx=^[a-z][a-z0-9_]*$ class-rgx=^_?[A-Z][a-zA-Z0-9]*$ # Regular expression matching correct module names -module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ +# { From [1] } +# module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# { From [2] } +module-rgx=^(_?[a-z][a-z0-9_]*)|__init__|PRESUBMIT|PRESUBMIT_unittest$ +# { From [3] } +#module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ # Regular expression matching correct method names -method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ +# { From [1] } +# method-rgx=^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ +# { From [2] } +method-rgx=^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*)|(setUp|tearDown))$ +# { From [3] } +#method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ # Regular expression which should only match function or class names that do # not require a docstring. -no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ +# { From [1] } +# no-docstring-rgx=(__.*__|main) +# { From [2] } +no-docstring-rgx=(__.*__|main|.*ArgParser) +# { From [3] } +#no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=10 +# Naming hint for variable names { From tensorflow/models/master/official } +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + [TYPECHECK] @@ -255,17 +316,28 @@ ignore-mixin-members=yes # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= + +# { From [2] } +ignored-modules=absl, absl.*, official, official.*, tensorflow, tensorflow.*, LazyLoader, google, google.cloud.* # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local +# { From [1] } +# ignored-classes=SQLObject +# { From [3] } +ignored-classes=optparse.Values,thread._local,_thread._local, SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +# { From [1] } +zope=no # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# expressions are accepted. +# { From [1] } +generated-members=REQUEST,acl_users,aq_parent [FORMAT] @@ -277,9 +349,26 @@ max-line-length=80 # lines made too long by directives to pytype. # Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=(?x)( - ^\s*(\#\ )??$| - ^\s*(from\s+\S+\s+)?import\s+.+$) +# { From [2] } +# ignore-long-lines=^\s*(?:(# )??$|# type:) + +# { From [1] } +ignore-long-lines=(?x) + (^\s*(import|from)\s + |\$Id:\s\/\/depot\/.+#\d+\s\$ + |^[a-zA-Z_][a-zA-Z0-9_]*\s*=\s*("[^"]\S+"|'[^']\S+') + |^\s*\#\ LINT\.ThenChange + |^[^#]*\#\ type:\ [a-zA-Z_][a-zA-Z0-9_.,[\] ]*$ + |pylint + |""" + |\# + |lambda + |(https?|ftp):) + +# { From [3] } +#ignore-long-lines=(?x)( +# ^\s*(\#\ )??$| +# ^\s*(from\s+\S+\s+)?import\s+.+$) # Allow the body of an if to be on the same line as the test if there is no # else. @@ -416,6 +505,10 @@ analyse-fallback-blocks=no [CLASSES] +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +# { From [1] } +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, @@ -437,6 +530,39 @@ valid-classmethod-first-arg=cls, # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs +[DESIGN] +# { From [2] } --> entire section + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 [EXCEPTIONS] @@ -445,3 +571,32 @@ valid-metaclass-classmethod-first-arg=mcs overgeneral-exceptions=StandardError, Exception, BaseException + + + +[AST] +# { From [1] } --> entire section + +# Maximum line length for lambdas +short-func-length=1 + +# List of module members that should be marked as deprecated. +# All of the string functions are listed in 4.1.4 Deprecated string functions +# in the Python 2.4 docs. +deprecated-members=string.atof,string.atoi,string.atol,string.capitalize,string.expandtabs,string.find,string.rfind,string.index,string.rindex,string.count,string.lower,string.split,string.rsplit,string.splitfields,string.join,string.joinfields,string.lstrip,string.rstrip,string.strip,string.swapcase,string.translate,string.upper,string.ljust,string.rjust,string.center,string.zfill,string.replace,sys.exitfunc + + +[DOCSTRING] +# { From [1] } --> entire section +default-docstring-type=google +# List of exceptions that do not need to be mentioned in the Raises section of +# a docstring. +ignore-exceptions=AssertionError,NotImplementedError,StopIteration,TypeError + + + +[GOOGLE LINES] +# { From [1] } --> entire section + +# Regexp for a proper copyright notice. +copyright=Copyright \d{4} The TensorFlow Authors\. +All [Rr]ights [Rr]eserved\. \ No newline at end of file diff --git a/test_folder_linter/test_file.py b/test_folder_linter/test_file.py index ecae67987..9c15b3a62 100755 --- a/test_folder_linter/test_file.py +++ b/test_folder_linter/test_file.py @@ -1,13 +1,12 @@ +import numpy as np +import pandas as pd +from sklearn import preprocessing from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score -from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split -from sklearn import preprocessing -import numpy as np -import pandas as pd +from sklearn.preprocessing import StandardScaler -from NFold_GridSearch import NFold -from NFold_GridSearch import GridSearch +from NFold_GridSearch import GridSearch, NFold np.set_printoptions(threshold=np.inf) diff --git a/utils/demos/coco.py b/utils/demos/coco.py index 0a541be77..e3be9428e 100755 --- a/utils/demos/coco.py +++ b/utils/demos/coco.py @@ -1,7 +1,7 @@ -import cv2 import colorsys -import numpy as np +import cv2 +import numpy as np import tensorflow as tf import tensorflow.keras.backend as K diff --git a/utils/demos/coco_dataset_converter.py b/utils/demos/coco_dataset_converter.py index f0f780505..a25fff57d 100755 --- a/utils/demos/coco_dataset_converter.py +++ b/utils/demos/coco_dataset_converter.py @@ -1,9 +1,10 @@ +import io +import json +import os + import tensorflow as tf from PIL import Image from tqdm import tqdm -import io -import os -import json def encode_image(image, fmt='JPEG'): diff --git a/utils/demos/utils.py b/utils/demos/utils.py index ee46ec100..f08fc6cdd 100755 --- a/utils/demos/utils.py +++ b/utils/demos/utils.py @@ -1,14 +1,15 @@ -import tensorflow as tf -import tensorflow.keras.backend as K +import colorsys +import multiprocessing as mp +import os import socket import struct +from contextlib import closing from typing import Callable -import numpy as np -import colorsys + import cv2 -from contextlib import closing -import multiprocessing as mp -import os +import numpy as np +import tensorflow as tf +import tensorflow.keras.backend as K def get_device(policy): diff --git a/utils/downloads/__init__.py b/utils/downloads/__init__.py index 8b1378917..e69de29bb 100755 --- a/utils/downloads/__init__.py +++ b/utils/downloads/__init__.py @@ -1 +0,0 @@ - diff --git a/utils/downloads/file_manager.py b/utils/downloads/file_manager.py index 492644194..aee14cc51 100755 --- a/utils/downloads/file_manager.py +++ b/utils/downloads/file_manager.py @@ -6,7 +6,6 @@ import io import os - from typing import Union # define PathABC type diff --git a/utils/export/get_cfg_yaml.py b/utils/export/get_cfg_yaml.py index bd09487c6..ccb9cf409 100755 --- a/utils/export/get_cfg_yaml.py +++ b/utils/export/get_cfg_yaml.py @@ -1,6 +1,7 @@ -from yolo.configs import yolo as exp_cfg import yaml +from yolo.configs import yolo as exp_cfg + config = exp_cfg.yolo_v4_coco() e = yaml.dump(config.as_dict(), default_flow_style=False) diff --git a/utils/export/tensor_rt.py b/utils/export/tensor_rt.py index 29980cdbe..0f710190d 100755 --- a/utils/export/tensor_rt.py +++ b/utils/export/tensor_rt.py @@ -1,10 +1,12 @@ import tensorflow as tf -# import tensorflow.experimental.tensorrt as trt - from tensorflow.python.compiler.tensorrt import trt_convert as trt from utils.run_utils import prep_gpu +# import tensorflow.experimental.tensorrt as trt + + + class TensorRT(object): diff --git a/utils/register.py b/utils/register.py index b253c94b8..80eb98e97 100644 --- a/utils/register.py +++ b/utils/register.py @@ -5,14 +5,12 @@ import warnings from typing import ClassVar, Union +from official.core import exp_factory, registry, task_factory +from official.modeling import hyperparams from official.vision.beta.configs import backbones, backbones_3d -from official.vision.beta.modeling.backbones import factory as backbones_factory from official.vision.beta.modeling import factory_3d as models_3d_factory -from official.core import task_factory -from official.core import exp_factory - -from official.core import registry -from official.modeling import hyperparams +from official.vision.beta.modeling.backbones import \ + factory as backbones_factory def _deduce_type(fn, config_param=1): diff --git a/utils/training/keras_task_trainer.py b/utils/training/keras_task_trainer.py index 376386697..ca54edc42 100755 --- a/utils/training/keras_task_trainer.py +++ b/utils/training/keras_task_trainer.py @@ -1,9 +1,10 @@ import tensorflow as tf +from tensorflow.keras.mixed_precision import experimental as mixed_precision + import official.core.base_task as bt from official.core import input_reader from yolo.dataloaders import yolo_input from yolo.dataloaders.decoders import tfds_coco_decoder -from tensorflow.keras.mixed_precision import experimental as mixed_precision class Trainer(tf.keras.Model): diff --git a/yolo/common/registry_imports.py b/yolo/common/registry_imports.py index 41bf498f5..0c16d420a 100755 --- a/yolo/common/registry_imports.py +++ b/yolo/common/registry_imports.py @@ -14,18 +14,14 @@ # ============================================================================== """All necessary imports for registration.""" +import yolo # pylint: disable=unused-import from official.common import registry_imports - -import yolo +from yolo.configs import darknet_classification, yolo +from yolo.configs.darknet_classification import (ImageClassificationTask, + image_classification) +from yolo.configs.yolo import YoloTask, yolo_custom from yolo.modeling.backbones import darknet -from yolo.configs import darknet_classification -from yolo.configs import yolo -from yolo.configs.yolo import yolo_custom -from yolo.configs.darknet_classification import image_classification -from yolo.configs.darknet_classification import ImageClassificationTask -from yolo.configs.yolo import YoloTask - from yolo.tasks.image_classification import ImageClassificationTask from yolo.tasks.yolo import YoloTask from yolo.tasks.yolo_subdiv import YoloSubDivTask diff --git a/yolo/common/train_lib.py b/yolo/common/train_lib.py index 46820ee85..8201df046 100755 --- a/yolo/common/train_lib.py +++ b/yolo/common/train_lib.py @@ -16,14 +16,12 @@ import os from typing import Any, Mapping, Tuple +import tensorflow as tf # Import libraries from absl import logging -import orbit -import tensorflow as tf -from official.core import base_task -from official.core import config_definitions -from official.core import train_utils +import orbit +from official.core import base_task, config_definitions, train_utils BestCheckpointExporter = train_utils.BestCheckpointExporter diff --git a/yolo/configs/backbones.py b/yolo/configs/backbones.py index 194547993..698cde27e 100755 --- a/yolo/configs/backbones.py +++ b/yolo/configs/backbones.py @@ -2,6 +2,7 @@ # Import libraries import dataclasses from typing import Optional + from official.modeling import hyperparams from official.vision.beta.configs import backbones diff --git a/yolo/configs/darknet_classification.py b/yolo/configs/darknet_classification.py index a76a76ba2..f686383c5 100755 --- a/yolo/configs/darknet_classification.py +++ b/yolo/configs/darknet_classification.py @@ -1,12 +1,11 @@ +import dataclasses import os from typing import List, Optional, Tuple -import dataclasses + from official.core import config_definitions as cfg from official.core import exp_factory -from official.modeling import hyperparams -from official.modeling import optimization +from official.modeling import hyperparams, optimization from official.vision.beta.configs import common - from yolo.configs import backbones diff --git a/yolo/configs/yolo.py b/yolo/configs/yolo.py index 886218529..befc41544 100755 --- a/yolo/configs/yolo.py +++ b/yolo/configs/yolo.py @@ -14,17 +14,16 @@ # limitations under the License. # ============================================================================== """YOLO configuration definition.""" -import tensorflow as tf -from typing import ClassVar, Dict, List, Optional, Tuple, Union import dataclasses import os +from typing import ClassVar, Dict, List, Optional, Tuple, Union + +import tensorflow as tf from official.core import exp_factory -from official.modeling import hyperparams -from official.modeling import optimization +from official.modeling import hyperparams, optimization from official.modeling.hyperparams import config_definitions as cfg from official.vision.beta.configs import common - from yolo.configs import backbones COCO_INPUT_PATH_BASE = 'coco' diff --git a/yolo/dataloaders/classification_input.py b/yolo/dataloaders/classification_input.py index e19585478..51c9d6ff2 100755 --- a/yolo/dataloaders/classification_input.py +++ b/yolo/dataloaders/classification_input.py @@ -2,8 +2,8 @@ # Import libraries import tensorflow as tf -import tensorflow_datasets as tfds import tensorflow_addons as tfa +import tensorflow_datasets as tfds from official.vision.beta.dataloaders import parser from official.vision.beta.ops import preprocess_ops diff --git a/yolo/dataloaders/classification_vision.py b/yolo/dataloaders/classification_vision.py index f443dc1d2..e17f17f2c 100755 --- a/yolo/dataloaders/classification_vision.py +++ b/yolo/dataloaders/classification_vision.py @@ -2,12 +2,11 @@ # Import libraries import tensorflow as tf -import tensorflow_datasets as tfds import tensorflow_addons as tfa +import tensorflow_datasets as tfds from official.vision.beta.dataloaders import parser -from official.vision.beta.ops import preprocess_ops -from official.vision.beta.ops import augment +from official.vision.beta.ops import augment, preprocess_ops class Parser(parser.Parser): diff --git a/yolo/dataloaders/decoders/classification_tfds_decoder.py b/yolo/dataloaders/decoders/classification_tfds_decoder.py index fabb6abe4..411831cb2 100755 --- a/yolo/dataloaders/decoders/classification_tfds_decoder.py +++ b/yolo/dataloaders/decoders/classification_tfds_decoder.py @@ -16,8 +16,7 @@ # Import libraries import tensorflow as tf -from official.vision.beta.dataloaders import decoder -from official.vision.beta.dataloaders import parser +from official.vision.beta.dataloaders import decoder, parser from official.vision.beta.ops import preprocess_ops diff --git a/yolo/dataloaders/decoders/tfds_coco_decoder.py b/yolo/dataloaders/decoders/tfds_coco_decoder.py index 67aa4a345..26b9a025b 100755 --- a/yolo/dataloaders/decoders/tfds_coco_decoder.py +++ b/yolo/dataloaders/decoders/tfds_coco_decoder.py @@ -3,6 +3,7 @@ protos for object detection. """ import csv + # Import libraries import tensorflow as tf diff --git a/yolo/dataloaders/priming_input.py b/yolo/dataloaders/priming_input.py index 77ac54b63..cecdf96db 100755 --- a/yolo/dataloaders/priming_input.py +++ b/yolo/dataloaders/priming_input.py @@ -2,8 +2,8 @@ # Import libraries import tensorflow as tf -import tensorflow_datasets as tfds import tensorflow_addons as tfa +import tensorflow_datasets as tfds from official.vision.beta.dataloaders import parser diff --git a/yolo/dataloaders/yolo_input.py b/yolo/dataloaders/yolo_input.py index bc55d6557..965d5678c 100755 --- a/yolo/dataloaders/yolo_input.py +++ b/yolo/dataloaders/yolo_input.py @@ -7,11 +7,11 @@ import tensorflow as tf import tensorflow_addons as tfa -from yolo.ops import preprocessing_ops -from yolo.ops import box_ops as box_utils -from official.vision.beta.ops import box_ops, preprocess_ops from official.vision.beta.dataloaders import parser, utils +from official.vision.beta.ops import box_ops, preprocess_ops +from yolo.ops import box_ops as box_utils from yolo.ops import loss_utils as loss_ops +from yolo.ops import preprocessing_ops def pad_max_instances(value, instances, pad_value=0, pad_axis=0): diff --git a/yolo/dataloaders/yolo_input_test.py b/yolo/dataloaders/yolo_input_test.py index ae4f6c471..2b69ff296 100755 --- a/yolo/dataloaders/yolo_input_test.py +++ b/yolo/dataloaders/yolo_input_test.py @@ -1,18 +1,21 @@ -from yolo.tasks import image_classification as imc -from yolo.utils.demos import utils, coco -from yolo.tasks import yolo -from yolo.configs import yolo as yolocfg -from yolo.configs import darknet_classification as dcfg +import dataclasses +import time + +import matplotlib.pyplot as plt +import tensorflow as tf + +from official.core import config_definitions as cfg from official.core import input_reader -from yolo.dataloaders import yolo_input as YOLO_Detection_Input +from official.modeling import hyperparams +from yolo.configs import darknet_classification as dcfg +from yolo.configs import yolo as yolocfg from yolo.dataloaders import classification_input +from yolo.dataloaders import yolo_input as YOLO_Detection_Input from yolo.dataloaders.decoders import tfds_coco_decoder from yolo.ops import box_ops -import matplotlib.pyplot as plt -import dataclasses -from official.modeling import hyperparams -from official.core import config_definitions as cfg -import tensorflow as tf +from yolo.tasks import image_classification as imc +from yolo.tasks import yolo +from yolo.utils.demos import coco, utils def test_yolo_input_task(): @@ -63,7 +66,6 @@ def test_classification_pipeline(): return -import time def test_pipeline(): diff --git a/yolo/demos/demo_lib/readers.py b/yolo/demos/demo_lib/readers.py index b365f9cbc..9bb2f4cfb 100755 --- a/yolo/demos/demo_lib/readers.py +++ b/yolo/demos/demo_lib/readers.py @@ -1,8 +1,9 @@ +import abc import threading as t -from queue import Queue import time +from queue import Queue + import cv2 -import abc class Reader(object): diff --git a/yolo/demos/demo_lib/video_detect_gpu.py b/yolo/demos/demo_lib/video_detect_gpu.py index f1669e5f7..ff9f981fb 100755 --- a/yolo/demos/demo_lib/video_detect_gpu.py +++ b/yolo/demos/demo_lib/video_detect_gpu.py @@ -1,27 +1,22 @@ -import cv2 -import datetime import colorsys -import numpy as np -import time - +import datetime import threading as t +import time from queue import Queue +import cv2 +import numpy as np import tensorflow as tf import tensorflow.keras as ks import tensorflow.keras.backend as K -from yolo.utils.run_utils import support_windows -from yolo.utils.demos.coco import draw_box -from yolo.utils.demos.coco import get_draw_fn -from yolo.utils.demos.coco import gen_colors -from yolo.utils.demos.coco import get_coco_names -from yolo.utils.demos.coco import int_scale_boxes -from yolo.utils.demos import utils -# from utils.demos import utils -from yolo.utils.run_utils import prep_gpu from yolo.configs import yolo as exp_cfg from yolo.tasks.yolo import YoloTask +from yolo.utils.demos import utils +from yolo.utils.demos.coco import (draw_box, gen_colors, get_coco_names, + get_draw_fn, int_scale_boxes) +# from utils.demos import utils +from yolo.utils.run_utils import prep_gpu, support_windows class FastVideo(object): diff --git a/yolo/demos/demo_lib/writer.py b/yolo/demos/demo_lib/writer.py index 95cd9ffd3..9ec911c43 100755 --- a/yolo/demos/demo_lib/writer.py +++ b/yolo/demos/demo_lib/writer.py @@ -1,8 +1,9 @@ +import abc import threading as t -from queue import Queue import time +from queue import Queue + import cv2 -import abc class Writer(object): diff --git a/yolo/demos/examples/server/app.py b/yolo/demos/examples/server/app.py index abc2a372c..48eb36e0b 100755 --- a/yolo/demos/examples/server/app.py +++ b/yolo/demos/examples/server/app.py @@ -1,24 +1,26 @@ +import base64 +import sys +import urllib.request +from queue import Queue + +import cv2 +import numpy as np +import tensorflow as tf + +from flask import Flask, jsonify, request +from yolo.configs import yolo as exp_cfg +from yolo.demos.three_servers import model_server as ms +from yolo.tasks.yolo import YoloTask +from yolo.utils.demos import coco, utils from yolo.utils.run_utils import prep_gpu + try: prep_gpu() except BaseException: print("GPU's already prepped") -from flask import Flask, request, jsonify -import numpy as np -import base64 -import cv2 -import sys -import tensorflow as tf -from yolo.demos.three_servers import model_server as ms -from yolo.configs import yolo as exp_cfg -from yolo.tasks.yolo import YoloTask -from yolo.utils.demos import utils -from yolo.utils.demos import coco -from queue import Queue -import urllib.request app = Flask(__name__) diff --git a/yolo/demos/model_val.py b/yolo/demos/model_val.py index c77dc434f..4ee1c18cc 100755 --- a/yolo/demos/model_val.py +++ b/yolo/demos/model_val.py @@ -1,8 +1,8 @@ import tensorflow as tf -from yolo.utils.run_utils import prep_gpu from yolo.configs import yolo as exp_cfg from yolo.tasks.yolo import YoloTask +from yolo.utils.run_utils import prep_gpu if __name__ == "__main__": # initialize YOLOv4 model diff --git a/yolo/demos/routed_webcam.py b/yolo/demos/routed_webcam.py index 74cbe0fa2..c826dfb69 100755 --- a/yolo/demos/routed_webcam.py +++ b/yolo/demos/routed_webcam.py @@ -1,25 +1,19 @@ -import cv2 -import datetime import colorsys -import numpy as np -import time - +import datetime import threading as t +import time from queue import Queue +import cv2 +import numpy as np import tensorflow as tf import tensorflow.keras as ks import tensorflow.keras.backend as K -from yolo.utils.run_utils import support_windows -from yolo.utils.run_utils import prep_gpu -from yolo.utils.demos.coco import draw_box -from yolo.utils.demos.coco import get_draw_fn -from yolo.utils.demos.coco import gen_colors -from yolo.utils.demos.coco import get_coco_names -from yolo.utils.demos.coco import int_scale_boxes - import pyfakewebcam +from yolo.utils.demos.coco import (draw_box, gen_colors, get_coco_names, + get_draw_fn, int_scale_boxes) +from yolo.utils.run_utils import prep_gpu, support_windows class FastVideo(object): diff --git a/yolo/demos/three_servers/model_server.py b/yolo/demos/three_servers/model_server.py index d22a1544a..c660aa083 100755 --- a/yolo/demos/three_servers/model_server.py +++ b/yolo/demos/three_servers/model_server.py @@ -1,23 +1,20 @@ -import yolo.demos.three_servers.video_server as video_t -import struct -import cv2 -import datetime import colorsys -import numpy as np -import time - +import datetime +import struct import threading as t +import time +import traceback from queue import Queue +import cv2 +import numpy as np import tensorflow as tf import tensorflow.keras as ks import tensorflow.keras.backend as K -from yolo.utils.run_utils import support_windows -from yolo.utils.run_utils import prep_gpu -from yolo.utils.demos import utils -from yolo.utils.demos import coco -import traceback +import yolo.demos.three_servers.video_server as video_t +from yolo.utils.demos import coco, utils +from yolo.utils.run_utils import prep_gpu, support_windows class ModelServer(object): diff --git a/yolo/demos/three_servers/udp_server.py b/yolo/demos/three_servers/udp_server.py index 92975f9b9..0cfc89f91 100755 --- a/yolo/demos/three_servers/udp_server.py +++ b/yolo/demos/three_servers/udp_server.py @@ -1,9 +1,11 @@ import socket -from yolo.utils.demos import utils -from queue import Queue +import struct import threading as t +from queue import Queue + import numpy as np -import struct + +from yolo.utils.demos import utils class UDPServer(object): diff --git a/yolo/demos/three_servers/video_server.py b/yolo/demos/three_servers/video_server.py index 3cfd8246e..5d34cbf8e 100755 --- a/yolo/demos/three_servers/video_server.py +++ b/yolo/demos/three_servers/video_server.py @@ -1,8 +1,9 @@ -import cv2 import threading as t -from queue import Queue -import traceback import time +import traceback +from queue import Queue + +import cv2 from yolo.demos.three_servers.frame_que import FrameQue from yolo.utils.demos import utils diff --git a/yolo/demos/video_detect_cpu.py b/yolo/demos/video_detect_cpu.py index 2f78191b0..e2ff5a4ea 100755 --- a/yolo/demos/video_detect_cpu.py +++ b/yolo/demos/video_detect_cpu.py @@ -1,9 +1,9 @@ #### ARGUMENT PARSER #### -from yolo.utils.demos import utils -from yolo.utils.demos import coco -from yolo.demos.three_servers import video_server as vs -import tensorflow as tf import cv2 +import tensorflow as tf + +from yolo.demos.three_servers import video_server as vs +from yolo.utils.demos import coco, utils def print_opt(latency, fps): diff --git a/yolo/demos/video_detect_gpu.py b/yolo/demos/video_detect_gpu.py index 28f82995f..b35130561 100755 --- a/yolo/demos/video_detect_gpu.py +++ b/yolo/demos/video_detect_gpu.py @@ -1,28 +1,23 @@ -import cv2 -import datetime import colorsys -import numpy as np -import time - +import datetime import threading as t +import time from queue import Queue +import cv2 +import numpy as np import tensorflow as tf import tensorflow.keras as ks import tensorflow.keras.backend as K -from yolo.utils.run_utils import support_windows -from yolo.utils.demos.coco import draw_box -from yolo.utils.demos.coco import get_draw_fn -from yolo.utils.demos.coco import gen_colors -from yolo.utils.demos.coco import get_coco_names -from yolo.utils.demos.coco import int_scale_boxes -from yolo.utils.demos import utils -# from utils.demos import utils -from yolo.utils.run_utils import prep_gpu +import yolo.demos.three_servers.video_server as video_t from yolo.configs import yolo as exp_cfg from yolo.tasks.yolo import YoloTask -import yolo.demos.three_servers.video_server as video_t +from yolo.utils.demos import utils +from yolo.utils.demos.coco import (draw_box, gen_colors, get_coco_names, + get_draw_fn, int_scale_boxes) +# from utils.demos import utils +from yolo.utils.run_utils import prep_gpu, support_windows class FastVideo(object): diff --git a/yolo/losses/gradient_aggregator.py b/yolo/losses/gradient_aggregator.py index 2b38f0214..ca7e718a9 100755 --- a/yolo/losses/gradient_aggregator.py +++ b/yolo/losses/gradient_aggregator.py @@ -1,5 +1,7 @@ import tensorflow as tf +from yolo import run + # TODO: run sub dividied training. take the gradients and added them together in the update # once you, when they ask for the result jsut return it, and reset the state # i think you need to zip and then add them together. @@ -79,7 +81,6 @@ def result(self): return self._zeros, self._count -from yolo import run if __name__ == '__main__': task, model, params = run.load_model( diff --git a/yolo/losses/yolo_loss.py b/yolo/losses/yolo_loss.py index d3ef33f61..d065da746 100755 --- a/yolo/losses/yolo_loss.py +++ b/yolo/losses/yolo_loss.py @@ -1,11 +1,11 @@ +import numpy as np import tensorflow as tf import tensorflow.keras as ks from tensorflow.keras import backend as K +from yolo.ops import box_ops, math_ops from yolo.ops.loss_utils import GridGenerator -from yolo.ops import box_ops -from yolo.ops import math_ops -import numpy as np + @tf.custom_gradient def gradient_trap(y): diff --git a/yolo/modeling/Yolo.py b/yolo/modeling/Yolo.py index c349cfbd3..23c39d85c 100755 --- a/yolo/modeling/Yolo.py +++ b/yolo/modeling/Yolo.py @@ -1,12 +1,12 @@ -from yolo.configs import yolo -from official.core import registry -from official.vision.beta.modeling.backbones import factory -from yolo.modeling.backbones.darknet import build_darknet +from typing import * + import tensorflow as tf import tensorflow.keras as ks -from typing import * -from yolo.modeling.backbones.darknet import Darknet +from official.core import registry +from official.vision.beta.modeling.backbones import factory +from yolo.configs import yolo +from yolo.modeling.backbones.darknet import Darknet, build_darknet from yolo.modeling.decoders.yolo_decoder import YoloDecoder from yolo.modeling.heads.yolo_head import YoloHead from yolo.modeling.layers.detection_generator import YoloLayer diff --git a/yolo/modeling/backbones/darknet.py b/yolo/modeling/backbones/darknet.py index 37e2674b5..16007924a 100755 --- a/yolo/modeling/backbones/darknet.py +++ b/yolo/modeling/backbones/darknet.py @@ -14,6 +14,7 @@ """ import collections + import tensorflow as tf import tensorflow.keras as ks diff --git a/yolo/modeling/backbones/darknet_test.py b/yolo/modeling/backbones/darknet_test.py index c362ab71a..1c2c730b1 100755 --- a/yolo/modeling/backbones/darknet_test.py +++ b/yolo/modeling/backbones/darknet_test.py @@ -1,10 +1,9 @@ # Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations from yolo.modeling.backbones import darknet diff --git a/yolo/modeling/backbones/spinenet.py b/yolo/modeling/backbones/spinenet.py index 5381bf0aa..7158d1480 100755 --- a/yolo/modeling/backbones/spinenet.py +++ b/yolo/modeling/backbones/spinenet.py @@ -21,13 +21,13 @@ """ import math +import tensorflow as tf # Import libraries from absl import logging -import tensorflow as tf + from official.modeling import tf_utils from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import nn_blocks -from official.vision.beta.modeling.layers import nn_layers +from official.vision.beta.modeling.layers import nn_blocks, nn_layers from official.vision.beta.ops import spatial_transform_ops layers = tf.keras.layers diff --git a/yolo/modeling/classification_model.py b/yolo/modeling/classification_model.py index 8d4e85765..c4ca48d15 100755 --- a/yolo/modeling/classification_model.py +++ b/yolo/modeling/classification_model.py @@ -16,6 +16,7 @@ # Import libraries import tensorflow as tf + from yolo.modeling.layers import nn_blocks layers = tf.keras.layers diff --git a/yolo/modeling/decoders/yolo_decoder.py b/yolo/modeling/decoders/yolo_decoder.py index 1cf67ff0b..f66ab21b4 100755 --- a/yolo/modeling/decoders/yolo_decoder.py +++ b/yolo/modeling/decoders/yolo_decoder.py @@ -16,6 +16,7 @@ """Feature Pyramid Network and Path Aggregation variants used in YOLO""" import tensorflow as tf + from yolo.modeling.layers import nn_blocks diff --git a/yolo/modeling/decoders/yolo_decoder_.py b/yolo/modeling/decoders/yolo_decoder_.py index 47c2a4d3b..72f1bb6cc 100755 --- a/yolo/modeling/decoders/yolo_decoder_.py +++ b/yolo/modeling/decoders/yolo_decoder_.py @@ -16,6 +16,7 @@ """Feature Pyramid Network and Path Aggregation variants used in YOLO""" import tensorflow as tf + from yolo.modeling.layers import nn_blocks diff --git a/yolo/modeling/decoders/yolo_decoder_test.py b/yolo/modeling/decoders/yolo_decoder_test.py index 8097323a3..57f2337b3 100755 --- a/yolo/modeling/decoders/yolo_decoder_test.py +++ b/yolo/modeling/decoders/yolo_decoder_test.py @@ -15,13 +15,12 @@ # ============================================================================== """Tests for resnet.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations # from yolo.modeling.backbones import darknet from yolo.modeling.decoders import yolo_decoder as decoders diff --git a/yolo/modeling/heads/yolo_head.py b/yolo/modeling/heads/yolo_head.py index 88ad67d45..451acae0a 100755 --- a/yolo/modeling/heads/yolo_head.py +++ b/yolo/modeling/heads/yolo_head.py @@ -1,4 +1,5 @@ import tensorflow as tf + from yolo.modeling.layers import nn_blocks diff --git a/yolo/modeling/heads/yolo_head_test.py b/yolo/modeling/heads/yolo_head_test.py index 19c335e75..bab951b67 100755 --- a/yolo/modeling/heads/yolo_head_test.py +++ b/yolo/modeling/heads/yolo_head_test.py @@ -15,13 +15,12 @@ # ============================================================================== """Tests for resnet.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations # from yolo.modeling.backbones import darknet from yolo.modeling.heads import yolo_head as heads diff --git a/yolo/modeling/layers/detection_generator.py b/yolo/modeling/layers/detection_generator.py index ff710d3f8..44d69d7a3 100755 --- a/yolo/modeling/layers/detection_generator.py +++ b/yolo/modeling/layers/detection_generator.py @@ -3,10 +3,9 @@ import tensorflow.keras as ks import tensorflow.keras.backend as K -from yolo.ops import loss_utils -from yolo.ops import box_ops as box_utils from yolo.losses.yolo_loss import Yolo_Loss -from yolo.ops import nms_ops +from yolo.ops import box_ops as box_utils +from yolo.ops import loss_utils, nms_ops @ks.utils.register_keras_serializable(package='yolo') diff --git a/yolo/modeling/layers/detection_generator_test.py b/yolo/modeling/layers/detection_generator_test.py index 70f22f3be..158304357 100755 --- a/yolo/modeling/layers/detection_generator_test.py +++ b/yolo/modeling/layers/detection_generator_test.py @@ -15,13 +15,12 @@ # ============================================================================== """Tests for resnet.""" -# Import libraries -from absl.testing import parameterized import numpy as np import tensorflow as tf +# Import libraries +from absl.testing import parameterized +from tensorflow.python.distribute import combinations, strategy_combinations -from tensorflow.python.distribute import combinations -from tensorflow.python.distribute import strategy_combinations # from yolo.modeling.backbones import darknet from yolo.modeling.layers import detection_generator as dg diff --git a/yolo/modeling/layers/nn_blocks.py b/yolo/modeling/layers/nn_blocks.py index 3fd9995cb..2f00c5724 100755 --- a/yolo/modeling/layers/nn_blocks.py +++ b/yolo/modeling/layers/nn_blocks.py @@ -1,9 +1,11 @@ """Contains common building blocks for yolo neural networks.""" from typing import Callable + import tensorflow as tf -from yolo.modeling.layers import subnormalization + from official.modeling import tf_utils from official.vision.beta.ops import spatial_transform_ops +from yolo.modeling.layers import subnormalization TPU_BASE = True @@ -1564,4 +1566,4 @@ def call(self, inputs): x_prev = x x = layer(x) output_prev = output - return x_prev, x \ No newline at end of file + return x_prev, x diff --git a/yolo/modeling/layers/nn_blocks_test.py b/yolo/modeling/layers/nn_blocks_test.py index f730d57ee..8b1f2948f 100755 --- a/yolo/modeling/layers/nn_blocks_test.py +++ b/yolo/modeling/layers/nn_blocks_test.py @@ -1,6 +1,6 @@ +import numpy as np import tensorflow as tf import tensorflow.keras as ks -import numpy as np from absl.testing import parameterized from yolo.modeling.layers import nn_blocks diff --git a/yolo/modeling/layers/subnormalization.py b/yolo/modeling/layers/subnormalization.py index d6b8cf0f4..dd0aca3ae 100755 --- a/yolo/modeling/layers/subnormalization.py +++ b/yolo/modeling/layers/subnormalization.py @@ -13,35 +13,22 @@ # limitations under the License. # ============================================================================== """Normalization layers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from tensorflow.python.distribute import distribution_strategy_context +import tensorflow as tf from tensorflow.python.distribute import distribution_strategy_context as ds -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape +from tensorflow.python.distribute import reduce_util +from tensorflow.python.framework import constant_op, dtypes, ops, tensor_shape from tensorflow.python.keras import backend as K -from tensorflow.python.keras import constraints -from tensorflow.python.keras import initializers -from tensorflow.python.keras import regularizers +from tensorflow.python.keras import constraints, initializers, regularizers from tensorflow.python.keras.engine.base_layer import Layer from tensorflow.python.keras.engine.input_spec import InputSpec +from tensorflow.python.keras.layers import normalization from tensorflow.python.keras.utils import control_flow_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import state_ops +from tensorflow.python.ops import (array_ops, control_flow_ops, init_ops, + math_ops, nn, state_ops) from tensorflow.python.ops import variables as tf_variables from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.distribute import reduce_util -from tensorflow.python.keras.layers import normalization - -import tensorflow as tf class SubDivBatchNormalization(normalization.BatchNormalizationBase): diff --git a/yolo/modeling/optimizers/AdamAccumulated.py b/yolo/modeling/optimizers/AdamAccumulated.py index 6719a8f3a..0591ddaf0 100755 --- a/yolo/modeling/optimizers/AdamAccumulated.py +++ b/yolo/modeling/optimizers/AdamAccumulated.py @@ -1,7 +1,7 @@ import tensorflow as tf -from tensorflow.python.keras.optimizer_v2.optimizer_v2 import OptimizerV2 -from tensorflow.python import ops, math_ops, state_ops, control_flow_ops +from tensorflow.python import control_flow_ops, math_ops, ops, state_ops from tensorflow.python.keras import backend_config +from tensorflow.python.keras.optimizer_v2.optimizer_v2 import OptimizerV2 __all__ = ['AdamAccumulated'] diff --git a/yolo/modeling/optimizers/SGDAccumulated.py b/yolo/modeling/optimizers/SGDAccumulated.py index 515587ca0..cd828b65a 100755 --- a/yolo/modeling/optimizers/SGDAccumulated.py +++ b/yolo/modeling/optimizers/SGDAccumulated.py @@ -1,6 +1,8 @@ import tensorflow as tf from tensorflow.python.keras.optimizer_v2.optimizer_v2 import OptimizerV2 -from tensorflow.python.ops import math_ops, state_ops, control_flow_ops, array_ops +from tensorflow.python.ops import (array_ops, control_flow_ops, math_ops, + state_ops) + # from tensorflow.python import ops # from tensorflow.python.keras import backend_config diff --git a/yolo/ops/box_ops.py b/yolo/ops/box_ops.py index adb913df6..eb4b79f68 100755 --- a/yolo/ops/box_ops.py +++ b/yolo/ops/box_ops.py @@ -1,10 +1,12 @@ """ bounding box utils file """ +import math +from typing import Tuple, Union + # import libraries import tensorflow as tf import tensorflow.keras.backend as K -from typing import Tuple, Union -import math + from yolo.ops import math_ops diff --git a/yolo/ops/kmeans_anchors.py b/yolo/ops/kmeans_anchors.py index 8427a539a..9004ace70 100755 --- a/yolo/ops/kmeans_anchors.py +++ b/yolo/ops/kmeans_anchors.py @@ -1,9 +1,8 @@ -import tensorflow as tf import numpy as np +import tensorflow as tf -from yolo.ops.box_ops import compute_iou -from yolo.ops.box_ops import yxyx_to_xcycwh from official.core import input_reader +from yolo.ops.box_ops import compute_iou, yxyx_to_xcycwh class AnchorKMeans: diff --git a/yolo/ops/math_ops.py b/yolo/ops/math_ops.py index 23aebf089..82db5558b 100755 --- a/yolo/ops/math_ops.py +++ b/yolo/ops/math_ops.py @@ -1,6 +1,7 @@ import tensorflow as tf import tensorflow.keras.backend as K + def rm_nan_inf(x, val=0.0): cond = tf.math.logical_or(tf.math.is_nan(x), tf.math.is_inf(x)) val = tf.cast(val, dtype=x.dtype) @@ -20,4 +21,4 @@ def divide_no_nan(a, b): return tf.where(b == zero, zero, a / b) def mul_no_nan(x, y): - return tf.where(x == 0, tf.cast(0, x.dtype), x * y) \ No newline at end of file + return tf.where(x == 0, tf.cast(0, x.dtype), x * y) diff --git a/yolo/ops/preprocessing_ops.py b/yolo/ops/preprocessing_ops.py index 29b1844e7..7065ad745 100755 --- a/yolo/ops/preprocessing_ops.py +++ b/yolo/ops/preprocessing_ops.py @@ -1,9 +1,10 @@ import tensorflow as tf -import tensorflow_addons as tfa import tensorflow.keras.backend as K -from yolo.ops import box_ops -from official.vision.beta.ops import preprocess_ops +import tensorflow_addons as tfa + from official.vision.beta.ops import box_ops as bbox_ops +from official.vision.beta.ops import preprocess_ops +from yolo.ops import box_ops def rand_uniform_strong(minval, maxval, dtype=tf.float32): diff --git a/yolo/run.py b/yolo/run.py index 35532d083..cc94a8fb3 100755 --- a/yolo/run.py +++ b/yolo/run.py @@ -1,29 +1,29 @@ +import os +import sys +from typing import List, Tuple + +import gin +import tensorflow as tf +from absl import app, flags + +# pylint: enable=unused-import +from official.common import flags as tfm_flags +from official.core import task_factory, train_utils +from official.modeling import performance +# pylint: disable=unused-import +from yolo.common import registry_imports +from yolo.demos import video_detect_cpu as vcu +from yolo.demos import video_detect_gpu as vgu from yolo.utils.run_utils import prep_gpu + # try: # # except BaseException: # print("GPUs ready") -from absl import app -from absl import flags -import gin -import sys -from official.core import train_utils -# pylint: disable=unused-import -from yolo.common import registry_imports -# pylint: enable=unused-import -from official.common import flags as tfm_flags -from typing import Tuple, List -from official.core import train_utils -from official.modeling import performance -from official.core import task_factory -import os -from yolo.demos import video_detect_gpu as vgu -from yolo.demos import video_detect_cpu as vcu -import tensorflow as tf """ export GOOGLE_APPLICATION_CREDENTIALS=.json python3.8 -m yolo.run --experiment=yolo_custom --out_resolution 416 --config_file=yolo/configs/experiments/yolov4-eval.yaml --video ../videos/nyc.mp4 --max_batch 5 diff --git a/yolo/run_image.py b/yolo/run_image.py index 5b7accc15..874ba3991 100755 --- a/yolo/run_image.py +++ b/yolo/run_image.py @@ -1,28 +1,24 @@ -from absl import app -from absl import flags -import gin +import os import sys +from typing import List, Tuple + +import cv2 +import gin +import numpy as np +import tensorflow as tf +from absl import app, flags -from official.core import train_utils -# pylint: disable=unused-import -from yolo.common import registry_imports # pylint: enable=unused-import from official.common import flags as tfm_flags - -from typing import Tuple, List -from official.core import train_utils +from official.core import task_factory, train_utils from official.modeling import performance -from official.core import task_factory -import os -import tensorflow as tf - +from skimage import io +# pylint: disable=unused-import +from yolo.common import registry_imports from yolo.run import load_model from yolo.utils.demos import utils -import cv2 -import numpy as np -from skimage import io - from yolo.utils.run_utils import prep_gpu + try: prep_gpu() except BaseException: diff --git a/yolo/tasks/backbone_test.py b/yolo/tasks/backbone_test.py index 630d9b823..78d9ebb47 100755 --- a/yolo/tasks/backbone_test.py +++ b/yolo/tasks/backbone_test.py @@ -1,10 +1,10 @@ -from yolo.configs import darknet_classification as dcfg -from yolo.tasks import image_classification as imc -from yolo.modeling.backbones import darknet +import matplotlib.pyplot as plt import tensorflow as tf +from yolo.configs import darknet_classification as dcfg +from yolo.modeling.backbones import darknet +from yolo.tasks import image_classification as imc from yolo.utils.run_utils import prep_gpu -import matplotlib.pyplot as plt def test_classification_input(): diff --git a/yolo/tasks/image_classification.py b/yolo/tasks/image_classification.py index c72d43272..280f38bf8 100755 --- a/yolo/tasks/image_classification.py +++ b/yolo/tasks/image_classification.py @@ -14,17 +14,17 @@ # limitations under the License. # ============================================================================== """Image classification task definition.""" +import logging + import tensorflow as tf -from official.core import input_reader -from official.core import task_factory + +from official.core import input_reader, task_factory +from official.modeling import tf_utils +from official.vision.beta.tasks import image_classification from yolo.configs import darknet_classification as exp_cfg +from yolo.dataloaders import classification_input, classification_vision from yolo.dataloaders.decoders import classification_tfds_decoder as cli -from yolo.dataloaders import classification_input -from yolo.dataloaders import classification_vision -from official.vision.beta.tasks import image_classification from yolo.losses import cross_entropy_loss -from official.modeling import tf_utils -import logging @task_factory.register_task_cls(exp_cfg.ImageClassificationTask) diff --git a/yolo/tasks/yolo.py b/yolo/tasks/yolo.py index 37b26bf50..50de5b6ae 100755 --- a/yolo/tasks/yolo.py +++ b/yolo/tasks/yolo.py @@ -1,22 +1,18 @@ import tensorflow as tf -from tensorflow.keras.mixed_precision import experimental as mixed_precision - from absl import logging -from official.core import base_task -from official.core import input_reader -from official.core import task_factory -from yolo.configs import yolo as exp_cfg +from tensorflow.keras.mixed_precision import experimental as mixed_precision +from official.core import base_task, input_reader, task_factory +from official.vision.beta.dataloaders import (tf_example_decoder, + tf_example_label_map_decoder, + tfds_detection_decoders) from official.vision.beta.evaluation import coco_evaluator -from official.vision.beta.dataloaders import tf_example_decoder -from official.vision.beta.dataloaders import tfds_detection_decoders -from official.vision.beta.dataloaders import tf_example_label_map_decoder - +from official.vision.beta.ops import box_ops, preprocess_ops +from yolo.configs import yolo as exp_cfg from yolo.dataloaders import yolo_input -from yolo.ops.kmeans_anchors import BoxGenInputReader from yolo.ops.box_ops import xcycwh_to_yxyx +from yolo.ops.kmeans_anchors import BoxGenInputReader -from official.vision.beta.ops import box_ops, preprocess_ops # from yolo.modeling.layers.detection_generator import YoloGTFilter diff --git a/yolo/tasks/yolo_subdiv.py b/yolo/tasks/yolo_subdiv.py index 0751edde4..4a80bc915 100755 --- a/yolo/tasks/yolo_subdiv.py +++ b/yolo/tasks/yolo_subdiv.py @@ -1,23 +1,18 @@ import tensorflow as tf -from tensorflow.keras.mixed_precision import experimental as mixed_precision - from absl import logging -from official.core import base_task -from official.core import input_reader -from official.core import task_factory -from yolo.configs import yolo as exp_cfg +from tensorflow.keras.mixed_precision import experimental as mixed_precision +from official.core import base_task, input_reader, task_factory from official.vision.beta.evaluation import coco_evaluator - +from official.vision.beta.ops import box_ops, preprocess_ops +from yolo.configs import yolo as exp_cfg from yolo.dataloaders import yolo_input from yolo.dataloaders.decoders import tfds_coco_decoder -from yolo.ops.kmeans_anchors import BoxGenInputReader +from yolo.losses import gradient_aggregator from yolo.ops.box_ops import xcycwh_to_yxyx - -from official.vision.beta.ops import box_ops, preprocess_ops +from yolo.ops.kmeans_anchors import BoxGenInputReader # from yolo.modeling.layers.detection_generator import YoloGTFilter from yolo.tasks import yolo -from yolo.losses import gradient_aggregator @task_factory.register_task_cls(exp_cfg.YoloSubDivTask) diff --git a/yolo/tasks/yolo_test.py b/yolo/tasks/yolo_test.py index 4f272e691..46c7ccdc8 100755 --- a/yolo/tasks/yolo_test.py +++ b/yolo/tasks/yolo_test.py @@ -1,11 +1,12 @@ -from yolo.tasks import yolo +import tensorflow as tf +from absl.testing import parameterized + import orbit from official.core import exp_factory from official.modeling import optimization - -import tensorflow as tf -from absl.testing import parameterized +from yolo.tasks import yolo from yolo.utils.run_utils import prep_gpu + # try: # prep_gpu() # except BaseException: diff --git a/yolo/test.py b/yolo/test.py index 855b4b3f5..c87a654cb 100755 --- a/yolo/test.py +++ b/yolo/test.py @@ -1,4 +1,4 @@ -from yolo.utils.run_utils import prep_gpu, expand_gpu +from yolo.utils.run_utils import expand_gpu, prep_gpu expand_gpu() try: diff --git a/yolo/train.py b/yolo/train.py index c00a5e3d4..751f5f7d7 100755 --- a/yolo/train.py +++ b/yolo/train.py @@ -14,27 +14,27 @@ # limitations under the License. # ============================================================================== """TensorFlow Model Garden Vision training driver.""" -from yolo.utils.run_utils import prep_gpu -try: - prep_gpu() -except BaseException: - print('GPUs ready') +import pprint +import sys -from absl import app -from absl import flags import gin -import sys +from absl import app, flags -from official.core import train_utils -# pylint: disable=unused-import -from yolo.common import registry_imports # pylint: enable=unused-import from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib +from official.core import task_factory, train_lib, train_utils from official.modeling import performance -import pprint +# pylint: disable=unused-import +from yolo.common import registry_imports +from yolo.utils.run_utils import prep_gpu + +try: + prep_gpu() +except BaseException: + print('GPUs ready') + + FLAGS = flags.FLAGS """ diff --git a/yolo/train_gilbreth.py b/yolo/train_gilbreth.py index ba9108a8c..9cb243afb 100755 --- a/yolo/train_gilbreth.py +++ b/yolo/train_gilbreth.py @@ -21,22 +21,19 @@ # except RuntimeError: # print("GPUs ready") -from absl import app -from absl import flags -import gin import sys -from official.core import train_utils -# pylint: disable=unused-import -from yolo.common import registry_imports +import gin +from absl import app, flags +from tensorflow import distribute + # pylint: enable=unused-import from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib +from official.core import task_factory, train_lib, train_utils from official.modeling import performance - -from tensorflow import distribute +# pylint: disable=unused-import +from yolo.common import registry_imports FLAGS = flags.FLAGS """ diff --git a/yolo/train_vm.py b/yolo/train_vm.py index b99928385..a93a798be 100755 --- a/yolo/train_vm.py +++ b/yolo/train_vm.py @@ -14,23 +14,20 @@ # limitations under the License. # ============================================================================== """TensorFlow Model Garden Vision training driver.""" -from yolo.utils.run_utils import prep_gpu +import sys -from absl import app -from absl import flags import gin -import sys +import tensorflow as tf +from absl import app, flags -from official.core import train_utils -# pylint: disable=unused-import -from yolo.common import registry_imports # pylint: enable=unused-import from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib +from official.core import task_factory, train_lib, train_utils from official.modeling import performance -import tensorflow as tf +# pylint: disable=unused-import +from yolo.common import registry_imports +from yolo.utils.run_utils import prep_gpu FLAGS = flags.FLAGS """ diff --git a/yolo/utils/_darknet2tf/__init__.py b/yolo/utils/_darknet2tf/__init__.py index fb53d1f34..1768ee28b 100755 --- a/yolo/utils/_darknet2tf/__init__.py +++ b/yolo/utils/_darknet2tf/__init__.py @@ -1,11 +1,10 @@ import collections import collections.abc import io +from typing import Type, TypeVar, Union from yolo.utils.downloads.file_manager import PathABC -from typing import Union, Type, TypeVar - T = TypeVar('T', bound='DarkNetModel') diff --git a/yolo/utils/_darknet2tf/__main__.py b/yolo/utils/_darknet2tf/__main__.py index 013abc20f..5129be61f 100755 --- a/yolo/utils/_darknet2tf/__main__.py +++ b/yolo/utils/_darknet2tf/__main__.py @@ -1,11 +1,12 @@ # !/usr/bin/env python3 import argparse as _argparse -from absl.flags import argparse_flags as _argparse_flags -from absl import flags as _flags -from . import DarkNetConverter -from absl import app import sys -from . import main, _parser + +from absl import app +from absl import flags as _flags +from absl.flags import argparse_flags as _argparse_flags + +from . import DarkNetConverter, _parser, main "Convert a DarkNet config file and weights into a TensorFlow model" diff --git a/yolo/utils/_darknet2tf/config_classes.py b/yolo/utils/_darknet2tf/config_classes.py index 1b31f2738..f9d7207e3 100755 --- a/yolo/utils/_darknet2tf/config_classes.py +++ b/yolo/utils/_darknet2tf/config_classes.py @@ -10,9 +10,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -import numpy as np +from typing import List, Tuple -from typing import Tuple, List +import numpy as np class Config(ABC): diff --git a/yolo/utils/_darknet2tf/dn2dicts.py b/yolo/utils/_darknet2tf/dn2dicts.py index 209eab066..95b3ad17b 100755 --- a/yolo/utils/_darknet2tf/dn2dicts.py +++ b/yolo/utils/_darknet2tf/dn2dicts.py @@ -4,7 +4,6 @@ import configparser import io import sys - from typing import Dict, List if sys.version_info < (3, 10): diff --git a/yolo/utils/_darknet2tf/dnnum.py b/yolo/utils/_darknet2tf/dnnum.py index 648fed07a..ed785d738 100755 --- a/yolo/utils/_darknet2tf/dnnum.py +++ b/yolo/utils/_darknet2tf/dnnum.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 "Number the blocks in a DarkNet config file" +import argparse + from absl import app from absl.flags import argparse_flags -import argparse from yolo.utils.downloads import file_manager diff --git a/yolo/utils/_darknet2tf/load_weights copy.py b/yolo/utils/_darknet2tf/load_weights copy.py index 6f7a617bd..d740f6cc0 100755 --- a/yolo/utils/_darknet2tf/load_weights copy.py +++ b/yolo/utils/_darknet2tf/load_weights copy.py @@ -3,8 +3,11 @@ format into TensorFlow layers """ import itertools + from tensorflow import keras as ks -from yolo.modeling.layers.nn_blocks import DarkConv # pylint: disable=unused-import + +from yolo.modeling.layers.nn_blocks import \ + DarkConv # pylint: disable=unused-import def split_converter(lst, i, j=None): diff --git a/yolo/utils/_darknet2tf/load_weights.py b/yolo/utils/_darknet2tf/load_weights.py index 14dc7e030..63afe5fc2 100755 --- a/yolo/utils/_darknet2tf/load_weights.py +++ b/yolo/utils/_darknet2tf/load_weights.py @@ -3,8 +3,11 @@ format into TensorFlow layers """ import itertools + from tensorflow import keras as ks + from yolo.modeling.layers.nn_blocks import ConvBN + from .config_classes import convCFG diff --git a/yolo/utils/_darknet2tf/load_weights2.py b/yolo/utils/_darknet2tf/load_weights2.py index 3916e96f2..207258c5d 100755 --- a/yolo/utils/_darknet2tf/load_weights2.py +++ b/yolo/utils/_darknet2tf/load_weights2.py @@ -1,6 +1,8 @@ +import numpy as np + from yolo.modeling.layers.nn_blocks import ConvBN + from .config_classes import convCFG, samCFG -import numpy as np def split_converter(lst, i, j=None): diff --git a/yolo/utils/_darknet2tf/read_weights.py b/yolo/utils/_darknet2tf/read_weights.py index 234ed09f6..9f0793ea4 100755 --- a/yolo/utils/_darknet2tf/read_weights.py +++ b/yolo/utils/_darknet2tf/read_weights.py @@ -2,9 +2,10 @@ This file contains the code to parse DarkNet weight files. """ +from yolo.utils.downloads.file_manager import get_size, open_if_not_open + from .config_classes import * # pylint: disable=wildcard-import, unused-wildcard-import from .dn2dicts import convertConfigFile -from yolo.utils.downloads.file_manager import get_size, open_if_not_open def build_layer(layer_dict, file, net): diff --git a/yolo/utils/demos/coco.py b/yolo/utils/demos/coco.py index 0a541be77..e3be9428e 100755 --- a/yolo/utils/demos/coco.py +++ b/yolo/utils/demos/coco.py @@ -1,7 +1,7 @@ -import cv2 import colorsys -import numpy as np +import cv2 +import numpy as np import tensorflow as tf import tensorflow.keras.backend as K diff --git a/yolo/utils/demos/coco_dataset_converter.py b/yolo/utils/demos/coco_dataset_converter.py index f0f780505..a25fff57d 100755 --- a/yolo/utils/demos/coco_dataset_converter.py +++ b/yolo/utils/demos/coco_dataset_converter.py @@ -1,9 +1,10 @@ +import io +import json +import os + import tensorflow as tf from PIL import Image from tqdm import tqdm -import io -import os -import json def encode_image(image, fmt='JPEG'): diff --git a/yolo/utils/demos/utils.py b/yolo/utils/demos/utils.py index 5ee359c20..7914342f2 100755 --- a/yolo/utils/demos/utils.py +++ b/yolo/utils/demos/utils.py @@ -1,14 +1,16 @@ -import tensorflow as tf -import tensorflow.keras.backend as K +import colorsys +import multiprocessing as mp +import os import socket import struct +from concurrent.futures import ThreadPoolExecutor as pooler from typing import Callable -import numpy as np -import colorsys + import cv2 -from concurrent.futures import ThreadPoolExecutor as pooler -import multiprocessing as mp -import os +import numpy as np +import tensorflow as tf +import tensorflow.keras.backend as K + # from concurrent.futures import ProcessPoolExecutor as pooler diff --git a/yolo/utils/downloads/__init__.py b/yolo/utils/downloads/__init__.py index 8b1378917..e69de29bb 100755 --- a/yolo/utils/downloads/__init__.py +++ b/yolo/utils/downloads/__init__.py @@ -1 +0,0 @@ - diff --git a/yolo/utils/downloads/file_manager.py b/yolo/utils/downloads/file_manager.py index 492644194..aee14cc51 100755 --- a/yolo/utils/downloads/file_manager.py +++ b/yolo/utils/downloads/file_manager.py @@ -6,7 +6,6 @@ import io import os - from typing import Union # define PathABC type diff --git a/yolo/utils/export/get_cfg_yaml.py b/yolo/utils/export/get_cfg_yaml.py index bd09487c6..ccb9cf409 100755 --- a/yolo/utils/export/get_cfg_yaml.py +++ b/yolo/utils/export/get_cfg_yaml.py @@ -1,6 +1,7 @@ -from yolo.configs import yolo as exp_cfg import yaml +from yolo.configs import yolo as exp_cfg + config = exp_cfg.yolo_v4_coco() e = yaml.dump(config.as_dict(), default_flow_style=False) diff --git a/yolo/utils/export/tensor_rt.py b/yolo/utils/export/tensor_rt.py index d3b033526..c4e81219a 100755 --- a/yolo/utils/export/tensor_rt.py +++ b/yolo/utils/export/tensor_rt.py @@ -1,10 +1,12 @@ import tensorflow as tf -# import tensorflow.experimental.tensorrt as trt - from tensorflow.python.compiler.tensorrt import trt_convert as trt from yolo.utils.run_utils import prep_gpu +# import tensorflow.experimental.tensorrt as trt + + + class TensorRT(object): diff --git a/yolo/utils/export/tflite_convert.py b/yolo/utils/export/tflite_convert.py index 075f860c1..3e510160d 100755 --- a/yolo/utils/export/tflite_convert.py +++ b/yolo/utils/export/tflite_convert.py @@ -1,10 +1,13 @@ import os + +import cv2 import tensorflow as tf -from yolo.utils.run_utils import prep_gpu + +from skimage import io from yolo.configs import yolo as exp_cfg from yolo.tasks.yolo import YoloTask -from skimage import io -import cv2 +from yolo.utils.run_utils import prep_gpu + prep_gpu() diff --git a/yolo/utils/export/tflite_convert_gpu.py b/yolo/utils/export/tflite_convert_gpu.py index 5f971024f..ece8c9511 100755 --- a/yolo/utils/export/tflite_convert_gpu.py +++ b/yolo/utils/export/tflite_convert_gpu.py @@ -1,9 +1,11 @@ +import cv2 import tensorflow as tf -from yolo.utils.run_utils import prep_gpu + +from skimage import io from yolo.configs import yolo as exp_cfg from yolo.tasks.yolo import YoloTask -from skimage import io -import cv2 +from yolo.utils.run_utils import prep_gpu + prep_gpu() diff --git a/yolo/utils/export/tflite_gpu_test.py b/yolo/utils/export/tflite_gpu_test.py index 4d1b16ac8..4c7bd442f 100755 --- a/yolo/utils/export/tflite_gpu_test.py +++ b/yolo/utils/export/tflite_gpu_test.py @@ -1,7 +1,8 @@ +import cv2 import numpy as np import tensorflow as tf + from skimage import io -import cv2 # from yolo.utils.run_utils import prep_gpu # prep_gpu() # from yolo.modeling.layers.detection_generator import YoloLayer as filters diff --git a/yolo/utils/export/tflite_inspect.py b/yolo/utils/export/tflite_inspect.py index 6b30df8f0..ff95b2c66 100755 --- a/yolo/utils/export/tflite_inspect.py +++ b/yolo/utils/export/tflite_inspect.py @@ -1,5 +1,7 @@ import tensorflow as tf + from skimage import io + # from yolo.utils.run_utils import prep_gpu # prep_gpu() # from yolo.modeling.layers.detection_generator import YoloLayer as filters diff --git a/yolo/utils/export/tflite_test.py b/yolo/utils/export/tflite_test.py index 12463c144..6dcad8904 100755 --- a/yolo/utils/export/tflite_test.py +++ b/yolo/utils/export/tflite_test.py @@ -1,7 +1,8 @@ +import cv2 import numpy as np import tensorflow as tf + from skimage import io -import cv2 # from yolo.utils.run_utils import prep_gpu # prep_gpu() # from yolo.modeling.layers.detection_generator import YoloLayer as filters diff --git a/yolo/utils/tests/test_darknet2tf.py b/yolo/utils/tests/test_darknet2tf.py index 301d5da97..482cadb4d 100755 --- a/yolo/utils/tests/test_darknet2tf.py +++ b/yolo/utils/tests/test_darknet2tf.py @@ -1,5 +1,5 @@ -from absl.testing import parameterized import tensorflow as tf +from absl.testing import parameterized from yolo import DarkNet53 diff --git a/yolo/utils/training/keras_task_trainer.py b/yolo/utils/training/keras_task_trainer.py index 376386697..ca54edc42 100755 --- a/yolo/utils/training/keras_task_trainer.py +++ b/yolo/utils/training/keras_task_trainer.py @@ -1,9 +1,10 @@ import tensorflow as tf +from tensorflow.keras.mixed_precision import experimental as mixed_precision + import official.core.base_task as bt from official.core import input_reader from yolo.dataloaders import yolo_input from yolo.dataloaders.decoders import tfds_coco_decoder -from tensorflow.keras.mixed_precision import experimental as mixed_precision class Trainer(tf.keras.Model): From 4665a16ef56ec291a00551e2e5195a1a920e3dc6 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 5 Apr 2021 16:25:48 -0400 Subject: [PATCH 081/132] still working through errors when running train --- centernet/configs/centernet.py | 62 +++++++++++++++++-- .../configs/experiments/centernet-eval.yaml | 2 +- centernet/dataloaders/centernet_input.py | 23 +++++-- centernet/modeling/CenterNet.py | 9 ++- centernet/modeling/backbones/hourglass.py | 2 +- .../modeling/decoders/centernet_decoder.py | 30 ++++++--- .../modeling/layers/detection_generator.py | 15 +++-- centernet/modeling/layers/nn_blocks.py | 3 +- centernet/tasks/centernet.py | 7 +-- .../utils/weight_utils/test_load_weights.py | 34 +++++----- 10 files changed, 134 insertions(+), 53 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index bc982ec02..8d4db6de1 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -25,6 +25,59 @@ from official.vision.beta.configs import common +# default param classes +@dataclasses.dataclass +class ModelConfig(hyperparams.Config): + + @property + def input_size(self): + if self._input_size is None: + return [None, None, 3] + else: + return self._input_size + + @input_size.setter + def input_size(self, input_size): + self._input_size = input_size + + @property + def backbone(self): + if isinstance(self.base, str): + # TODO: remove the automatic activation setter + # self.norm_activation.activation = Yolo._DEFAULTS[self.base].activation + return CenterNet._DEFAULTS[self.base].backbone + else: + return self.base.backbone + + @backbone.setter + def backbone(self, val): + self.base.backbone = val + + @property + def decoder(self): + if isinstance(self.base, str): + return CenterNet._DEFAULTS[self.base].decoder + else: + return self.base.decoder + + @decoder.setter + def decoder(self, val): + self.base.decoder = val + + @property + def odapi_weights_file(self): + if isinstance(self.base, str): + return CenterNet._DEFAULTS[self.base].odapi_weights + else: + return self.base.odapi_weights + + @property + def extremenet_weights_file(self): + if isinstance(self.base, str): + return CenterNet._DEFAULTS[self.base].extremenet_weights + else: + return self.base.extremenet_weights + @dataclasses.dataclass class TfExampleDecoder(hyperparams.Config): regenerate_source_id: bool = False @@ -66,9 +119,9 @@ class Parser(hyperparams.Config): @dataclasses.dataclass class DataConfig(cfg.DataConfig): """Input config for training.""" - input_path: str = 'D:\\Datasets\\coco\\2017\\1.1.0*' #'gs://tensorflow2/coco_records/train/2017*' + input_path: str = '' # 'D:\\Datasets\\coco\\2017\\1.1.0*' #'gs://tensorflow2/coco_records/train/2017*' tfds_name: str = None #'coco' - tfds_split: str = 'val' #'train' + tfds_split: str = None #'train' #'val' global_batch_size: int = 32 is_training: bool = True dtype: str = 'float16' @@ -138,13 +191,12 @@ class CenterNetBase(hyperparams.OneOfConfig): decoder_name: str = 'detection_2d' @dataclasses.dataclass -class CenterNet(hyperparams.Config): +class CenterNet(ModelConfig): base: Union[str, CenterNetBase] = CenterNetBase() num_classes: int = 90 gaussian_iou: float = 0.7 max_num_instances: int = 200 - input_size: Optional[List[int]] = dataclasses.field( - default_factory=lambda: [None, None, 3]) + _input_size: Optional[List[int]] = None filter: CenterNetLayer = CenterNetLayer() @dataclasses.dataclass diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index 8099641fc..af8a07cb7 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -16,7 +16,7 @@ task: decoder_name: detection_2d num_classes: 90 gaussian_iou: 0.7 - input_size: [512, 512, 3] # null + _input_size: [512, 512, 3] # null filter: max_detections: 100 peak_error: 0.000001 diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index a2808334b..f1a7f1385 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -139,7 +139,7 @@ def _generate_heatmap(self, boxes, output_size, input_size): } return labels - def _parse_train_data(self, decoded_tensors): + def _parse_train_data(self, data): """Generates images and labels that are usable for model training. Args: @@ -149,12 +149,25 @@ def _parse_train_data(self, decoded_tensors): images: the image tensor. labels: a dict of Tensors that contains labels. """ - # TODO: input size, output size - image = decoded_tensors["image"] + # FIXME: This is a copy of parse eval data + image = data["image"] / 255 + boxes = data['groundtruth_boxes'] + classes = data['groundtruth_classes'] + + image, boxes, info = yolo_preprocessing_ops.letter_box( + image, boxes, xs = 0.5, ys = 0.5, target_dim=self._image_w) + + image = tf.cast(image, self._dtype) + shape = tf.shape(image) + height = shape[0] + width = shape[1] + labels = self._generate_heatmap( - decoded_tensors["groundtruth_boxes"], - output_size, input_size + boxes=boxes, output_size=[self._image_h, self._image_w], input_size=[height, width] ) + + labels.update({'bbox': boxes}) + return image, labels def _parse_eval_data(self, data): diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index af8689c94..b6de5d93d 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -60,7 +60,7 @@ def build_centernet_decoder(input_specs, task_config, num_inputs): # task specific task_outputs = task_config._get_output_length_dict() model = CenterNetDecoder( - input_specs = input_specs + input_specs = input_specs, task_outputs=task_outputs, heatmap_bias=heatmap_bias, num_inputs=num_inputs) @@ -92,7 +92,12 @@ def build_centernet(input_specs, task_config, l2_regularization): filter = build_centernet_filter(model_config) model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) - model.build(input_specs.shape) + + + test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.float32) + model(test_input) + # model.build(input_specs.shape) + model.summary() # TODO: uncommend when filter is implemented # losses = filter.losses diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index de3ffd45e..1e21305d1 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -115,7 +115,7 @@ def __init__( self._blocks_per_stage = blocks_per_stage self._num_hourglasses = num_hourglasses self._initial_downsample = initial_downsample - self._output_specs = all_heatmaps[-1].get_shape() + self._output_specs = [hm.get_shape() for hm in all_heatmaps] def get_config(self): layer_config = { diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index 0862a60e7..2cec7250f 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -10,7 +10,7 @@ class CenterNetDecoder(tf.keras.Model): CenterNet Decoder """ def __init__(self, - input_specs + input_specs, task_outputs: dict, heatmap_bias: float = -2.19, num_inputs: int = 2, @@ -28,22 +28,30 @@ def __init__(self, and the respective output tensor """ # super().__init__(**kwargs) + # self.outputs_layers = {} - # self._task_outputs = task_outputs - # self._heatmap_bias = heatmap_bias - # self._num_inputs = num_inputs + self._input_specs = input_specs + self._task_outputs = task_outputs + self._heatmap_bias = heatmap_bias + self._num_inputs = num_inputs - # self.outputs_layers = {} - inputs = [tf.keras.layers.Input(shape=value[1:]) for value in input_specs] + inputs = [tf.keras.layers.Input(shape=value[1:]) for value in self._input_specs] outputs = dict() - for key in task_outputs: - num_filters = self.task_outputs[key] + + for key in self._task_outputs: + num_filters = self._task_outputs[key] bias = 0 if 'heatmaps' in key: bias = self._heatmap_bias + outputs[key] = [CenterNetDecoderConv(output_filters=num_filters, - name=key, bias_init=bias)() for _ in range(self.num_inputs)] + name=key + str(i), bias_init=bias)(inputs[i]) for i in range(self._num_inputs)] + + self._output_specs = { + key: [value[i].get_shape() for i in range(num_inputs)] + for key, value in outputs.items() + } super().__init__(inputs=inputs, outputs=outputs, name='CenterNetDecoder') @@ -64,6 +72,10 @@ def __init__(self, # outputs[key] = [self.outputs_layers[key][i](inputs[i]) for i in range(self._num_inputs)] # return outputs + + # @property + # def output_specs(self): + # return self._output_specs def get_config(self): layer_config = { diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index be9ae8e38..6fde7e0a9 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -1,8 +1,11 @@ import tensorflow as tf -from tensorflow import keras as ks +import tensorflow.keras as ks from yolo.ops.nms_ops import nms +# from tensorflow import keras as ks + + @tf.keras.utils.register_keras_serializable(package='centernet') class CenterNetLayer(ks.Model): @@ -94,11 +97,11 @@ def process_heatmap(self, # Zero out everything that is not a peak. feature_map_peaks = ( - feature_map * tf.cast(feature_map_peak_mask, tf.float32)) + feature_map * tf.cast(feature_map_peak_mask, tf.float16)) # Zero out peaks whose scores do not exceed threshold valid_peaks_mask = feature_map_peaks > center_thresh - feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.float32) + feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.float16) return feature_map_peaks @@ -269,8 +272,8 @@ class prediction for each box. y_offsets = offsets[..., 0] x_offsets = offsets[..., 1] - y_indices = tf.cast(y_indices, dtype=tf.float32) - x_indices = tf.cast(x_indices, dtype=tf.float32) + y_indices = tf.cast(y_indices, dtype=tf.float16) + x_indices = tf.cast(x_indices, dtype=tf.float16) detection_classes = channel_indices + self._class_offset @@ -306,7 +309,7 @@ def call(self, inputs): y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) # Normalize bounding boxes - boxes = boxes / tf.cast(height, tf.float32) + boxes = boxes / tf.cast(height, tf.float16) # Apply nms if self._use_nms: diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 05412e018..37d41018f 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -322,9 +322,10 @@ def __init__(self, convolution layer name: string, layer name """ + super().__init__(name=name, **kwargs) self._output_filters = output_filters self._bias_init = bias_init - super().__init__(name=name, **kwargs) + def build(self, input_shape): n_channels = input_shape[-1] diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 1645dfc57..7d342cf6a 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -36,17 +36,12 @@ def build_inputs(self, params, input_context=None): gaussian_iou=model.gaussian_iou, ) - if params.is_training: - post_process_fn = parser.postprocess_fn() - else: - post_process_fn = None - reader = input_reader.InputReader( params, dataset_fn=tf.data.TFRecordDataset, decoder_fn=decoder.decode, parser_fn=parser.parse_fn(params.is_training), - postprocess_fn=post_process_fn) + postprocess_fn=parser.postprocess_fn(params.is_training)) dataset = reader.read(input_context=input_context) return dataset diff --git a/centernet/utils/weight_utils/test_load_weights.py b/centernet/utils/weight_utils/test_load_weights.py index 4fd9dc4ba..fd41318f7 100644 --- a/centernet/utils/weight_utils/test_load_weights.py +++ b/centernet/utils/weight_utils/test_load_weights.py @@ -25,21 +25,21 @@ # extreme_net_weights_dict, _ = get_model_weights_as_dict(EXTREMENET_CKPT_PATH) # load_weights_backbone(model.backbone, extreme_net_weights_dict['feature_extractor'], backbone_name='extremenet') - # # Test for loading ODAPI weights and running inference using webcam - # weights_dict, _ = get_model_weights_as_dict(CENTERNET_CKPT_PATH) - # load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + # Test for loading ODAPI weights and running inference using webcam + weights_dict, _ = get_model_weights_as_dict(CENTERNET_CKPT_PATH) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') - # cap = FastVideo( - # 0, - # model=model, - # process_width=512, - # process_height=512, - # preprocess_with_gpu=False, - # classes=91, - # print_conf=True, - # max_batch=1, - # disp_h=512, - # scale_que=1, - # wait_time='dynamic') - # cap.run() - # runner(model, 0, 512, 512) + cap = FastVideo( + 0, + model=model, + process_width=512, + process_height=512, + preprocess_with_gpu=False, + classes=91, + print_conf=True, + max_batch=1, + disp_h=512, + scale_que=1, + wait_time='dynamic') + cap.run() + runner(model, 0, 512, 512) From 9d18970bf7969a9cf6a87ffa6c2974a4a2035a75 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 7 Apr 2021 02:56:28 -0400 Subject: [PATCH 082/132] getting to eval step, but errors in build_losses --- centernet/configs/centernet.py | 4 ++-- centernet/ops/loss_ops.py | 13 +++++++++++-- centernet/tasks/centernet.py | 10 +++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 8d4db6de1..94aa530ed 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -167,13 +167,13 @@ class CenterNetLayer(hyperparams.Config): class_offset: int = 1 @dataclasses.dataclass -class CenterNetDetection(cfg.TaskConfig): +class CenterNetDetection(hyperparams.Config): use_centers: bool = True use_corners: bool = False predict_3d: bool = False @dataclasses.dataclass -class CenterNetSubTasks(cfg.TaskConfig): +class CenterNetSubTasks(hyperparams.Config): detection: CenterNetDetection = CenterNetDetection() # kp_detection: bool = False segmentation: bool = False diff --git a/centernet/ops/loss_ops.py b/centernet/ops/loss_ops.py index 227fd0ebc..a05cf3a26 100644 --- a/centernet/ops/loss_ops.py +++ b/centernet/ops/loss_ops.py @@ -45,8 +45,17 @@ def get_num_instances_from_weights(groundtruth_weights_list): images in the batch. Note that this function is usually used to normalize the loss so the minimum return value is 1 to avoid weird behavior. """ - num_instances = tf.reduce_sum( - [tf.math.count_nonzero(w) for w in groundtruth_weights_list]) + # num_instances = tf.reduce_sum( + # [tf.math.count_nonzero(w) for w in groundtruth_weights_list]) + + # This can execute in graph mode + groundtruth_weights_list = tf.convert_to_tensor( + groundtruth_weights_list, dtype=groundtruth_weights_list[0].dtype) + num_instances = tf.map_fn( + fn=lambda x: tf.math.count_nonzero(x, dtype=groundtruth_weights_list[0].dtype), + elems=groundtruth_weights_list) + + num_instances = tf.reduce_sum(num_instances) num_instances = tf.maximum(num_instances, 1) return num_instances diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 7d342cf6a..4cfee4a07 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -124,18 +124,18 @@ def build_losses(self, outputs, labels, aux_losses=None): object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) - outputs['ct_heatmaps'] = loss_ops._flatten_spatial_dimensions(outputs['ct_heatmaps']) + output_ct_heatmaps = loss_ops._flatten_spatial_dimensions(outputs['ct_heatmaps'][-1]) total_loss += object_center_loss( - flattened_ct_heatmaps, outputs['ct_heatmaps']) #removed weight parameter (weight = per_pixel_weight) + flattened_ct_heatmaps, output_ct_heatmaps) #removed weight parameter (weight = per_pixel_weight) center_loss = tf.reduce_sum(total_loss) / ( - float(len(outputs['ct_heatmaps'])) * num_boxes) + float(len(output_ct_heatmaps)) * num_boxes) loss += center_loss metric_dict['ct_loss'] = center_loss localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) # Compute the scale loss. - scale_pred = outputs['ct_size'] - offset_pred = outputs['ct_offset'] + scale_pred = outputs['ct_size'][-1] + offset_pred = outputs['ct_offset'][-1] total_scale_loss += localization_loss_fn( labels['ct_size'], scale_pred) #removed weights=batch_weights # Compute the offset loss. From 1b6bc89c91a5a18a5ea429ec272366d527c0d696 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 7 Apr 2021 16:11:20 -0400 Subject: [PATCH 083/132] task uses parser config params --- centernet/configs/centernet.py | 39 ++++++++++--------- .../configs/experiments/centernet-eval.yaml | 9 +++-- centernet/dataloaders/centernet_input.py | 26 ++++++++----- centernet/tasks/centernet.py | 12 ++++-- centernet/train.py | 2 +- 5 files changed, 53 insertions(+), 35 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 94aa530ed..afbc1073c 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -98,23 +98,28 @@ class DataDecoder(hyperparams.OneOfConfig): class Parser(hyperparams.Config): image_w: int = 512 image_h: int = 512 - fixed_size: bool = True + num_classes: int = 90 max_num_instances: int = 200 - min_process_size: int = 320 - max_process_size: int = 608 - letter_box: bool = True - random_flip: bool = True - pct_rand: float = 0.0 - jitter_im: float = 0.0 - jitter_boxes: float = 0.000 - aug_rand_transalate: float = 0.0 - aug_rand_saturation: float = 0.0 - aug_rand_brightness: float = 0.0 - aug_rand_zoom: float = 0.0 - aug_rand_hue: float = 0.0 - keep_thresh: float = 0.0 - mosaic_frequency: float = 1.0 - use_tie_breaker: bool = True + gaussian_iou: float = 0.7 + output_dims: int = 128 + dtype: str = 'float32' + # fixed_size: bool = True + # max_num_instances: int = 200 + # min_process_size: int = 320 + # max_process_size: int = 608 + # letter_box: bool = True + # random_flip: bool = True + # pct_rand: float = 0.0 + # jitter_im: float = 0.0 + # jitter_boxes: float = 0.000 + # aug_rand_transalate: float = 0.0 + # aug_rand_saturation: float = 0.0 + # aug_rand_brightness: float = 0.0 + # aug_rand_zoom: float = 0.0 + # aug_rand_hue: float = 0.0 + # keep_thresh: float = 0.0 + # mosaic_frequency: float = 1.0 + # use_tie_breaker: bool = True @dataclasses.dataclass class DataConfig(cfg.DataConfig): @@ -194,8 +199,6 @@ class CenterNetBase(hyperparams.OneOfConfig): class CenterNet(ModelConfig): base: Union[str, CenterNetBase] = CenterNetBase() num_classes: int = 90 - gaussian_iou: float = 0.7 - max_num_instances: int = 200 _input_size: Optional[List[int]] = None filter: CenterNetLayer = CenterNetLayer() diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index af8a07cb7..840f34e2f 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -15,7 +15,6 @@ task: backbone_name: hourglass104_512 decoder_name: detection_2d num_classes: 90 - gaussian_iou: 0.7 _input_size: [512, 512, 3] # null filter: max_detections: 100 @@ -35,11 +34,13 @@ task: is_training: false drop_remainder: true parser: - fixed_size: true image_h: 512 image_w: 512 - letter_box: true - use_tie_breaker: true + num_classes: 90 + max_num_instances: 200 + gaussian_iou: 0.7 + output_dims: 128 + dtype: float32 shuffle_buffer_size: 10000 subtasks: detection: diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index f1a7f1385..de218668d 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -10,8 +10,9 @@ def __init__(self, image_w: int = 512, image_h: int = 512, num_classes: int = 90, - max_num_instances: int = 200, + max_num_instances: int = 200, # 200 or 100? gaussian_iou: float = 0.7, + output_dims: int = 128, # should be 128 for all models dtype: str = 'float32'): self._image_w = image_w @@ -19,6 +20,7 @@ def __init__(self, self._num_classes = num_classes self._max_num_instances = max_num_instances self._gaussian_iou = gaussian_iou + self._output_dims = output_dims self._gaussian_bump = True self._gaussian_rad = -1 @@ -33,8 +35,9 @@ def __init__(self, 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' ) - def _generate_heatmap(self, boxes, output_size, input_size): + def _generate_heatmap(self, boxes, classes, output_size, input_size): boxes = tf.cast(boxes, dtype=tf.float32) + classes = tf.cast(classes, dtype=tf.float32) tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) @@ -57,9 +60,10 @@ def _generate_heatmap(self, boxes, output_size, input_size): for tag_ind in tf.range(num_boxes): detection = boxes[tag_ind] - - category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary - category = 0 # FIXME: For testing only + category = classes[tag_ind] + + # category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary + # category = 0 # FIXME: For testing only xtl, ytl = detection[0], detection[1] xbr, ybr = detection[2], detection[3] @@ -150,7 +154,7 @@ def _parse_train_data(self, data): labels: a dict of Tensors that contains labels. """ # FIXME: This is a copy of parse eval data - image = data["image"] / 255 + image = data['image'] / 255 boxes = data['groundtruth_boxes'] classes = data['groundtruth_classes'] @@ -163,7 +167,9 @@ def _parse_train_data(self, data): width = shape[1] labels = self._generate_heatmap( - boxes=boxes, output_size=[self._image_h, self._image_w], input_size=[height, width] + boxes=boxes, classes=classes, + output_size=[self._output_dims, self._output_dims], + input_size=[self._image_h, self._image_w] ) labels.update({'bbox': boxes}) @@ -171,7 +177,7 @@ def _parse_train_data(self, data): return image, labels def _parse_eval_data(self, data): - image = data["image"] / 255 + image = data['image'] / 255 boxes = data['groundtruth_boxes'] classes = data['groundtruth_classes'] @@ -184,7 +190,9 @@ def _parse_eval_data(self, data): width = shape[1] labels = self._generate_heatmap( - boxes=boxes, output_size=[self._image_h, self._image_w], input_size=[height, width] + boxes=boxes, classes=classes, + output_size=[self._output_dims, self._output_dims], + input_size=[self._image_h, self._image_w] ) labels.update({'bbox': boxes}) diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 4cfee4a07..74c587a7c 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -31,9 +31,13 @@ def build_inputs(self, params, input_context=None): model = self.task_config.model parser = centernet_input.CenterNetParser( - num_classes=model.num_classes, - max_num_instances=model.max_num_instances, - gaussian_iou=model.gaussian_iou, + image_w=params.parser.image_w, + image_h=params.parser.image_h, + num_classes=model.num_classes, + max_num_instances=params.parser.max_num_instances, + gaussian_iou=params.parser.gaussian_iou, + output_dims=params.parser.output_dims, + dtype=params.parser.dtype ) reader = input_reader.InputReader( @@ -109,6 +113,8 @@ def get_decoder(self, params): def build_losses(self, outputs, labels, aux_losses=None): + print('\n\nOUTPUT: ', outputs, '\n\n') + print('\n\nLABEL: ', labels, '\n\n') total_loss = 0.0 total_scale_loss = 0.0 total_offset_loss = 0.0 diff --git a/centernet/train.py b/centernet/train.py index 5860accfa..f9f9bdbc3 100755 --- a/centernet/train.py +++ b/centernet/train.py @@ -54,7 +54,7 @@ nohup python3 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov4- --config_file=yolo/configs/experiments/yolov4-1gpu.yaml >> yolov4-1gpu.log & tail -f yolov4-1gpu.log nohup python3.8 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov3-1gpu_mosaic --config_file=yolo/configs/experiments/yolov3-1gpu.yaml >> yolov3-1gpu.log & tail -f yolov3-1gpu.log evalaute CenterNet: -nohup python -m centernet.train --mode=train_and_eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval.yaml >> centernet-eval.log & tail -f centernet-eval.log +nohup python -m centernet.train --mode=eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval.yaml >> centernet-eval.log & tail -f centernet-eval.log """ From 8a1ab85793b21782f46ae4660eee801b7183db1e Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 9 Apr 2021 02:13:41 -0400 Subject: [PATCH 084/132] added box indices and masks to build_label --- centernet/configs/centernet.py | 2 +- .../configs/experiments/centernet-eval.yaml | 4 +- centernet/dataloaders/centernet_input.py | 197 ++++++++++------- centernet/dataloaders/centernet_input_test.py | 203 +++++++----------- centernet/tasks/centernet.py | 68 +++--- centernet/utils/weight_utils/load_weights.py | 2 +- 6 files changed, 237 insertions(+), 239 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index afbc1073c..8b62e5818 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -99,7 +99,7 @@ class Parser(hyperparams.Config): image_w: int = 512 image_h: int = 512 num_classes: int = 90 - max_num_instances: int = 200 + max_num_instances: int = 128 gaussian_iou: float = 0.7 output_dims: int = 128 dtype: str = 'float32' diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index 840f34e2f..930fbd2bc 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -26,7 +26,7 @@ task: class_offset: 1 validation_data: tfds_data_dir: 'D:\\Datasets' - tfds_name: coco + tfds_name: coco/2017 tfds_split: validation global_batch_size: 8 tfds_download: true @@ -37,7 +37,7 @@ task: image_h: 512 image_w: 512 num_classes: 90 - max_num_instances: 200 + max_num_instances: 128 gaussian_iou: 0.7 output_dims: 128 dtype: float32 diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index de218668d..4351eb4fe 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -7,13 +7,13 @@ class CenterNetParser(parser.Parser): def __init__(self, - image_w: int = 512, - image_h: int = 512, - num_classes: int = 90, - max_num_instances: int = 200, # 200 or 100? - gaussian_iou: float = 0.7, - output_dims: int = 128, # should be 128 for all models - dtype: str = 'float32'): + image_w: int = 512, + image_h: int = 512, + num_classes: int = 90, + max_num_instances: int = 128, # 200 or 128? + gaussian_iou: float = 0.7, + output_dims: int = 128, + dtype: str = 'float32'): self._image_w = image_w self._image_h = image_h @@ -34,45 +34,89 @@ def __init__(self, raise Exception( 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' ) - - def _generate_heatmap(self, boxes, classes, output_size, input_size): + + def _build_labels(self, boxes, classes, output_size, input_size): + """ Generates the ground truth labels for centernet. + + Ground truth labels are generated by splatting gaussians on heatmaps for + corners and centers. Regressed features (offsets and sizes) are also + generated. + + Args: + boxes: Tensor of shape [num_boxes, 4], where the last dimension + corresponds to the top left x, top left y, bottom right x, and + bottom left y coordinates of the bounding box + classes: Tensor of shape [num_boxes], that contains the class of each + box, given in the same order as the boxes + output_size: A list containing the desired output height and width of the + heatmaps + input_size: A list the expected input height and width of the image + Returns: + Dictionary of labels with the following fields: + 'tl_heatmaps': Tensor of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the top left location and class of the object + 'br_heatmaps': Tensor of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the bottom right location and class of the object + 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the center location and class of the object + 'tl_offset': Tensor of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the top-left + corner of an object. All other entires are 0 + 'br_offset': Tensor of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the + bottom-right corner of an object. All other entires are 0 + 'ct_offset': Tensor of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the center of + an object. All other entires are 0 + 'size': Tensor of shape [max_num_instances, 2], where the first + num_boxes entries contain the width and height of an object. All + other entires are 0 + 'mask_indices': Tensor of shape [max_num_instances], where the first + num_boxes entries are 1. All other entires are 0 + 'box_indices': Tensor of shape [max_num_instances, 3], where the first + num_boxes entries contain the class, y-center, and-x center of a + valid box. These are referenced to extract the regressed box features + from the prediction when computing the loss. + """ boxes = tf.cast(boxes, dtype=tf.float32) classes = tf.cast(classes, dtype=tf.float32) + input_h, input_w = input_size + output_h, output_w = output_size + + # We will transpose these at the end + tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=tf.float32) + br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=tf.float32) + ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=tf.float32) - tl_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - br_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) - ct_heatmaps = tf.zeros((self._num_classes, output_size[0], output_size[1]), dtype=tf.float32) tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - tl_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - br_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - ct_size = tf.zeros((self._max_num_instances), dtype=tf.int64) - tag_masks = tf.zeros((self._max_num_instances), dtype=tf.uint8) + size = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - width_ratio = output_size[1] / input_size[1] - height_ratio = output_size[0] / input_size[0] + mask_indices = tf.zeros((self._max_num_instances), dtype=tf.uint32) + box_indices = tf.zeros((self._max_num_instances, 3), dtype=tf.uint32) - width_ratio = tf.cast(width_ratio, tf.float32) - height_ratio = tf.cast(height_ratio, tf.float32) + # Scaling factor for determining center/corners + width_ratio = tf.cast(output_w / input_w, tf.float32) + height_ratio = tf.cast(output_h / input_h, tf.float32) num_boxes = tf.shape(boxes)[0] for tag_ind in tf.range(num_boxes): - detection = boxes[tag_ind] - category = classes[tag_ind] + box = boxes[tag_ind] + obj_class = classes[tag_ind] # TODO: See if subtracting 1 from the class like the paper is unnecessary - # category = detection[-1] # TODO: See if subtracting 1 from the class like the paper is unnecessary - # category = 0 # FIXME: For testing only - - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] + ytl, xtl, ybr, xbr = box[0], box[1], box[2], box[3] xct, yct = ( - (detection[2] + detection[0]) / 2, - (detection[3] + detection[1]) / 2 + (xtl + xbr) / 2, + (ytl + ybr) / 2 ) + # Scale center and corner locations fxtl = (xtl * width_ratio) fytl = (ytl * height_ratio) fxbr = (xbr * width_ratio) @@ -80,16 +124,18 @@ def _generate_heatmap(self, boxes, classes, output_size, input_size): fxct = (xct * width_ratio) fyct = (yct * height_ratio) + # Fit center and corners onto the output image xtl = tf.math.floor(fxtl) ytl = tf.math.floor(fytl) xbr = tf.math.floor(fxbr) ybr = tf.math.floor(fybr) xct = tf.math.floor(fxct) yct = tf.math.floor(fyct) - + + # Splat gaussian at for the center/corner heatmaps if self._gaussian_bump: - width = detection[2] - detection[0] - height = detection[3] - detection[1] + width = box[3] - box[1] + height = box[2] - box[0] width = tf.math.ceil(width * width_ratio) height = tf.math.ceil(height * height_ratio) @@ -100,46 +146,42 @@ def _generate_heatmap(self, boxes, classes, output_size, input_size): else: radius = self._gaussian_rad - # test - # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[category], category, [xtl, ytl], radius) - # inputs heatmap, center, radius, k=1 - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[category, xtl, ytl, radius]]) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[category, xbr, ybr, radius]]) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[category, xct, yct, radius]], scaling_factor=5) + tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[obj_class, xtl, ytl, radius]]) + br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[obj_class, xbr, ybr, radius]]) + ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[obj_class, xct, yct, radius]], scaling_factor=5) else: - # TODO: See if this is a typo - # tl_heatmaps[category, ytl, xtl] = 1 - # br_heatmaps[category, ybr, xbr] = 1 - # ct_heatmaps[category, yct, xct] = 1 - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[category, ytl, xtl]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[category, ybr, xbr]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[category, yct, xct]], [1]) - - # tl_offset[tag_ind, :] = [fxtl - xtl, fytl - ytl] - # br_offset[tag_ind, :] = [fxbr - xbr, fybr - ybr] - # ct_offset[tag_ind, :] = [fxct - xct, fyct - yct] - # tl_size[tag_ind] = ytl * output_size[1] + xtl - # br_size[tag_ind] = ybr * output_size[1] + xbr - # ct_size[tag_ind] = yct * output_size[1] + xct + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[obj_class, ytl, xtl]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[obj_class, ybr, xbr]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[obj_class, yct, xct]], [1]) + + # Add box offset and size to the ground truth tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - tl_size = tf.tensor_scatter_nd_update(tl_size, [[tag_ind]], [ytl * output_size[1] + xtl]) - br_size = tf.tensor_scatter_nd_update(br_size, [[tag_ind]], [ybr * output_size[1] + xbr]) - ct_size = tf.tensor_scatter_nd_update(ct_size, [[tag_ind]], [yct * output_size[1] + xct]) + size = tf.tensor_scatter_nd_update(size, [[tag_ind, 0], [tag_ind, 1]], [width, height]) + + # Initialy the mask is zeros, but each valid box needs to be unmasked + mask_indices = tf.tensor_scatter_nd_update(mask_indices, [[tag_ind]], [1]) + + # Contains the class, y, and x coordinate of the box center in the heatmap + box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1], [tag_ind, 2]], [obj_class, yct, xct]) + + # Make heatmaps of shape [height, width, num_classes] + tl_heatmaps = tf.transpose(tl_heatmaps, perm=[1, 2, 0]) + br_heatmaps = tf.transpose(br_heatmaps, perm=[1, 2, 0]) + ct_heatmaps = tf.transpose(ct_heatmaps, perm=[1, 2, 0]) labels = { - 'tl_size': tl_size, - 'br_size': br_size, - 'ct_size': ct_size, 'tl_heatmaps': tl_heatmaps, 'br_heatmaps': br_heatmaps, 'ct_heatmaps': ct_heatmaps, - 'tag_masks': tag_masks, 'tl_offset': tl_offset, 'br_offset': br_offset, 'ct_offset': ct_offset, + 'size': size, + 'mask_indices': mask_indices, + 'box_indices': box_indices } return labels @@ -166,7 +208,7 @@ def _parse_train_data(self, data): height = shape[0] width = shape[1] - labels = self._generate_heatmap( + labels = self._build_labels( boxes=boxes, classes=classes, output_size=[self._output_dims, self._output_dims], input_size=[self._image_h, self._image_w] @@ -189,7 +231,7 @@ def _parse_eval_data(self, data): height = shape[0] width = shape[1] - labels = self._generate_heatmap( + labels = self._build_labels( boxes=boxes, classes=classes, output_size=[self._output_dims, self._output_dims], input_size=[self._image_h, self._image_w] @@ -205,35 +247,28 @@ def postprocess_fn(self, is_training): else: return None -class ObjectDetectionTest(tf.test.TestCase): - def generate_heatmaps(self, dectections): - detections = [[ - (10, 30, 15, 17, 0) - ]] - - labels = generate_heatmaps(1, 2, (416, 416), detections) - pass if __name__ == '__main__': # This code is for visualization import matplotlib.pyplot as plt - detections = [ - (10, 300, 15, 370, 0), - (100, 300, 150, 370, 0), - (200, 100, 15, 170, 0), + boxes = [ + (10, 300, 15, 370), + (100, 300, 150, 370), + (200, 100, 15, 170), ] - #labels = tf.function(CenterNetParser(2, 200, 0.7)._generate_heatmap)( - labels = CenterNetParser()._generate_heatmap( - tf.constant(detections, dtype=tf.float32), [416, 416], [416, 416] + classes = (0, 1, 2) + + labels = CenterNetParser()._build_labels( + tf.constant(boxes, dtype=tf.float32), + tf.constant(classes, dtype=tf.float32), + [512, 512], [512, 512] ) tl_heatmaps = labels['tl_heatmaps'] br_heatmaps = labels['br_heatmaps'] ct_heatmaps = labels['ct_heatmaps'] -# tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) - # ct_heatmaps[batch_id, class_id, ...] - plt.imshow(ct_heatmaps[0, ...]) + # plt.imshow(ct_heatmaps[0, ...]) + plt.imshow(ct_heatmaps[..., 1]) + # plt.imshow(ct_heatmaps[2, ...]) plt.show() - # This is to run the test - # tf.test.main() diff --git a/centernet/dataloaders/centernet_input_test.py b/centernet/dataloaders/centernet_input_test.py index 60e2cd85f..dd8abdd65 100755 --- a/centernet/dataloaders/centernet_input_test.py +++ b/centernet/dataloaders/centernet_input_test.py @@ -1,124 +1,85 @@ -import dataclasses - -import matplotlib.pyplot as plt import tensorflow as tf - -from official.core import config_definitions as cfg -from official.core import input_reader -from official.modeling import hyperparams -from yolo.configs import yolo as yolocfg -from yolo.dataloaders import yolo_input as YOLO_Detection_Input -from yolo.dataloaders.decoders import tfds_coco_decoder -from yolo.ops import box_ops -from yolo.tasks import yolo - - -@dataclasses.dataclass -class Parser(hyperparams.Config): - image_w: int = 416 - image_h: int = 416 - fixed_size: bool = False - jitter_im: float = 0.1 - jitter_boxes: float = 0.005 - min_level: int = 3 - max_level: int = 5 - min_process_size: int = 320 - max_process_size: int = 608 - max_num_instances: int = 200 - random_flip: bool = True - pct_rand: float = 0.5 - aug_rand_saturation: bool = True - aug_rand_brightness: bool = True - aug_rand_zoom: bool = True - aug_rand_hue: bool = True - seed: int = 10 - shuffle_buffer_size: int = 10000 - - -@dataclasses.dataclass -class DataConfig(cfg.DataConfig): - """Input config for training.""" - input_path: str = '' - tfds_name: str = 'coco' - tfds_split: str = 'train' - global_batch_size: int = 10 - is_training: bool = True - dtype: str = 'float16' - decoder = None - parser: Parser = Parser() - shuffle_buffer_size: int = 10000 - tfds_download: bool = True - - -def test_yolo_input_task(): - with tf.device('/CPU:0'): - config = yolocfg.YoloTask( - model=yolocfg.Yolo( - base='v4', - min_level=3, - norm_activation=yolocfg.common.NormActivation(activation='mish'), - #norm_activation = yolocfg.common.NormActivation(activation="leaky"), - #_boxes = ['(10, 14)', '(23, 27)', '(37, 58)', '(81, 82)', '(135, 169)', '(344, 319)'], - #_boxes = ["(10, 13)", "(16, 30)", "(33, 23)","(30, 61)", "(62, 45)", "(59, 119)","(116, 90)", "(156, 198)", "(373, 326)"], - _boxes=[ - '(12, 16)', '(19, 36)', '(40, 28)', '(36, 75)', '(76, 55)', - '(72, 146)', '(142, 110)', '(192, 243)', '(459, 401)' - ], - filter=yolocfg.YoloLossLayer(use_nms=False))) - task = yolo.YoloTask(config) - - # loading both causes issues, but oen at a time is not issue, why? - train_data = task.build_inputs(config.train_data) - test_data = task.build_inputs(config.validation_data) - return train_data, test_data - - -def test_yolo_input(): - with tf.device('/CPU:0'): - params = DataConfig(is_training=True) - num_boxes = 9 - - decoder = tfds_coco_decoder.MSCOCODecoder() - - #anchors = box_rd.read(k = num_boxes, image_width = params.parser.image_w, input_context=None) - anchors = [[12.0, 19.0], [31.0, 46.0], [96.0, 54.0], [46.0, 114.0], - [133.0, 127.0], [79.0, 225.0], [301.0, 150.0], [172.0, 286.0], - [348.0, 340.0]] - # write the boxes to a file - - parser = YOLO_Detection_Input.Parser( - image_w=params.parser.image_w, - fixed_size=params.parser.fixed_size, - jitter_im=params.parser.jitter_im, - jitter_boxes=params.parser.jitter_boxes, - min_level=params.parser.min_level, - max_level=params.parser.max_level, - min_process_size=params.parser.min_process_size, - max_process_size=params.parser.max_process_size, - max_num_instances=params.parser.max_num_instances, - random_flip=params.parser.random_flip, - pct_rand=params.parser.pct_rand, - seed=params.parser.seed, - anchors=anchors) - - reader = input_reader.InputReader( - params, - dataset_fn=tf.data.TFRecordDataset, - decoder_fn=decoder.decode, - parser_fn=parser.parse_fn(params.is_training)) - dataset = reader.read(input_context=None) - return dataset - +from absl.testing import parameterized + +from centernet.dataloaders.centernet_input import CenterNetParser + + +class CenterNetInputTest(tf.test.TestCase, parameterized.TestCase): + def check_labels_correct(self, boxes, classes, output_size, input_size): + parser = CenterNetParser() + labels = parser._build_labels( + boxes=tf.constant(boxes, dtype=tf.float32), + classes=tf.constant(classes, dtype=tf.float32), + output_size=output_size, input_size=input_size) + + tl_heatmaps = labels['tl_heatmaps'] + br_heatmaps = labels['br_heatmaps'] + ct_heatmaps = labels['ct_heatmaps'] + tl_offset = labels['tl_offset'] + br_offset = labels['br_offset'] + ct_offset = labels['ct_offset'] + size = labels['size'] + mask_indices = labels['mask_indices'] + box_indices = labels['box_indices'] + + boxes = tf.cast(boxes, tf.float32) + classes = tf.cast(classes, tf.float32) + height_ratio = output_size[0] / input_size[0] + width_ratio = output_size[1] / input_size[1] + + # Shape checks + self.assertEqual(tl_heatmaps.shape, (512, 512, 90)) + self.assertEqual(br_heatmaps.shape, (512, 512, 90)) + self.assertEqual(ct_heatmaps.shape, (512, 512, 90)) + + self.assertEqual(tl_offset.shape, (parser._max_num_instances, 2)) + self.assertEqual(br_offset.shape, (parser._max_num_instances, 2)) + self.assertEqual(ct_offset.shape, (parser._max_num_instances, 2)) + + self.assertEqual(size.shape, (parser._max_num_instances, 2)) + self.assertEqual(mask_indices.shape, (parser._max_num_instances)) + self.assertEqual(box_indices.shape, (parser._max_num_instances, 3)) + + # Not checking heatmaps, but we can visually validate them + + for i in range(len(boxes)): + # Check sizes + self.assertAllEqual(size[i], [boxes[i][3] - boxes[i][1], boxes[i][2] - boxes[i][0]]) + + # Check box indices + y = tf.math.floor((boxes[i][0] + boxes[i][2]) / 2 * height_ratio) + x = tf.math.floor((boxes[i][1] + boxes[i][3]) / 2 * width_ratio) + self.assertAllEqual(box_indices[i], [classes[i], y, x]) + + # check offsets + true_y = (boxes[i][0] + boxes[i][2]) / 2 * height_ratio + true_x = (boxes[i][1] + boxes[i][3]) / 2 * width_ratio + self.assertAllEqual(ct_offset[i], [true_x - x, true_y - y]) + + for i in range(len(boxes), parser._max_num_instances): + # Make sure rest are zero + self.assertAllEqual(size[i], [0, 0]) + self.assertAllEqual(box_indices[i], [0, 0, 0]) + self.assertAllEqual(ct_offset[i], [0, 0]) + + # Check mask indices + self.assertAllEqual(tf.cast(mask_indices[3:], tf.int32), + tf.repeat(0, repeats=parser._max_num_instances-3)) + self.assertAllEqual(tf.cast(mask_indices[:3], tf.int32), + tf.repeat(1, repeats=3)) + + + def test_generate_heatmap_no_scale(self): + boxes = [ + (10, 300, 15, 370), + (100, 300, 150, 370), + (15, 100, 200, 170), + ] + classes = (0, 1, 2) + sizes = [512, 512] + + self.check_labels_correct(boxes=boxes, classes=classes, + output_size=sizes, input_size=sizes) if __name__ == '__main__': - dataset, dsp = test_yolo_input_task() - - for l, (i, j) in enumerate(dataset): - - boxes = box_ops.xcycwh_to_yxyx(j['bbox']) - i = tf.image.draw_bounding_boxes(i, boxes, [[1.0, 0.0, 1.0]]) - plt.imshow(i[0].numpy()) - plt.show() - - if l > 30: - break + tf.test.main() diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 74c587a7c..8c7a75e83 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -12,6 +12,7 @@ tf_example_label_map_decoder, tfds_detection_decoders) from official.vision.beta.evaluation import coco_evaluator +from official.vision.beta.ops import box_ops @task_factory.register_task_cls(exp_cfg.CenterNetTask) @@ -113,8 +114,6 @@ def get_decoder(self, params): def build_losses(self, outputs, labels, aux_losses=None): - print('\n\nOUTPUT: ', outputs, '\n\n') - print('\n\nLABEL: ', labels, '\n\n') total_loss = 0.0 total_scale_loss = 0.0 total_offset_loss = 0.0 @@ -124,36 +123,37 @@ def build_losses(self, outputs, labels, aux_losses=None): metric_dict = dict() - # TODO: Calculate loss - flattened_ct_heatmaps = loss_ops._flatten_spatial_dimensions(labels['ct_heatmaps']) - num_boxes = loss_ops._to_float32(loss_ops.get_num_instances_from_weights(labels['tag_masks'])) - - object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) - - output_ct_heatmaps = loss_ops._flatten_spatial_dimensions(outputs['ct_heatmaps'][-1]) - total_loss += object_center_loss( - flattened_ct_heatmaps, output_ct_heatmaps) #removed weight parameter (weight = per_pixel_weight) - center_loss = tf.reduce_sum(total_loss) / ( - float(len(output_ct_heatmaps)) * num_boxes) - loss += center_loss - metric_dict['ct_loss'] = center_loss - - localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) - # Compute the scale loss. - scale_pred = outputs['ct_size'][-1] - offset_pred = outputs['ct_offset'][-1] - total_scale_loss += localization_loss_fn( - labels['ct_size'], scale_pred) #removed weights=batch_weights - # Compute the offset loss. - total_offset_loss += localization_loss_fn( - labels['ct_offset'], offset_pred) #removed weights=batch_weights - scale_loss += tf.reduce_sum(total_scale_loss) / ( - float(len(outputs['ct_size'])) * num_boxes) - offset_loss += tf.reduce_sum(total_offset_loss) / ( - float(len(outputs['ct_size'])) * num_boxes) - metric_dict['ct_scale_loss'] = scale_loss - metric_dict['ct_offset_loss'] = offset_loss - + # # TODO: Calculate loss + # flattened_ct_heatmaps = loss_ops._flatten_spatial_dimensions(labels['ct_heatmaps']) + # num_boxes = loss_ops._to_float32(loss_ops.get_num_instances_from_weights(labels['tag_masks'])) + + # object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) + + # output_ct_heatmaps = loss_ops._flatten_spatial_dimensions(outputs['ct_heatmaps'][-1]) + # total_loss += object_center_loss( + # flattened_ct_heatmaps, output_ct_heatmaps) #removed weight parameter (weight = per_pixel_weight) + # center_loss = tf.reduce_sum(total_loss) / ( + # float(len(output_ct_heatmaps)) * num_boxes) + # loss += center_loss + # metric_dict['ct_loss'] = center_loss + + # localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) + # # Compute the scale loss. + # scale_pred = outputs['ct_size'][-1] + # offset_pred = outputs['ct_offset'][-1] + # total_scale_loss += localization_loss_fn( + # labels['ct_size'], scale_pred) #removed weights=batch_weights + # # Compute the offset loss. + # total_offset_loss += localization_loss_fn( + # labels['ct_offset'], offset_pred) #removed weights=batch_weights + # scale_loss += tf.reduce_sum(total_scale_loss) / ( + # float(len(outputs['ct_size'])) * num_boxes) + # offset_loss += tf.reduce_sum(total_offset_loss) / ( + # float(len(outputs['ct_size'])) * num_boxes) + # metric_dict['ct_scale_loss'] = scale_loss + # metric_dict['ct_offset_loss'] = offset_loss + + metric_dict['total_loss'] = loss return loss, metric_dict def build_metrics(self, training=True): @@ -177,9 +177,11 @@ def validation_step(self, inputs, model, metrics=None): # computer detivative and apply gradients y_pred = model(image, training=False) y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) - loss_metrics = self.build_losses(y_pred['raw_output'], label) + loss, loss_metrics = self.build_losses(y_pred['raw_output'], label) logs = {self.loss: loss_metrics['total_loss']} + image_shape = tf.shape(image)[1:-1] + coco_model_outputs = { 'detection_boxes': box_ops.denormalize_boxes( diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index 799310037..916380fa4 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -98,7 +98,7 @@ def get_decoder_layer_cfgs(weights_dict, decoder_name): Returns: A list containing the config classes of the backbone building block """ - print("Fetching decoder config classes\n") + print("Fetching decoder config classes for {}\n".format(decoder_name)) cfgs = DecoderConfigData(weights_dict=weights_dict).get_cfg_list(decoder_name) return cfgs From 424959a696b17b230b746bc354e8402fdba84eaf Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 9 Apr 2021 05:35:01 -0400 Subject: [PATCH 085/132] new loss and ground truth --- centernet/configs/centernet.py | 3 +- .../configs/experiments/centernet-eval.yaml | 2 +- centernet/dataloaders/centernet_input.py | 21 ++-- centernet/dataloaders/centernet_input_test.py | 14 +-- centernet/ops/loss_ops.py | 41 +++++++ centernet/tasks/centernet.py | 115 +++++++++++------- 6 files changed, 133 insertions(+), 63 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 8b62e5818..e98578233 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -143,7 +143,8 @@ class DetectionLoss(Loss): detection_weight: float = 1.0 corner_pull_weight: float = 0.1 # alpha corner_push_weight: float = 0.1 # beta - offset_weight: float = 1 # gamma + offset_weight: float = 1.0 # gamma + scale_weight: float = 0.1 @dataclasses.dataclass diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index 930fbd2bc..84180b187 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -28,7 +28,7 @@ task: tfds_data_dir: 'D:\\Datasets' tfds_name: coco/2017 tfds_split: validation - global_batch_size: 8 + global_batch_size: 1 tfds_download: true dtype: float16 is_training: false diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 4351eb4fe..73bc33cc9 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -74,7 +74,7 @@ def _build_labels(self, boxes, classes, output_size, input_size): 'size': Tensor of shape [max_num_instances, 2], where the first num_boxes entries contain the width and height of an object. All other entires are 0 - 'mask_indices': Tensor of shape [max_num_instances], where the first + 'box_mask': Tensor of shape [max_num_instances], where the first num_boxes entries are 1. All other entires are 0 'box_indices': Tensor of shape [max_num_instances, 3], where the first num_boxes entries contain the class, y-center, and-x center of a @@ -96,8 +96,8 @@ def _build_labels(self, boxes, classes, output_size, input_size): ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) size = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - mask_indices = tf.zeros((self._max_num_instances), dtype=tf.uint32) - box_indices = tf.zeros((self._max_num_instances, 3), dtype=tf.uint32) + box_mask = tf.zeros((self._max_num_instances), dtype=tf.int32) + box_indices = tf.zeros((self._max_num_instances, 2), dtype=tf.int32) # Scaling factor for determining center/corners width_ratio = tf.cast(output_w / input_w, tf.float32) @@ -162,10 +162,10 @@ def _build_labels(self, boxes, classes, output_size, input_size): size = tf.tensor_scatter_nd_update(size, [[tag_ind, 0], [tag_ind, 1]], [width, height]) # Initialy the mask is zeros, but each valid box needs to be unmasked - mask_indices = tf.tensor_scatter_nd_update(mask_indices, [[tag_ind]], [1]) + box_mask = tf.tensor_scatter_nd_update(box_mask, [[tag_ind]], [1]) - # Contains the class, y, and x coordinate of the box center in the heatmap - box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1], [tag_ind, 2]], [obj_class, yct, xct]) + # Contains the y and x coordinate of the box center in the heatmap + box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1]], [yct, xct]) # Make heatmaps of shape [height, width, num_classes] tl_heatmaps = tf.transpose(tl_heatmaps, perm=[1, 2, 0]) @@ -180,8 +180,8 @@ def _build_labels(self, boxes, classes, output_size, input_size): 'br_offset': br_offset, 'ct_offset': ct_offset, 'size': size, - 'mask_indices': mask_indices, - 'box_indices': box_indices + 'box_mask': box_mask, + 'box_indices': box_indices, } return labels @@ -238,6 +238,11 @@ def _parse_eval_data(self, data): ) labels.update({'bbox': boxes}) + labels.update({'boxes': boxes}) + labels.update({'source_id': data['source_id']}) + labels.update({'height': data['height']}) + labels.update({'width': data['width']}) + labels.update({'classes': classes}) return image, labels diff --git a/centernet/dataloaders/centernet_input_test.py b/centernet/dataloaders/centernet_input_test.py index dd8abdd65..fc43aeaa4 100755 --- a/centernet/dataloaders/centernet_input_test.py +++ b/centernet/dataloaders/centernet_input_test.py @@ -19,7 +19,7 @@ def check_labels_correct(self, boxes, classes, output_size, input_size): br_offset = labels['br_offset'] ct_offset = labels['ct_offset'] size = labels['size'] - mask_indices = labels['mask_indices'] + box_mask = labels['box_mask'] box_indices = labels['box_indices'] boxes = tf.cast(boxes, tf.float32) @@ -37,8 +37,8 @@ def check_labels_correct(self, boxes, classes, output_size, input_size): self.assertEqual(ct_offset.shape, (parser._max_num_instances, 2)) self.assertEqual(size.shape, (parser._max_num_instances, 2)) - self.assertEqual(mask_indices.shape, (parser._max_num_instances)) - self.assertEqual(box_indices.shape, (parser._max_num_instances, 3)) + self.assertEqual(box_mask.shape, (parser._max_num_instances)) + self.assertEqual(box_indices.shape, (parser._max_num_instances, 2)) # Not checking heatmaps, but we can visually validate them @@ -49,7 +49,7 @@ def check_labels_correct(self, boxes, classes, output_size, input_size): # Check box indices y = tf.math.floor((boxes[i][0] + boxes[i][2]) / 2 * height_ratio) x = tf.math.floor((boxes[i][1] + boxes[i][3]) / 2 * width_ratio) - self.assertAllEqual(box_indices[i], [classes[i], y, x]) + self.assertAllEqual(box_indices[i], [y, x]) # check offsets true_y = (boxes[i][0] + boxes[i][2]) / 2 * height_ratio @@ -59,13 +59,13 @@ def check_labels_correct(self, boxes, classes, output_size, input_size): for i in range(len(boxes), parser._max_num_instances): # Make sure rest are zero self.assertAllEqual(size[i], [0, 0]) - self.assertAllEqual(box_indices[i], [0, 0, 0]) + self.assertAllEqual(box_indices[i], [0, 0]) self.assertAllEqual(ct_offset[i], [0, 0]) # Check mask indices - self.assertAllEqual(tf.cast(mask_indices[3:], tf.int32), + self.assertAllEqual(tf.cast(box_mask[3:], tf.int32), tf.repeat(0, repeats=parser._max_num_instances-3)) - self.assertAllEqual(tf.cast(mask_indices[:3], tf.int32), + self.assertAllEqual(tf.cast(box_mask[:3], tf.int32), tf.repeat(1, repeats=3)) diff --git a/centernet/ops/loss_ops.py b/centernet/ops/loss_ops.py index a05cf3a26..b0f25c4fe 100644 --- a/centernet/ops/loss_ops.py +++ b/centernet/ops/loss_ops.py @@ -59,6 +59,35 @@ def get_num_instances_from_weights(groundtruth_weights_list): num_instances = tf.maximum(num_instances, 1) return num_instances +def multi_range(limit, + value_repetitions=1, + range_repetitions=1, + dtype=tf.int32): + """ Creates a sequence with optional value duplication and range repetition. + + As an example (see the Args section for more details), + _multi_range(limit=2, value_repetitions=3, range_repetitions=4) returns: + [0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1] + NOTE: Repurposed from Google OD API. + + Args: + limit: A 0-D Tensor (scalar). Upper limit of sequence, exclusive. + value_repetitions: Integer. The number of times a value in the sequence is + repeated. With value_repetitions=3, the result is [0, 0, 0, 1, 1, 1, ..]. + range_repetitions: Integer. The number of times the range is repeated. With + range_repetitions=3, the result is [0, 1, 2, .., 0, 1, 2, ..]. + dtype: The type of the elements of the resulting tensor. + + Returns: + A 1-D tensor of type `dtype` and size + [`limit` * `value_repetitions` * `range_repetitions`] that contains the + specified range with given repetitions. + """ + return tf.reshape( + tf.tile( + tf.expand_dims(tf.range(limit, dtype=dtype), axis=-1), + multiples=[range_repetitions, value_repetitions]), [-1]) + def get_batch_predictions_from_indices(batch_predictions, indices): """Gets the values of predictions in a batch at the given indices. The indices are expected to come from the offset targets generation functions @@ -74,4 +103,16 @@ def get_batch_predictions_from_indices(batch_predictions, indices): values: A tensor of shape [num_instances, channels] holding the predicted values at the given indices. """ + # indices right now is shape (8, 128, 2), we need to make it (8, 128, 3), where + # the last dimension corresponds to the batch it belongs t + return tf.gather_nd(batch_predictions, indices) + +def add_batch_to_indices(indices): + shape = tf.shape(indices) + batch_size = shape[0] + num_instances = shape[1] + batch_range = multi_range(limit=batch_size, value_repetitions=num_instances) + batch_range = tf.reshape(batch_range, shape=(batch_size, num_instances, 1)) + + return tf.concat([indices, batch_range], axis=-1) diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 8c7a75e83..224292f9c 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -113,48 +113,70 @@ def get_decoder(self, params): return decoder - def build_losses(self, outputs, labels, aux_losses=None): + def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): total_loss = 0.0 - total_scale_loss = 0.0 - total_offset_loss = 0.0 loss = 0.0 - scale_loss = 0.0 - offset_loss = 0.0 metric_dict = dict() - # # TODO: Calculate loss - # flattened_ct_heatmaps = loss_ops._flatten_spatial_dimensions(labels['ct_heatmaps']) - # num_boxes = loss_ops._to_float32(loss_ops.get_num_instances_from_weights(labels['tag_masks'])) - - # object_center_loss = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) - - # output_ct_heatmaps = loss_ops._flatten_spatial_dimensions(outputs['ct_heatmaps'][-1]) - # total_loss += object_center_loss( - # flattened_ct_heatmaps, output_ct_heatmaps) #removed weight parameter (weight = per_pixel_weight) - # center_loss = tf.reduce_sum(total_loss) / ( - # float(len(output_ct_heatmaps)) * num_boxes) - # loss += center_loss - # metric_dict['ct_loss'] = center_loss - - # localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) - # # Compute the scale loss. - # scale_pred = outputs['ct_size'][-1] - # offset_pred = outputs['ct_offset'][-1] - # total_scale_loss += localization_loss_fn( - # labels['ct_size'], scale_pred) #removed weights=batch_weights - # # Compute the offset loss. - # total_offset_loss += localization_loss_fn( - # labels['ct_offset'], offset_pred) #removed weights=batch_weights - # scale_loss += tf.reduce_sum(total_scale_loss) / ( - # float(len(outputs['ct_size'])) * num_boxes) - # offset_loss += tf.reduce_sum(total_offset_loss) / ( - # float(len(outputs['ct_size'])) * num_boxes) - # metric_dict['ct_scale_loss'] = scale_loss - # metric_dict['ct_offset_loss'] = offset_loss - - metric_dict['total_loss'] = loss - return loss, metric_dict + # Create loss functions + object_center_loss_fn = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) + localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) + + # Set up box indicies so that they have a batch element as well + box_indices = loss_ops.add_batch_to_indices(labels['box_indices']) + + box_mask = tf.cast(labels['box_mask'], dtype=tf.float32) + num_boxes = loss_ops._to_float32(loss_ops.get_num_instances_from_weights(labels['box_mask'])) + + # Calculate center heatmap loss + pred_ct_heatmap_list = outputs['ct_heatmaps'] + true_flattened_ct_heatmap = loss_ops._flatten_spatial_dimensions( + labels['ct_heatmaps']) + + total_center_loss = 0.0 + for ct_heatmap in pred_ct_heatmap_list: + pred_flattened_ct_heatmap = loss_ops._flatten_spatial_dimensions( + ct_heatmap) + total_center_loss += object_center_loss_fn( + pred_flattened_ct_heatmap, true_flattened_ct_heatmap) + + center_loss = tf.reduce_sum(total_center_loss) / float(len(pred_ct_heatmap_list) * num_boxes) + metric_dict['ct_loss'] = center_loss + + # Calculate scale loss + pred_scale_list = outputs['ct_size'] + true_scale = labels['size'] + + total_scale_loss = 0.0 + for scale_map in pred_scale_list: + pred_scale = loss_ops.get_batch_predictions_from_indices(scale_map, box_indices) + # Only apply loss for boxes that appear in the ground truth + total_scale_loss += tf.reduce_sum(localization_loss_fn(pred_scale, true_scale), axis=-1) * box_mask + + scale_loss = tf.reduce_sum(total_scale_loss) / float(len(pred_scale_list) * num_boxes) + metric_dict['scale_loss'] = scale_loss + + # Calculate offset loss + pred_offset_list = outputs['ct_offset'] + true_offset = labels['ct_offset'] + + total_offset_loss = 0.0 + for offset_map in pred_offset_list: + pred_offset = loss_ops.get_batch_predictions_from_indices(offset_map, box_indices) + # Only apply loss for boxes that appear in the ground truth + total_offset_loss += tf.reduce_sum(localization_loss_fn(pred_offset, true_offset), axis=-1) * box_mask + + offset_loss = tf.reduce_sum(total_offset_loss) / float(len(pred_offset_list) * num_boxes) + metric_dict['ct_offset_loss'] = offset_loss + + loss_weights = self.task_config.losses.detection + total_loss = (center_loss + + loss_weights.scale_weight * scale_loss + + loss_weights.offset_weight * offset_loss) + + metric_dict['total_loss'] = total_loss + return total_loss, metric_dict def build_metrics(self, training=True): metrics = [] @@ -191,7 +213,9 @@ def validation_step(self, inputs, model, metrics=None): 'detection_classes': y_pred['classes'], 'num_detections': - tf.shape(y_pred['bbox'])[:-1], + y_pred['num_dets'], + 'source_id': + label['source_id'], } logs.update({self.coco_metric.name: (label, coco_model_outputs)}) @@ -204,13 +228,12 @@ def validation_step(self, inputs, model, metrics=None): return logs def aggregate_logs(self, state=None, step_outputs=None): - pass + if not state: + self.coco_metric.reset_states() + state = self.coco_metric + self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], + step_outputs[self.coco_metric.name][1]) + return state def reduce_aggregated_logs(self, aggregated_logs): - pass - - def _get_masks(self, - xy_exponential=True, - exp_base=2, - xy_scale_base='default_value'): - pass + return self.coco_metric.result() From 24ed6eb74c4e58f0b66618cd028e7abbdf0e67d1 Mon Sep 17 00:00:00 2001 From: Feny Patel Date: Sat, 10 Apr 2021 19:46:37 -0400 Subject: [PATCH 086/132] build_metrics --- centernet/tasks/centernet.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 224292f9c..80ab0f3ea 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -180,7 +180,12 @@ def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): def build_metrics(self, training=True): metrics = [] + metric_names = self._metric_names + for name in metric_names: + metrics.append(tf.keras.metrics.Mean(name, dtype=tf.float32)) + + self._metrics = metrics if not training: self.coco_metric = coco_evaluator.COCOEvaluator( annotation_file=self.task_config.annotation_file, From 7677482d886d6bca1a8c74555b1c6386946d288e Mon Sep 17 00:00:00 2001 From: David Li Date: Sun, 11 Apr 2021 23:56:46 -0400 Subject: [PATCH 087/132] eval running, can try on vm --- centernet/configs/centernet.py | 19 +- .../configs/experiments/centernet-eval.yaml | 2 + centernet/dataloaders/centernet_input.py | 176 +++++++++++++----- .../modeling/decoders/centernet_decoder.py | 26 --- centernet/tasks/centernet.py | 5 +- 5 files changed, 140 insertions(+), 88 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index e98578233..5aee4fe26 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -100,26 +100,11 @@ class Parser(hyperparams.Config): image_h: int = 512 num_classes: int = 90 max_num_instances: int = 128 + use_gaussian_bump: bool = True + gaussian_rad: int = -1 gaussian_iou: float = 0.7 output_dims: int = 128 dtype: str = 'float32' - # fixed_size: bool = True - # max_num_instances: int = 200 - # min_process_size: int = 320 - # max_process_size: int = 608 - # letter_box: bool = True - # random_flip: bool = True - # pct_rand: float = 0.0 - # jitter_im: float = 0.0 - # jitter_boxes: float = 0.000 - # aug_rand_transalate: float = 0.0 - # aug_rand_saturation: float = 0.0 - # aug_rand_brightness: float = 0.0 - # aug_rand_zoom: float = 0.0 - # aug_rand_hue: float = 0.0 - # keep_thresh: float = 0.0 - # mosaic_frequency: float = 1.0 - # use_tie_breaker: bool = True @dataclasses.dataclass class DataConfig(cfg.DataConfig): diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index 84180b187..621b448d0 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -38,6 +38,8 @@ task: image_w: 512 num_classes: 90 max_num_instances: 128 + use_gaussian_bump: true + gaussian_rad: -1 gaussian_iou: 0.7 output_dims: 128 dtype: float32 diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 73bc33cc9..ab8a87742 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -1,28 +1,60 @@ import tensorflow as tf from centernet.ops import preprocessing_ops -from official.vision.beta.dataloaders import parser +from official.vision.beta.dataloaders import parser, utils from yolo.ops import preprocessing_ops as yolo_preprocessing_ops +def pad_max_instances(value, instances, pad_value=0, pad_axis=0): + shape = tf.shape(value) + if pad_axis < 0: + pad_axis = tf.shape(shape)[0] + pad_axis + dim1 = shape[pad_axis] + take = tf.math.reduce_min([instances, dim1]) + value, _ = tf.split( + value, [take, -1], axis=pad_axis) # value[:instances, ...] + pad = tf.convert_to_tensor([tf.math.reduce_max([instances - dim1, 0])]) + nshape = tf.concat([shape[:pad_axis], pad, shape[(pad_axis + 1):]], axis=0) + pad_tensor = tf.fill(nshape, tf.cast(pad_value, dtype=value.dtype)) + value = tf.concat([value, pad_tensor], axis=pad_axis) + return value + class CenterNetParser(parser.Parser): + """ Parser to parse an image and its annotations into a dictionary of tensors """ + def __init__(self, image_w: int = 512, image_h: int = 512, num_classes: int = 90, - max_num_instances: int = 128, # 200 or 128? + max_num_instances: int = 128, + use_gaussian_bump: bool = True, + gaussian_rad: int = -1, gaussian_iou: float = 0.7, output_dims: int = 128, dtype: str = 'float32'): - + """Initializes parameters for parsing annotations in the dataset. + Args: + image_w: A `Tensor` or `int` for width of input image. + image_h: A `Tensor` or `int` for height of input image. + num_classes: A `Tensor` or `int` for the number of classes. + max_num_instances: An `int` number of maximum number of instances in an image. + use_gaussian_bump: A `boolean` indicating whether or not to splat a + gaussian onto the heatmaps. If set to False, a value of 1 is placed at + the would-be center of the gaussian. + gaussian_rad: A `int` for the desired radius of the gaussian. If this + value is set to -1, then the radius is computed using gaussian_iou. + gaussian_iou: A `float` number for the minimum desired IOU used when + determining the gaussian radius of center locations in the heatmap. + output_dims: A `Tensor` or `int` for output dimensions of the heatmap. + """ self._image_w = image_w self._image_h = image_h self._num_classes = num_classes self._max_num_instances = max_num_instances self._gaussian_iou = gaussian_iou - self._output_dims = output_dims - self._gaussian_bump = True + self._use_gaussian_bump = True self._gaussian_rad = -1 + self._output_dims = output_dims if dtype == 'float16': self._dtype = tf.float16 @@ -34,8 +66,13 @@ def __init__(self, raise Exception( 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' ) - - def _build_labels(self, boxes, classes, output_size, input_size): + + def _build_heatmap_and_regressed_features(self, + boxes, + classes, + num_objects, + output_size=[128, 128], + input_size=[512, 512]): """ Generates the ground truth labels for centernet. Ground truth labels are generated by splatting gaussians on heatmaps for @@ -43,43 +80,45 @@ def _build_labels(self, boxes, classes, output_size, input_size): generated. Args: - boxes: Tensor of shape [num_boxes, 4], where the last dimension + boxes: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension corresponds to the top left x, top left y, bottom right x, and bottom left y coordinates of the bounding box - classes: Tensor of shape [num_boxes], that contains the class of each + classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each box, given in the same order as the boxes - output_size: A list containing the desired output height and width of the - heatmaps - input_size: A list the expected input height and width of the image + num_objects: A `Tensor` or int that gives the number of objects + output_size: A `list` of length 2 containing the desired output height + and width of the heatmaps + input_size: A `list` of length 2 the expected input height and width of + the image Returns: Dictionary of labels with the following fields: - 'tl_heatmaps': Tensor of shape [output_h, output_w, num_classes], + 'tl_heatmaps': A `Tensor` of shape [output_h, output_w, num_classes], heatmap with splatted gaussians centered at the positions and channels corresponding to the top left location and class of the object - 'br_heatmaps': Tensor of shape [output_h, output_w, num_classes], + 'br_heatmaps': `Tensor` of shape [output_h, output_w, num_classes], heatmap with splatted gaussians centered at the positions and channels corresponding to the bottom right location and class of the object 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], heatmap with splatted gaussians centered at the positions and channels corresponding to the center location and class of the object - 'tl_offset': Tensor of shape [max_num_instances, 2], where the first + 'tl_offset': `Tensor` of shape [max_num_instances, 2], where the first num_boxes entries contain the x-offset and y-offset of the top-left corner of an object. All other entires are 0 - 'br_offset': Tensor of shape [max_num_instances, 2], where the first + 'br_offset': `Tensor` of shape [max_num_instances, 2], where the first num_boxes entries contain the x-offset and y-offset of the bottom-right corner of an object. All other entires are 0 - 'ct_offset': Tensor of shape [max_num_instances, 2], where the first + 'ct_offset': `Tensor` of shape [max_num_instances, 2], where the first num_boxes entries contain the x-offset and y-offset of the center of an object. All other entires are 0 - 'size': Tensor of shape [max_num_instances, 2], where the first + 'size': `Tensor` of shape [max_num_instances, 2], where the first num_boxes entries contain the width and height of an object. All other entires are 0 - 'box_mask': Tensor of shape [max_num_instances], where the first + 'box_mask': `Tensor` of shape [max_num_instances], where the first num_boxes entries are 1. All other entires are 0 - 'box_indices': Tensor of shape [max_num_instances, 3], where the first - num_boxes entries contain the class, y-center, and-x center of a - valid box. These are referenced to extract the regressed box features - from the prediction when computing the loss. + 'box_indices': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the y-center and x-center of a valid box. + These are used to extract the regressed box features from the + prediction when computing the loss """ boxes = tf.cast(boxes, dtype=tf.float32) classes = tf.cast(classes, dtype=tf.float32) @@ -105,7 +144,9 @@ def _build_labels(self, boxes, classes, output_size, input_size): num_boxes = tf.shape(boxes)[0] - for tag_ind in tf.range(num_boxes): + height = 0.0 + width = 0.0 + for tag_ind in tf.range(num_objects): box = boxes[tag_ind] obj_class = classes[tag_ind] # TODO: See if subtracting 1 from the class like the paper is unnecessary @@ -133,7 +174,7 @@ def _build_labels(self, boxes, classes, output_size, input_size): yct = tf.math.floor(fyct) # Splat gaussian at for the center/corner heatmaps - if self._gaussian_bump: + if self._use_gaussian_bump: width = box[3] - box[1] height = box[2] - box[0] @@ -181,7 +222,7 @@ def _build_labels(self, boxes, classes, output_size, input_size): 'ct_offset': ct_offset, 'size': size, 'box_mask': box_mask, - 'box_indices': box_indices, + 'box_indices': box_indices } return labels @@ -189,7 +230,7 @@ def _parse_train_data(self, data): """Generates images and labels that are usable for model training. Args: - decoded_tensors: a dict of Tensors produced by the decoder. + data: a dict of Tensors produced by the decoder. Returns: images: the image tensor. @@ -208,17 +249,22 @@ def _parse_train_data(self, data): height = shape[0] width = shape[1] - labels = self._build_labels( - boxes=boxes, classes=classes, - output_size=[self._output_dims, self._output_dims], - input_size=[self._image_h, self._image_w] + image, labels = self._build_label( + image, boxes, classes, width, height, info, data, is_training=False ) - - labels.update({'bbox': boxes}) return image, labels def _parse_eval_data(self, data): + """Generates images and labels that are usable for model evaluation. + + Args: + decoded_tensors: a dict of Tensors produced by the decoder. + + Returns: + images: the image tensor. + labels: a dict of Tensors that contains labels. + """ image = data['image'] / 255 boxes = data['groundtruth_boxes'] classes = data['groundtruth_classes'] @@ -231,21 +277,63 @@ def _parse_eval_data(self, data): height = shape[0] width = shape[1] - labels = self._build_labels( - boxes=boxes, classes=classes, - output_size=[self._output_dims, self._output_dims], - input_size=[self._image_h, self._image_w] + image, labels = self._build_label( + image, boxes, classes, width, height, info, data, is_training=False ) - - labels.update({'bbox': boxes}) - labels.update({'boxes': boxes}) - labels.update({'source_id': data['source_id']}) - labels.update({'height': data['height']}) - labels.update({'width': data['width']}) - labels.update({'classes': classes}) return image, labels + def _build_label(self, image, boxes, classes, width, height, info, data, + is_training): + imshape = image.get_shape().as_list() + imshape[-1] = 3 + image.set_shape(imshape) + + bshape = boxes.get_shape().as_list() + boxes = pad_max_instances(boxes, self._max_num_instances, 0) + bshape[0] = self._max_num_instances + boxes.set_shape(bshape) + + cshape = classes.get_shape().as_list() + classes = pad_max_instances(classes, + self._max_num_instances, -1) + cshape[0] = self._max_num_instances + classes.set_shape(cshape) + + area = data['groundtruth_area'] + ashape = area.get_shape().as_list() + area = pad_max_instances(area, self._max_num_instances,0) + ashape[0] = self._max_num_instances + area.set_shape(ashape) + + is_crowd = data['groundtruth_is_crowd'] + ishape = is_crowd.get_shape().as_list() + is_crowd = pad_max_instances( + tf.cast(is_crowd, tf.int32), self._max_num_instances, 0) + ishape[0] = self._max_num_instances + is_crowd.set_shape(ishape) + + num_detections = tf.shape(data['groundtruth_classes'])[0] + labels = { + 'source_id': utils.process_source_id(data['source_id']), + 'bbox': tf.cast(boxes, self._dtype), + 'classes': tf.cast(classes, self._dtype), + 'area': tf.cast(area, self._dtype), + 'is_crowd': is_crowd, + 'width': width, + 'height': height, + 'info': info, + 'num_detections': num_detections + } + + heatmap_feature_labels = self._build_heatmap_and_regressed_features( + boxes=boxes, classes=classes, num_objects=num_detections, + output_size=[self._output_dims, self._output_dims], + input_size=[self._image_w, self._image_w] + ) + labels.update(heatmap_feature_labels) + return image, labels + def postprocess_fn(self, is_training): if is_training: #or self._cutmix return None # if not self._fixed_size or self._mosaic else None diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index 2cec7250f..fe0c1d2c0 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -27,9 +27,6 @@ def __init__(self, dictionary where the keys-value pairs denote the names of the output and the respective output tensor """ - # super().__init__(**kwargs) - # self.outputs_layers = {} - self._input_specs = input_specs self._task_outputs = task_outputs self._heatmap_bias = heatmap_bias @@ -55,33 +52,10 @@ def __init__(self, super().__init__(inputs=inputs, outputs=outputs, name='CenterNetDecoder') - # def build(self, inputs): - # for key in self._task_outputs: - # num_filters = self._task_outputs[key] - # bias = 0 - # if 'heatmaps' in key: - # bias = self._heatmap_bias - - # self.outputs_layers[key] = [CenterNetDecoderConv(output_filters=num_filters, - # name=key, bias_init=bias) for _ in range(self._num_inputs)] - - # def call(self, inputs): - # outputs = dict() - - # for key in self._task_outputs: - # outputs[key] = [self.outputs_layers[key][i](inputs[i]) for i in range(self._num_inputs)] - - # return outputs - - # @property - # def output_specs(self): - # return self._output_specs - def get_config(self): layer_config = { 'task_outputs': self._task_outputs, 'heatmap_bias': self._heatmap_bias } - #layer_config.update(super().get_config()) return layer_config diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 224292f9c..f3bc802c2 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -196,13 +196,16 @@ def validation_step(self, inputs, model, metrics=None): # get the data point image, label = inputs - # computer detivative and apply gradients y_pred = model(image, training=False) y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) loss, loss_metrics = self.build_losses(y_pred['raw_output'], label) logs = {self.loss: loss_metrics['total_loss']} image_shape = tf.shape(image)[1:-1] + + label['boxes'] = box_ops.denormalize_boxes( + tf.cast(label['bbox'], tf.float32), image_shape) + del label['bbox'] coco_model_outputs = { 'detection_boxes': From f6f40f2731bc14bc28c888c3da70279e6a9aa550 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 12 Apr 2021 16:14:31 +0000 Subject: [PATCH 088/132] testing on vm --- centernet/configs/centernet.py | 3 ++- centernet/configs/experiments/centernet-eval.yaml | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index e98578233..a18354338 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -124,7 +124,7 @@ class Parser(hyperparams.Config): @dataclasses.dataclass class DataConfig(cfg.DataConfig): """Input config for training.""" - input_path: str = '' # 'D:\\Datasets\\coco\\2017\\1.1.0*' #'gs://tensorflow2/coco_records/train/2017*' + input_path: str = 'gs://tensorflow2/coco_records/val/2017*' tfds_name: str = None #'coco' tfds_split: str = None #'train' #'val' global_batch_size: int = 32 @@ -134,6 +134,7 @@ class DataConfig(cfg.DataConfig): parser: Parser = Parser() shuffle_buffer_size: int = 10000 tfds_download: bool = False + cache: bool = False class Loss(hyperparams.Config): pass diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index 84180b187..f9374f47c 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -10,8 +10,8 @@ task: type: hourglass decoder: heatmap_bias: -2.19 - odapi_weights: 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' - extremenet_weights: 'D:\\weights\extremenet' + odapi_weights: '/home/cam2tensorflow/centernet_hg104_512x512_coco17_tpu-8/checkpoint' + # extremenet_weights: 'D:\\weights\extremenet' backbone_name: hourglass104_512 decoder_name: detection_2d num_classes: 90 @@ -25,10 +25,11 @@ task: iou_thresh: 0.4 class_offset: 1 validation_data: - tfds_data_dir: 'D:\\Datasets' - tfds_name: coco/2017 - tfds_split: validation - global_batch_size: 1 + input_path: 'gs://tensorflow2/coco_records/val/2017*' + # tfds_data_dir: 'D:\\Datasets'' + # tfds_name: coco/2017 + # tfds_split: validation + global_batch_size: 8 tfds_download: true dtype: float16 is_training: false @@ -84,4 +85,4 @@ trainer: warmup: type: 'linear' linear: - warmup_steps: 1000 #learning rate rises from 0 to 0.0013 over 1000 steps \ No newline at end of file + warmup_steps: 1000 #learning rate rises from 0 to 0.0013 over 1000 steps From 696c971c30e37c181152709c09cdf7f4dad6bcf8 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 12 Apr 2021 19:49:25 +0000 Subject: [PATCH 089/132] dataloader --- centernet/dataloaders/centernet_input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index ab8a87742..71c7f0986 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -148,7 +148,7 @@ def _build_heatmap_and_regressed_features(self, width = 0.0 for tag_ind in tf.range(num_objects): box = boxes[tag_ind] - obj_class = classes[tag_ind] # TODO: See if subtracting 1 from the class like the paper is unnecessary + obj_class = classes[tag_ind] - 1 # TODO: See if subtracting 1 from the class like the paper is unnecessary ytl, xtl, ybr, xbr = box[0], box[1], box[2], box[3] From ec7fdda5a01b79fd8afad771a2a1ef51291e0dfd Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 12 Apr 2021 15:54:37 -0400 Subject: [PATCH 090/132] train step --- .../configs/experiments/centernet-eval.yaml | 3 +- centernet/tasks/centernet.py | 39 ++++++++++++++++++- centernet/train.py | 2 +- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index 1f54006d1..325b1bbcf 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -10,6 +10,7 @@ task: type: hourglass decoder: heatmap_bias: -2.19 + # odapi_weights: 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8/checkpoint' odapi_weights: '/home/cam2tensorflow/centernet_hg104_512x512_coco17_tpu-8/checkpoint' # extremenet_weights: 'D:\\weights\extremenet' backbone_name: hourglass104_512 @@ -26,7 +27,7 @@ task: class_offset: 1 validation_data: input_path: 'gs://tensorflow2/coco_records/val/2017*' - # tfds_data_dir: 'D:\\Datasets'' + # tfds_data_dir: 'D:\\Datasets' # tfds_name: coco/2017 # tfds_split: validation global_batch_size: 8 diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 57242bc3f..2a53e999a 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -195,7 +195,44 @@ def build_metrics(self, training=True): return metrics def train_step(self, inputs, model, optimizer, metrics=None): - pass + # get the data point + image, label = inputs + + with tf.GradientTape() as tape: + # compute a prediction + # cast to float32 + y_pred = model(image, training=True) + y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred['raw_output']) + loss_metrics = self.build_losses(y_pred, label) + + #scale the loss for numerical stability + if isinstance(optimizer, mixed_precision.LossScaleOptimizer): + total_loss = optimizer.get_scaled_loss(loss_metrics['total_loss']) + + #compute the gradient + train_vars = model.trainable_variables + gradients = tape.gradient(total_loss, train_vars) + + #get unscaled loss if the scaled loss was used + if isinstance(optimizer, mixed_precision.LossScaleOptimizer): + gradients = optimizer.get_unscaled_gradients(gradients) + + if self.task_config.gradient_clip_norm > 0.0: + gradients, _ = tf.clip_by_global_norm(gradients, + self.task_config.gradient_clip_norm) + + optimizer.apply_gradients(zip(gradients, train_vars)) + + logs = {self.loss: loss_metrics['total_loss']} + if metrics: + for m in metrics: + m.update_state(loss_metrics[m.name]) + logs.update({m.name: m.result()}) + + tf.print(logs, end='\n') + ret = '\033[F' * (len(logs.keys()) + 1) + tf.print(ret, end='\n') + return logs def validation_step(self, inputs, model, metrics=None): # get the data point diff --git a/centernet/train.py b/centernet/train.py index f9f9bdbc3..5860accfa 100755 --- a/centernet/train.py +++ b/centernet/train.py @@ -54,7 +54,7 @@ nohup python3 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov4- --config_file=yolo/configs/experiments/yolov4-1gpu.yaml >> yolov4-1gpu.log & tail -f yolov4-1gpu.log nohup python3.8 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov3-1gpu_mosaic --config_file=yolo/configs/experiments/yolov3-1gpu.yaml >> yolov3-1gpu.log & tail -f yolov3-1gpu.log evalaute CenterNet: -nohup python -m centernet.train --mode=eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval.yaml >> centernet-eval.log & tail -f centernet-eval.log +nohup python -m centernet.train --mode=train_and_eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval.yaml >> centernet-eval.log & tail -f centernet-eval.log """ From 462db566919a33e2c2cd0641cc782a3e04022f76 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 13 Apr 2021 10:53:30 -0400 Subject: [PATCH 091/132] tpu config --- .../experiments/centernet-eval-tpu.yaml | 84 ++ .../configs/experiments/centernet-eval.yaml | 14 +- centernet/modeling/CenterNet.py | 6 +- .../modeling/layers/detection_generator.py | 10 +- centernet/modeling/layers/nn_blocks.py | 173 ++-- centernet/modeling/layers/subnormalization.py | 901 ++++++++++++++++++ .../utils/weight_utils/test_load_weights.py | 3 - 7 files changed, 1097 insertions(+), 94 deletions(-) create mode 100644 centernet/configs/experiments/centernet-eval-tpu.yaml create mode 100644 centernet/modeling/layers/subnormalization.py diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml new file mode 100644 index 000000000..af3f93028 --- /dev/null +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -0,0 +1,84 @@ +runtime: + distribution_strategy: 'tpu' + mixed_precision_dtype: 'bfloat16' +task: + model: + base: + backbone: + type: hourglass + decoder: + heatmap_bias: -2.19 + odapi_weights: '/home/cam2tensorflow/centernet_hg104_512x512_coco17_tpu-8/checkpoint' + backbone_name: hourglass104_512 + decoder_name: detection_2d + num_classes: 90 + _input_size: [512, 512, 3] # null + filter: + max_detections: 100 + peak_error: 0.000001 + peak_extract_kernel_size: 3 + use_nms: false + center_thresh: 0.1 + iou_thresh: 0.4 + class_offset: 1 + validation_data: + input_path: 'gs://tensorflow2/coco_records/val/2017*' + global_batch_size: 8 + tfds_download: true + dtype: float16 + is_training: false + drop_remainder: true + parser: + image_h: 512 + image_w: 512 + num_classes: 90 + max_num_instances: 128 + use_gaussian_bump: true + gaussian_rad: -1 + gaussian_iou: 0.7 + output_dims: 128 + dtype: float32 + shuffle_buffer_size: 10000 + subtasks: + detection: + use_centers: true + use_corners: false + predict_3d: false + segmentation: false + losses: + detection: + detection_weight: 1.0 + corner_pull_weight: 0.1 + corner_push_weight: 0.1 + offset_weight: 1.0 + segmentation: + per_category_metrics: false + weight_decay: 0.0005 + init_checkpoint: '' + annotation_file: null + gradient_clip_norm: 0.0 + load_odapi_weights: true + load_extremenet_weights: false +trainer: + train_steps: 10000 # 160 epochs at 64 batchsize -> 500500 * 64/16 + validation_steps: 625 # 5063 #625 + steps_per_loop: 10000 + validation_interval: 1 + summary_interval: 10000 + checkpoint_interval: 10000 + optimizer_config: + learning_rate: + type: stepwise + stepwise: + boundaries: [800000, 900000] # [400000, 450000] + name: PiecewiseConstantDecay + values: [0.00065, 0.000065, 0.0000065] #[0.0013, 0.00013, 0.000013] + optimizer: + type: sgd + sgd: + momentum: 0.949 + name: SGD + warmup: + type: 'linear' + linear: + warmup_steps: 1000 #learning rate rises from 0 to 0.0013 over 1000 steps diff --git a/centernet/configs/experiments/centernet-eval.yaml b/centernet/configs/experiments/centernet-eval.yaml index 325b1bbcf..46b0be69f 100644 --- a/centernet/configs/experiments/centernet-eval.yaml +++ b/centernet/configs/experiments/centernet-eval.yaml @@ -10,8 +10,8 @@ task: type: hourglass decoder: heatmap_bias: -2.19 - # odapi_weights: 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8/checkpoint' - odapi_weights: '/home/cam2tensorflow/centernet_hg104_512x512_coco17_tpu-8/checkpoint' + odapi_weights: 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8/checkpoint' + # odapi_weights: '/home/cam2tensorflow/centernet_hg104_512x512_coco17_tpu-8/checkpoint' # extremenet_weights: 'D:\\weights\extremenet' backbone_name: hourglass104_512 decoder_name: detection_2d @@ -26,10 +26,10 @@ task: iou_thresh: 0.4 class_offset: 1 validation_data: - input_path: 'gs://tensorflow2/coco_records/val/2017*' - # tfds_data_dir: 'D:\\Datasets' - # tfds_name: coco/2017 - # tfds_split: validation + # input_path: 'gs://tensorflow2/coco_records/val/2017*' + tfds_data_dir: 'D:\\Datasets' + tfds_name: coco/2017 + tfds_split: validation global_batch_size: 8 tfds_download: true dtype: float16 @@ -88,4 +88,4 @@ trainer: warmup: type: 'linear' linear: - warmup_steps: 1000 #learning rate rises from 0 to 0.0013 over 1000 steps + warmup_steps: 1000 #learning rate rises from 0 to 0.0013 over 1000 steps diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index b6de5d93d..2b9edbab6 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -94,9 +94,9 @@ def build_centernet(input_specs, task_config, l2_regularization): model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) - test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.float32) - model(test_input) - # model.build(input_specs.shape) + # test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.float32) + # model(test_input) + model.build(input_specs.shape) model.summary() # TODO: uncommend when filter is implemented diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 6fde7e0a9..81ae80e60 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -97,11 +97,11 @@ def process_heatmap(self, # Zero out everything that is not a peak. feature_map_peaks = ( - feature_map * tf.cast(feature_map_peak_mask, tf.float16)) + feature_map * tf.cast(feature_map_peak_mask, tf.float32)) # Zero out peaks whose scores do not exceed threshold valid_peaks_mask = feature_map_peaks > center_thresh - feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.float16) + feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.float32) return feature_map_peaks @@ -272,8 +272,8 @@ class prediction for each box. y_offsets = offsets[..., 0] x_offsets = offsets[..., 1] - y_indices = tf.cast(y_indices, dtype=tf.float16) - x_indices = tf.cast(x_indices, dtype=tf.float16) + y_indices = tf.cast(y_indices, dtype=tf.float32) + x_indices = tf.cast(x_indices, dtype=tf.float32) detection_classes = channel_indices + self._class_offset @@ -309,7 +309,7 @@ def call(self, inputs): y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) # Normalize bounding boxes - boxes = boxes / tf.cast(height, tf.float16) + boxes = boxes / tf.cast(height, tf.float32) # Apply nms if self._use_nms: diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 37d41018f..a8edab423 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -3,7 +3,9 @@ from official.modeling import tf_utils from official.vision.beta.modeling.layers import \ nn_blocks as official_nn_blocks +from yolo.modeling.layers import subnormalization +TPU_BASE = True @tf.keras.utils.register_keras_serializable(package='centernet') class Identity(tf.keras.layers.Layer): @@ -18,26 +20,38 @@ def call(self, input): @tf.keras.utils.register_keras_serializable(package='centernet') class ConvBN(tf.keras.layers.Layer): """ - Modified Convolution layer to match that of the DarkNet Library. The Layer is a standards combination of Conv BatchNorm Activation, - however, the use of bias in the conv is determined by the use of batch normalization. + Modified Convolution layer to match that of the DarkNet Library. + The Layer is a standards combination of Conv BatchNorm Activation, + however, the use of bias in the conv is determined by the use of batch + normalization. Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang Chen, Jun-Wei Hsieh - CSPNet: A New Backbone that can Enhance Learning Capability of CNN. arXiv:1911.11929 + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, + Ping-Yang Chen, Jun-Wei Hsieh + CSPNet: A New Backbone that can Enhance Learning Capability of CNN. + arXiv:1911.11929 Args: filters: integer for output depth, or the number of features to learn - kernel_size: integer or tuple for the shape of the weight matrix or kernel to learn - strides: integer of tuple how much to move the kernel after each kernel use - padding: string 'valid' or 'same', if same, then pad the image, else do not + kernel_size: integer or tuple for the shape of the weight matrix or kernel + to learn + strides: integer of tuple how much to move the kernel after each kernel + use + padding: string 'valid' or 'same', if same, then pad the image, else do + not dialtion_rate: tuple to indicate how much to modulate kernel weights and how many pixels in a feature map to skip - kernel_initializer: string to indicate which function to use to initialize weights - bias_initializer: string to indicate which function to use to initialize bias - kernel_regularizer: string to indicate which function to use to regularizer weights - bias_regularizer: string to indicate which function to use to regularizer bias + kernel_initializer: string to indicate which function to use to initialize + weights + bias_initializer: string to indicate which function to use to initialize + bias + kernel_regularizer: string to indicate which function to use to + regularizer weights + bias_regularizer: string to indicate which function to use to regularizer + bias use_bn: boolean for whether to use batch normalization use_sync_bn: boolean for whether sync batch normalization statistics - of all batch norm layers to the models global statistics (across all input batches) - norm_moment: float for moment to use for batch normalization + of all batch norm layers to the models global statistics + (across all input batches) + norm_momentum: float for moment to use for batch normalization norm_epsilon: float for batch normalization epsilon activation: string or None for activation function to use in layer, if None activation is replaced by linear @@ -45,24 +59,24 @@ class ConvBN(tf.keras.layers.Layer): **kwargs: Keyword Arguments """ - def __init__( - self, - filters=1, - kernel_size=(1, 1), - strides=(1, 1), - padding='same', - dilation_rate=(1, 1), - kernel_initializer='glorot_uniform', - bias_initializer='zeros', - bias_regularizer=None, - kernel_regularizer=None, # Specify the weight decay as the default will not work. - use_bn=True, - use_sync_bn=False, - norm_momentum=0.99, - norm_epsilon=0.001, - activation='leaky', - leaky_alpha=0.1, - **kwargs): + def __init__(self, + filters=1, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + dilation_rate=(1, 1), + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + subdivisions=1, + bias_regularizer=None, + kernel_regularizer=None, + use_bn=True, + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + activation='leaky', + leaky_alpha=0.1, + **kwargs): # convolution params self._filters = filters @@ -73,12 +87,14 @@ def __init__( self._kernel_initializer = kernel_initializer self._bias_initializer = bias_initializer self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + self._subdivisions = subdivisions # batch normalization params self._use_bn = use_bn self._use_sync_bn = use_sync_bn - self._norm_moment = norm_momentum + self._norm_momentum = norm_momentum self._norm_epsilon = norm_epsilon if tf.keras.backend.image_data_format() == 'channels_last': @@ -95,57 +111,59 @@ def __init__( super().__init__(**kwargs) def build(self, input_shape): - kernel_size = self._kernel_size if isinstance(self._kernel_size, - int) else self._kernel_size[0] - dilation_rate = self._dilation_rate if isinstance( - self._dilation_rate, int) else self._dilation_rate[0] - if self._padding == 'same' and kernel_size != 1: - padding = (dilation_rate * (kernel_size - 1)) - left_shift = padding // 2 - self._zeropad = tf.keras.layers.ZeroPadding2D([[left_shift, left_shift], - [left_shift, left_shift]]) - else: - self._zeropad = Identity() - use_bias = not self._use_bn - # self.conv = tf.keras.layers.Conv2D( - # filters=self._filters, - # kernel_size=self._kernel_size, - # strides=self._strides, - # padding= self._padding,# 'valid', - # dilation_rate=self._dilation_rate, - # use_bias=use_bias, - # kernel_initializer=self._kernel_initializer, - # bias_initializer=self._bias_initializer, - # kernel_regularizer=self._kernel_regularizer, - # bias_regularizer=self._bias_regularizer) - - self.conv = tf.keras.layers.Conv2D( - filters=self._filters, - kernel_size=self._kernel_size, - strides=self._strides, - padding='valid', - dilation_rate=self._dilation_rate, - use_bias=use_bias, - kernel_initializer=self._kernel_initializer, - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) + if not TPU_BASE: + kernel_size = self._kernel_size if isinstance( + self._kernel_size, int) else self._kernel_size[0] + dilation_rate = self._dilation_rate if isinstance( + self._dilation_rate, int) else self._dilation_rate[0] + if self._padding == 'same' and kernel_size != 1: + padding = (dilation_rate * (kernel_size - 1)) + left_shift = padding // 2 + self._paddings = tf.constant([[0, 0], [left_shift, left_shift], + [left_shift, left_shift], [0, 0]]) + # self._zeropad = tf.keras.layers.ZeroPadding2D() + else: + self._paddings = tf.constant([[0, 0], [0, 0], [0, 0], [0, 0]]) + + self.conv = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=self._kernel_size, + strides=self._strides, + padding='valid', + dilation_rate=self._dilation_rate, + use_bias=use_bias, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + else: + self.conv = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=self._kernel_size, + strides=self._strides, + padding=self._padding, # 'valid', + dilation_rate=self._dilation_rate, + use_bias=use_bias, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) if self._use_bn: if self._use_sync_bn: - self.bn = tf.keras.layers.experimental.SyncBatchNormalization( - momentum=self._norm_moment, + self.bn = subnormalization.SubDivSyncBatchNormalization( + subdivisions=self._subdivisions, + momentum=self._norm_momentum, epsilon=self._norm_epsilon, axis=self._bn_axis) else: - self.bn = tf.keras.layers.BatchNormalization( - momentum=self._norm_moment, + self.bn = subnormalization.SubDivBatchNormalization( + subdivisions=self._subdivisions, + momentum=self._norm_momentum, epsilon=self._norm_epsilon, axis=self._bn_axis) - else: - self.bn = Identity() if self._activation == 'leaky': self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) @@ -156,9 +174,11 @@ def build(self, input_shape): self._activation) # tf.keras.layers.Activation(self._activation) def call(self, x): - x = self._zeropad(x) + if not TPU_BASE: + x = tf.pad(x, self._paddings, mode='CONSTANT', constant_values=0) x = self.conv(x) - x = self.bn(x) + if self._use_bn: + x = self.bn(x) x = self._activation_fn(x) return x @@ -176,7 +196,7 @@ def get_config(self): 'kernel_regularizer': self._kernel_regularizer, 'use_bn': self._use_bn, 'use_sync_bn': self._use_sync_bn, - 'norm_moment': self._norm_moment, + 'norm_momentum': self._norm_momentum, 'norm_epsilon': self._norm_epsilon, 'activation': self._activation, 'leaky_alpha': self._leaky_alpha @@ -185,6 +205,7 @@ def get_config(self): return layer_config + @tf.keras.utils.register_keras_serializable(package='centernet') class HourglassBlock(tf.keras.layers.Layer): """ diff --git a/centernet/modeling/layers/subnormalization.py b/centernet/modeling/layers/subnormalization.py new file mode 100644 index 000000000..dd0aca3ae --- /dev/null +++ b/centernet/modeling/layers/subnormalization.py @@ -0,0 +1,901 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Normalization layers.""" +from __future__ import absolute_import, division, print_function + +import tensorflow as tf +from tensorflow.python.distribute import distribution_strategy_context as ds +from tensorflow.python.distribute import reduce_util +from tensorflow.python.framework import constant_op, dtypes, ops, tensor_shape +from tensorflow.python.keras import backend as K +from tensorflow.python.keras import constraints, initializers, regularizers +from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.engine.input_spec import InputSpec +from tensorflow.python.keras.layers import normalization +from tensorflow.python.keras.utils import control_flow_util +from tensorflow.python.ops import (array_ops, control_flow_ops, init_ops, + math_ops, nn, state_ops) +from tensorflow.python.ops import variables as tf_variables +from tensorflow.python.platform import tf_logging as logging + + +class SubDivBatchNormalization(normalization.BatchNormalizationBase): + + _USE_V2_BEHAVIOR = True + + def __init__(self, + axis=-1, + subdivisions=1, + momentum=0.99, + epsilon=1e-3, + center=True, + scale=True, + beta_initializer='zeros', + gamma_initializer='ones', + moving_mean_initializer='zeros', + moving_variance_initializer='ones', + beta_regularizer=None, + gamma_regularizer=None, + beta_constraint=None, + gamma_constraint=None, + renorm=False, + renorm_clipping=None, + renorm_momentum=0.99, + fused=None, + trainable=True, + adjustment=None, + name=None, + **kwargs): + + super(SubDivBatchNormalization, self).__init__( + axis=axis, + momentum=momentum, + epsilon=epsilon, + center=center, + scale=scale, + beta_initializer=beta_initializer, + gamma_initializer=gamma_initializer, + moving_mean_initializer=moving_mean_initializer, + moving_variance_initializer=moving_variance_initializer, + beta_regularizer=beta_regularizer, + gamma_regularizer=gamma_regularizer, + beta_constraint=beta_constraint, + gamma_constraint=gamma_constraint, + renorm=renorm, + renorm_clipping=renorm_clipping, + renorm_momentum=renorm_momentum, + fused=None + if not renorm else False, #if subdivisions <= 1 else False, #alter this + trainable=trainable, + virtual_batch_size=None, + name=name, + **kwargs) + + self.subdivisions = subdivisions + + def build(self, input_shape): + super().build(input_shape) + input_shape = tensor_shape.TensorShape(input_shape) + if not input_shape.ndims: + raise ValueError('Input has undefined rank:', input_shape) + ndims = len(input_shape) + + # Convert axis to list and resolve negatives + if isinstance(self.axis, int): + self.axis = [self.axis] + + for idx, x in enumerate(self.axis): + if x < 0: + self.axis[idx] = ndims + x + + # Validate axes + for x in self.axis: + if x < 0 or x >= ndims: + raise ValueError('Invalid axis: %d' % x) + if len(self.axis) != len(set(self.axis)): + raise ValueError('Duplicate axis: %s' % self.axis) + + # TODO(yaozhang): if input is not 4D, reshape it to 4D and reshape the + # output back to its original shape accordingly. + # self.fused = None + + if self.fused: + if self._USE_V2_BEHAVIOR: + # TODO(b/173253101): Using fused in the 5D case is currently disabled + # due to a regression on UNet, so it is only currently only supported in + # the 4D case. + if self.fused is None: + self.fused = ndims == 4 + elif self.fused and ndims != 4: + raise ValueError('Batch normalization layers with `fused=True` only ' + 'support 4D or 5D input tensors. ' + 'Received tensor with shape: %s' % + (tuple(input_shape),)) + else: + assert self.fused is not None + self.fused = (ndims == 4 and self._fused_can_be_used()) + # TODO(chrisying): fused batch norm is currently not supported for + # multi-axis batch norm and by extension virtual batches. In some cases, + # it might be possible to use fused batch norm but would require reshaping + # the Tensor to 4D with the axis in 1 or 3 (preferred 1) which is + # particularly tricky. A compromise might be to just support the most + # common use case (turning 5D w/ virtual batch to NCHW) + + if self.axis == [1] and ndims == 4: + self._data_format = 'NCHW' + elif self.axis == [1] and ndims == 5: + self._data_format = 'NCDHW' + elif self.axis == [3] and ndims == 4: + self._data_format = 'NHWC' + elif self.axis == [4] and ndims == 5: + self._data_format = 'NDHWC' + elif ndims == 5: + # 5D tensors that can be passed in but should not use fused batch norm + # due to unsupported axis. + self.fused = False + else: + raise ValueError('Unsupported axis, fused batch norm only supports ' + 'axis == [1] or axis == [3] for 4D input tensors or ' + 'axis == [1] or axis == [4] for 5D input tensors') + + axis_to_dim = {x: input_shape.dims[x].value for x in self.axis} + for x in axis_to_dim: + if axis_to_dim[x] is None: + raise ValueError('Input has undefined `axis` dimension. Input shape: ', + input_shape) + self.input_spec = InputSpec(ndim=ndims, axes=axis_to_dim) + + # get the shape for my weights based on input shape + if len(axis_to_dim) == 1 and self.virtual_batch_size is None: + # Single axis batch norm (most common/default use-case) + param_shape = (list(axis_to_dim.values())[0],) + else: + # Parameter shape is the original shape but with 1 in all non-axis dims + param_shape = [ + axis_to_dim[i] if i in axis_to_dim else 1 for i in range(ndims) + ] + if self.virtual_batch_size is not None: + # When using virtual batches, add an extra dim at index 1 + param_shape.insert(1, 1) + for idx, x in enumerate(self.axis): + self.axis[idx] = x + 1 # Account for added dimension + + try: + # Disable variable partitioning when creating the moving mean and variance + if hasattr(self, '_scope') and self._scope: + partitioner = self._scope.partitioner + self._scope.set_partitioner(None) + else: + partitioner = None + + if self.subdivisions > 1: + self.aggregated_sum_batch = self.add_weight( + name='agg_sum', + shape=param_shape, + dtype=self._param_dtype, + initializer=self.moving_mean_initializer, + synchronization=tf_variables.VariableSynchronization.ON_READ, + trainable=False, + aggregation=tf_variables.VariableAggregation.SUM, + experimental_autocast=False) + + self.aggregated_square_sum_batch = self.add_weight( + name='agg_square_sum', + shape=param_shape, + dtype=self._param_dtype, + initializer=self.moving_variance_initializer, + synchronization=tf_variables.VariableSynchronization.ON_READ, + trainable=False, + aggregation=tf_variables.VariableAggregation.SUM, + experimental_autocast=False) + + self.local_count = self.add_weight( + name='local_sum', + shape=(), + dtype=tf.int32, + initializer=tf.zeros_initializer(), + synchronization=tf_variables.VariableSynchronization.ON_READ, + trainable=False, + aggregation=tf_variables.VariableAggregation.SUM, + experimental_autocast=False) + + self.aggregated_batch_size = self.add_weight( + name='net_batches', + shape=(), + dtype=tf.int32, + initializer=tf.zeros_initializer(), + synchronization=tf_variables.VariableSynchronization.ON_READ, + trainable=False, + aggregation=tf_variables.VariableAggregation.SUM, + experimental_autocast=False) + + finally: + if partitioner: + self._scope.set_partitioner(partitioner) + self.built = True + + def _assign_subdiv_moving_average(self, variable, value, momentum, + subdivsions, count): + with K.name_scope('AssignSubDivMovingAvg') as scope: + with ops.colocate_with(variable): + decay = ops.convert_to_tensor_v2_with_dispatch( + 1.0 - momentum, name='decay') + if decay.dtype != variable.dtype.base_dtype: + decay = math_ops.cast(decay, variable.dtype.base_dtype) + + # get the aggregated update + update_delta = (variable - math_ops.cast(value, variable.dtype)) * decay + + # update at the end of last step + update_delta = array_ops.where((count + 1) % subdivisions == 0, + update_delta, K.zeros_like(update_delta)) + return state_ops.assign_sub(variable, update_delta, name=scope) + + def _assign_subdiv_new_value(self, variable, value, subdivisions, count): + with K.name_scope('AssignNewValue') as scope: + with ops.colocate_with(variable): + update_value = array_ops.where((count + 1) % subdivisions == 0, value, + variable) + return state_ops.assign(variable, update_value, name=scope) + + def _assign_subdiv_rotating_sum(self, variable, value, subdivisions, count, + inputs_size): + with K.name_scope('AssignSubDivRotatedSum') as scope: + with ops.colocate_with(variable): + # reduce it for the current + update_delta = value #/subdivisions + + # if the input size is 0 + if inputs_size is not None: + update_delta = array_ops.where(inputs_size > 0, update_delta, + K.zeros_like(update_delta)) + + # if we are starting a new batch set the variable to 0 by removing it + # from update delta then add the delta to the variable to get + # rid of the value variable + update_delta = array_ops.where(count % subdivisions == 0, + update_delta - variable, update_delta) + return state_ops.assign_add(variable, update_delta, name=scope) + + def _subdiv_calculate_mean_and_var(self, inputs, reduction_axes, keep_dims): + # calculate the + net_sum = math_ops.reduce_sum( + inputs, axis=reduction_axes, keepdims=keep_dims) + squared_mean = math_ops.reduce_sum( + math_ops.square(inputs), axis=reduction_axes, keepdims=keep_dims) + + if self._support_zero_size_input(): + # Keras assumes that batch dimension is the first dimension for Batch + # Normalization. + input_batch_size = array_ops.shape(inputs)[0] + else: + input_batch_size = None + + # get the number of total params you are averaging including batchsize(local) + axes_vals = [ + (array_ops.shape_v2(inputs))[i] for i in range(1, len(reduction_axes)) + ] + multiplier = math_ops.cast(math_ops.reduce_prod(axes_vals), dtypes.float32) + + squared_mean = squared_mean / multiplier + net_sum = net_sum / multiplier + + if input_batch_size is None: + mean, variance = nn.moments(inputs, reduction_axes, keep_dims=keep_dims) + else: + batches_ = math_ops.cast(input_batch_size, self._param_dtype) + mean = net_sum / batches_ + variance = squared_mean / batches_ - math_ops.square( + array_ops.stop_gradient(mean)) + + return mean, net_sum, variance, squared_mean, input_batch_size + + def subdiv_moments(self, inputs, reduction_axes, keep_dims): + # mean and variance only for the current batch + mean, net_sum, variance, squared_mean, input_batch_size = self._subdiv_calculate_mean_and_var( + inputs, reduction_axes, keep_dims) + + if self._support_zero_size_input(): + input_batch_size = 0 if input_batch_size is None else input_batch_size + mean = array_ops.where(input_batch_size > 0, mean, K.zeros_like(mean)) + net_sum = array_ops.where(input_batch_size > 0, net_sum, + K.zeros_like(net_sum)) + variance = array_ops.where(input_batch_size > 0, variance, + K.zeros_like(variance)) + squared_mean = array_ops.where(input_batch_size > 0, squared_mean, + K.zeros_like(squared_mean)) + return mean, net_sum, variance, squared_mean, input_batch_size + + def _subdiv_batch_norm(self, inputs, training=None): + # tf.print('bn', self.local_count) + training = self._get_training_value(training) + + inputs_dtype = inputs.dtype.base_dtype + if inputs_dtype in (dtypes.float16, dtypes.bfloat16): + # Do all math in float32 if given 16-bit inputs for numeric stability. + # In particular, it's very easy for variance to overflow in float16 and + # for safety we also choose to cast bfloat16 to float32. + inputs = math_ops.cast(inputs, dtypes.float32) + + params_dtype = self._param_dtype + + # Compute the axes along which to reduce the mean / variance + input_shape = inputs.shape + ndims = len(input_shape) + reduction_axes = [i for i in range(ndims) if i not in self.axis] + if self.virtual_batch_size is not None: + del reduction_axes[1] # Do not reduce along virtual batch dim + + # Broadcasting only necessary for single-axis batch norm where the axis is + # not the last dimension + broadcast_shape = [1] * ndims + broadcast_shape[self.axis[0]] = input_shape.dims[self.axis[0]].value + + def _broadcast(v): + if (v is not None and len(v.shape) != ndims and + reduction_axes != list(range(ndims - 1))): + return array_ops.reshape(v, broadcast_shape) + return v + + scale, offset = _broadcast(self.gamma), _broadcast(self.beta) + + # what does this do... + def _compose_transforms(scale, offset, then_scale, then_offset): + if then_scale is not None: + scale *= then_scale + offset *= then_scale + if then_offset is not None: + offset += then_offset + return (scale, offset) + + # is training value true false or None + training_value = control_flow_util.constant_value(training) + update_value = (self.local_count + 1) % self.subdivisions == 0 + if training_value == False: # pylint: disable=singleton-comparison,g-explicit-bool-comparison + mean, variance = self.moving_mean, self.moving_variance + else: + # training_value could be True or None -> None means determine at runtime + if self.adjustment: + adj_scale, adj_bias = self.adjustment(array_ops.shape(inputs)) + # Adjust only during training. + adj_scale = control_flow_util.smart_cond( + training, lambda: adj_scale, lambda: array_ops.ones_like(adj_scale)) + adj_bias = control_flow_util.smart_cond( + training, lambda: adj_bias, lambda: array_ops.zeros_like(adj_bias)) + scale, offset = _compose_transforms(adj_scale, adj_bias, scale, offset) + + keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1 + + # normalization stats for the current batch important = mean and squared_mean + mean, net_sum, variance, squared_mean, input_batch_size = self.subdiv_moments( + math_ops.cast(inputs, self._param_dtype), + reduction_axes, + keep_dims=keep_dims) + + # aggregate the things + def _update_aggragate_sum(): + return self._assign_subdiv_rotating_sum(self.aggregated_sum_batch, + net_sum, self.subdivisions, + self.local_count, + input_batch_size) + + def _update_aggragate_squared_sum(): + return self._assign_subdiv_rotating_sum( + self.aggregated_square_sum_batch, squared_mean, self.subdivisions, + self.local_count, input_batch_size) + + def _update_aggragate_batch_size(): + return self._assign_subdiv_rotating_sum(self.aggregated_batch_size, + input_batch_size, + self.subdivisions, + self.local_count, + input_batch_size) + + self.add_update(_update_aggragate_sum) + self.add_update(_update_aggragate_squared_sum) + self.add_update(_update_aggragate_batch_size) + + aggregated_mean = self.aggregated_sum_batch / math_ops.cast( + self.aggregated_batch_size, params_dtype) + aggregated_squared_mean = self.aggregated_square_sum_batch / math_ops.cast( + self.aggregated_batch_size, params_dtype) + aggregated_variance = aggregated_squared_mean - math_ops.square( + aggregated_mean) + + moving_mean = self.moving_mean + moving_variance = self.moving_variance + + # if we are training use the stats for this batch for normalizing this + # value other wise use the moving average + + # should only happen when we update the moving values + mean = control_flow_util.smart_cond( + training, + true_fn=lambda: mean, + false_fn=lambda: ops.convert_to_tensor_v2_with_dispatch(moving_mean)) + variance = control_flow_util.smart_cond( + training, + true_fn=lambda: variance, + false_fn=lambda: ops.convert_to_tensor_v2_with_dispatch( + moving_variance)) + + # circular update of the mean and variance + new_mean = control_flow_util.smart_cond( + update_value, + true_fn=lambda: ops.convert_to_tensor_v2_with_dispatch(aggregated_mean + ), + false_fn=lambda: moving_mean) + + new_variance = control_flow_util.smart_cond( + update_value, + true_fn=lambda: ops.convert_to_tensor_v2_with_dispatch( + aggregated_variance), + false_fn=lambda: moving_variance) + + # # should only be done when the moving mean is updated + # tf.print(new_variance, self.local_count, update_value, self.aggregated_batch_size, self.aggregated_sum_batch) + + if self.renorm: + r, d, new_mean, new_variance = self._renorm_correction_and_moments( + new_mean, new_variance, training, input_batch_size) + # When training, the normalized values (say, x) will be transformed as + # x * gamma + beta without renorm, and (x * r + d) * gamma + beta + # = x * (r * gamma) + (d * gamma + beta) with renorm. + r = _broadcast(array_ops.stop_gradient(r, name='renorm_r')) + d = _broadcast(array_ops.stop_gradient(d, name='renorm_d')) + scale, offset = _compose_transforms(r, d, scale, offset) + + def _do_update(var, value): + """Compute the updates for mean and variance.""" + return self._assign_moving_average(var, value, self.momentum, + self.aggregated_batch_size) + + def mean_update(): + true_branch = lambda: _do_update(self.moving_mean, new_mean) + false_branch = lambda: self.moving_mean + return control_flow_util.smart_cond(training, true_branch, false_branch) + + def variance_update(): + """Update the moving variance.""" + + def true_branch_renorm(): + # We apply epsilon as part of the moving_stddev to mirror the training + # code path. + moving_stddev = _do_update(self.moving_stddev, + math_ops.sqrt(new_variance + self.epsilon)) + return self._assign_new_value( + self.moving_variance, + # Apply relu in case floating point rounding causes it to go + # negative. + K.relu(moving_stddev * moving_stddev - self.epsilon)) + + if self.renorm: + true_branch = true_branch_renorm + else: + true_branch = lambda: _do_update(self.moving_variance, new_variance) + + false_branch = lambda: self.moving_variance + return control_flow_util.smart_cond(training, true_branch, false_branch) + + def update_count(): + with K.name_scope('update_count') as scope: + # update the local count + return state_ops.assign_add( + self.local_count, tf.cast(1, self.local_count.dtype), name=scope) + + self.add_update(mean_update) + self.add_update(variance_update) + self.add_update(update_count) + + mean = math_ops.cast(mean, inputs.dtype) + variance = math_ops.cast(variance, inputs.dtype) + if offset is not None: + offset = math_ops.cast(offset, inputs.dtype) + if scale is not None: + scale = math_ops.cast(scale, inputs.dtype) + outputs = nn.batch_normalization(inputs, _broadcast(mean), + _broadcast(variance), offset, scale, + self.epsilon) + if inputs_dtype in (dtypes.float16, dtypes.bfloat16): + outputs = math_ops.cast(outputs, inputs_dtype) + + # If some components of the shape got lost due to adjustments, fix that. + outputs.set_shape(input_shape) + + if self.virtual_batch_size is not None: + outputs = undo_virtual_batching(outputs) + return outputs + + def call(self, inputs, training=None): + training = self._get_training_value(training) + if self.subdivisions <= 1 or self.subdivisions is None: + return super().call(inputs, training=training) + else: + if self.renorm is False and training is False and self.fused: + # outputs = self._fused_batch_norm(inputs, training=False) + beta = self.beta if self.center else self._beta_const + gamma = self.gamma if self.scale else self._gamma_const + outputs, mean, variance = nn.fused_batch_norm( + inputs, + gamma, + beta, + mean=self.moving_mean, + variance=self.moving_variance, + epsilon=self.epsilon, + is_training=False, + data_format=self._data_format) + return outputs + return self._subdiv_batch_norm(inputs, training=training) + + +class SubDivSyncBatchNormalization(SubDivBatchNormalization): + r"""Normalize and scale inputs or activations synchronously across replicas. + Applies batch normalization to activations of the previous layer at each batch + by synchronizing the global batch statistics across all devices that are + training the model. For specific details about batch normalization please + refer to the `tf.keras.layers.BatchNormalization` layer docs. + If this layer is used when using tf.distribute strategy to train models + across devices/workers, there will be an allreduce call to aggregate batch + statistics across all replicas at every training step. Without tf.distribute + strategy, this layer behaves as a regular `tf.keras.layers.BatchNormalization` + layer. + Example usage: + ``` + strategy = tf.distribute.MirroredStrategy() + with strategy.scope(): + model = tf.keras.Sequential() + model.add(tf.keras.layers.Dense(16)) + model.add(tf.keras.layers.experimental.SyncBatchNormalization()) + ``` + Arguments: + axis: Integer, the axis that should be normalized + (typically the features axis). + For instance, after a `Conv2D` layer with + `data_format="channels_first"`, + set `axis=1` in `BatchNormalization`. + momentum: Momentum for the moving average. + epsilon: Small float added to variance to avoid dividing by zero. + center: If True, add offset of `beta` to normalized tensor. + If False, `beta` is ignored. + scale: If True, multiply by `gamma`. + If False, `gamma` is not used. + When the next layer is linear (also e.g. `nn.relu`), + this can be disabled since the scaling + will be done by the next layer. + beta_initializer: Initializer for the beta weight. + gamma_initializer: Initializer for the gamma weight. + moving_mean_initializer: Initializer for the moving mean. + moving_variance_initializer: Initializer for the moving variance. + beta_regularizer: Optional regularizer for the beta weight. + gamma_regularizer: Optional regularizer for the gamma weight. + beta_constraint: Optional constraint for the beta weight. + gamma_constraint: Optional constraint for the gamma weight. + renorm: Whether to use [Batch Renormalization]( + https://arxiv.org/abs/1702.03275). This adds extra variables during + training. The inference is the same for either value of this parameter. + renorm_clipping: A dictionary that may map keys 'rmax', 'rmin', 'dmax' to + scalar `Tensors` used to clip the renorm correction. The correction + `(r, d)` is used as `corrected_value = normalized_value * r + d`, with + `r` clipped to [rmin, rmax], and `d` to [-dmax, dmax]. Missing rmax, rmin, + dmax are set to inf, 0, inf, respectively. + renorm_momentum: Momentum used to update the moving means and standard + deviations with renorm. Unlike `momentum`, this affects training + and should be neither too small (which would add noise) nor too large + (which would give stale estimates). Note that `momentum` is still applied + to get the means and variances for inference. + trainable: Boolean, if `True` the variables will be marked as trainable. + Call arguments: + inputs: Input tensor (of any rank). + training: Python boolean indicating whether the layer should behave in + training mode or in inference mode. + - `training=True`: The layer will normalize its inputs using the + mean and variance of the current batch of inputs. + - `training=False`: The layer will normalize its inputs using the + mean and variance of its moving statistics, learned during training. + Input shape: + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + Output shape: + Same shape as input. + """ + + def __init__(self, + axis=-1, + subdivisions=1, + momentum=0.99, + epsilon=1e-3, + center=True, + scale=True, + beta_initializer='zeros', + gamma_initializer='ones', + moving_mean_initializer='zeros', + moving_variance_initializer='ones', + beta_regularizer=None, + gamma_regularizer=None, + beta_constraint=None, + gamma_constraint=None, + renorm=False, + renorm_clipping=None, + renorm_momentum=0.99, + trainable=True, + adjustment=None, + name=None, + **kwargs): + + # Currently we only support aggregating over the global batch size. + super(SubDivSyncBatchNormalization, self).__init__( + axis=axis, + subdivisions=subdivisions, + momentum=momentum, + epsilon=epsilon, + center=center, + scale=scale, + beta_initializer=beta_initializer, + gamma_initializer=gamma_initializer, + moving_mean_initializer=moving_mean_initializer, + moving_variance_initializer=moving_variance_initializer, + beta_regularizer=beta_regularizer, + gamma_regularizer=gamma_regularizer, + beta_constraint=beta_constraint, + gamma_constraint=gamma_constraint, + renorm=renorm, + renorm_clipping=renorm_clipping, + renorm_momentum=renorm_momentum, + fused=False, + trainable=trainable, + name=name, + **kwargs) + + def _calculate_mean_and_var(self, x, axes, keep_dims): + + with K.name_scope('moments'): + # The dynamic range of fp16 is too limited to support the collection of + # sufficient statistics. As a workaround we simply perform the operations + # on 32-bit floats before converting the mean and variance back to fp16 + y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x + replica_ctx = ds.get_replica_context() + if replica_ctx: + # local to me + local_sum = math_ops.reduce_sum(y, axis=axes, keepdims=True) + local_squared_sum = math_ops.reduce_sum( + math_ops.square(y), axis=axes, keepdims=True) + batch_size = math_ops.cast(array_ops.shape_v2(y)[0], dtypes.float32) + # TODO(b/163099951): batch the all-reduces once we sort out the ordering + # issue for NCCL. We don't have a mechanism to launch NCCL in the same + # order in each replica nowadays, so we limit NCCL to batch all-reduces. + + # get the sum of all replicas (converge all devices) + y_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, local_sum) + # get the sum from all replicas (converge all devices) + y_squared_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, + local_squared_sum) + # get the net batch size from all devices (converge all devices) + global_batch_size = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, + batch_size) + + # get the number of total params you are averaging (local) + axes_vals = [(array_ops.shape_v2(y))[i] for i in range(1, len(axes))] + multiplier = math_ops.cast( + math_ops.reduce_prod(axes_vals), dtypes.float32) + multiplier = multiplier * global_batch_size + + # conver mean var (locally) + mean = y_sum / multiplier + y_squared_mean = y_squared_sum / multiplier + # var = E(x^2) - E(x)^2 + variance = y_squared_mean - math_ops.square(mean) + else: + # if you only have one replica dont worry about it + # Compute true mean while keeping the dims for proper broadcasting. + mean = math_ops.reduce_mean(y, axes, keepdims=True, name='mean') + # sample variance, not unbiased variance + # Note: stop_gradient does not change the gradient that gets + # backpropagated to the mean from the variance calculation, + # because that gradient is zero + variance = math_ops.reduce_mean( + math_ops.squared_difference(y, mean), + axes, + keepdims=True, + name='variance') + if not keep_dims: + mean = array_ops.squeeze(mean, axes) + variance = array_ops.squeeze(variance, axes) + if x.dtype == dtypes.float16: + return (math_ops.cast(mean, dtypes.float16), + math_ops.cast(variance, dtypes.float16)) + else: + return (mean, variance) + + def _subdiv_calculate_mean_and_var(self, x, axes, keep_dims): + + with K.name_scope('moments'): + # The dynamic range of fp16 is too limited to support the collection of + # sufficient statistics. As a workaround we simply perform the operations + # on 32-bit floats before converting the mean and variance back to fp16 + y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x + replica_ctx = ds.get_replica_context() + + if replica_ctx: + # local to me + + local_sum = math_ops.reduce_sum(y, axis=axes, keepdims=True) + local_squared_sum = math_ops.reduce_sum( + math_ops.square(y), axis=axes, keepdims=True) + batch_size = math_ops.cast(array_ops.shape_v2(y)[0], dtypes.float32) + # TODO(b/163099951): batch the all-reduces once we sort out the ordering + # issue for NCCL. We don't have a mechanism to launch NCCL in the same + # order in each replica nowadays, so we limit NCCL to batch all-reduces. + # get the sum of all replicas (converge all devices) + y_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, local_sum) + # get the sum from all replicas (converge all devices) + y_squared_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, + local_squared_sum) + # get the net batch size from all devices (converge all devices) + input_batch_size = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, + batch_size) + + #tf.print(replica_ctx.replica_id_in_sync_group, replica_ctx.num_replicas_in_sync, batch_size, self.aggregated_square_sum_batch, axes) + # get the number of total params you are averaging (local) + axes_vals = [(array_ops.shape_v2(y))[i] for i in range(1, len(axes))] + multiplier_ = math_ops.cast( + math_ops.reduce_prod(axes_vals), dtypes.float32) + multiplier = multiplier_ * input_batch_size + + # conver mean var (locally) + mean = y_sum / multiplier + y_squared_mean = y_squared_sum / multiplier + # var = E(x^2) - E(x)^2 + variance = y_squared_mean - math_ops.square(mean) + net_sum = y_sum / multiplier_ + squared_mean = y_squared_sum / multiplier_ + + else: + # mean = math_ops.reduce_mean(y, axes, keepdims=True, name='mean') + # # sample variance, not unbiased variance + # # Note: stop_gradient does not change the gradient that gets + # # backpropagated to the mean from the variance calculation, + # # because that gradient is zero + # variance = math_ops.reduce_mean( + # math_ops.squared_difference(y, array_ops.stop_gradient(mean)), + # axes, + # keepdims=True, + # name='variance') + + net_sum = math_ops.reduce_sum(y, axis=axes, keepdims=True) + squared_mean = math_ops.reduce_sum( + math_ops.square(y), axis=axes, keepdims=True) + + if self._support_zero_size_input(): + # Keras assumes that batch dimension is the first dimension for Batch + # Normalization. + input_batch_size = array_ops.shape(y)[0] + else: + input_batch_size = None + + # get the number of total params you are averaging including batchsize(local) + axes_vals = [(array_ops.shape_v2(y))[i] for i in range(1, len(axes))] + multiplier = math_ops.cast( + math_ops.reduce_prod(axes_vals), dtypes.float32) + + squared_mean = squared_mean / multiplier + net_sum = net_sum / multiplier + + if input_batch_size is None: + mean, variance = nn.moments(y, axes, keep_dims=True) + input_batch_size = 0 + else: + batches_ = math_ops.cast(input_batch_size, self._param_dtype) + # # if you only have one replica dont worry about it + # # Compute true mean while keeping the dims for proper broadcasting. + mean = net_sum / batches_ + variance = squared_mean / batches_ - math_ops.square(mean) + + input_batch_size = math_ops.cast(input_batch_size, dtypes.int32) + if not keep_dims: + mean = array_ops.squeeze(mean, axes) + net_sum = array_ops.squeeze(net_sum, axes) + variance = array_ops.squeeze(variance, axes) + squared_mean = array_ops.squeeze(squared_mean, axes) + if x.dtype == dtypes.float16: + return (math_ops.cast(mean, dtypes.float16), + math_ops.cast(net_sum, dtypes.float16), + math_ops.cast(variance, dtypes.float16), + math_ops.cast(squared_mean, dtypes.float16), input_batch_size) + else: + return (mean, net_sum, variance, squared_mean, input_batch_size) + + +class ShuffleBatchNormalization(normalization.BatchNormalizationBase): + + def __init__(self, + axis=-1, + momentum=0.99, + epsilon=1e-3, + center=True, + scale=True, + beta_initializer='zeros', + gamma_initializer='ones', + moving_mean_initializer='zeros', + moving_variance_initializer='ones', + beta_regularizer=None, + gamma_regularizer=None, + beta_constraint=None, + gamma_constraint=None, + renorm=False, + renorm_clipping=None, + renorm_momentum=0.99, + trainable=True, + adjustment=None, + name=None, + **kwargs): + + # Currently we only support aggregating over the global batch size. + super(ShuffleBatchNormalization, self).__init__( + axis=axis, + momentum=momentum, + epsilon=epsilon, + center=center, + scale=scale, + beta_initializer=beta_initializer, + gamma_initializer=gamma_initializer, + moving_mean_initializer=moving_mean_initializer, + moving_variance_initializer=moving_variance_initializer, + beta_regularizer=beta_regularizer, + gamma_regularizer=gamma_regularizer, + beta_constraint=beta_constraint, + gamma_constraint=gamma_constraint, + renorm=renorm, + renorm_clipping=renorm_clipping, + renorm_momentum=renorm_momentum, + fused=False, + trainable=trainable, + virtual_batch_size=None, + name=name, + **kwargs) + + def _calculate_mean_and_var(self, x, axes, keep_dims): + + with K.name_scope('moments'): + # The dynamic range of fp16 is too limited to support the collection of + # sufficient statistics. As a workaround we simply perform the operations + # on 32-bit floats before converting the mean and variance back to fp16 + y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x + # if you only have one replica dont worry about it + # Compute true mean while keeping the dims for proper broadcasting. + mean = math_ops.reduce_mean(y, axes, keepdims=True, name='mean') + # sample variance, not unbiased variance + # Note: stop_gradient does not change the gradient that gets + # backpropagated to the mean from the variance calculation, + # because that gradient is zero + variance = math_ops.reduce_mean( + math_ops.squared_difference(y, array_ops.stop_gradient(mean)), + axes, + keepdims=True, + name='variance') + + replica_ctx = ds.get_replica_context() + if replica_ctx: + tf.print(replica_ctx.num_replicas_in_sync) + tf.print(replica_ctx.replica_id_in_sync_group) + + if not keep_dims: + mean = array_ops.squeeze(mean, axes) + variance = array_ops.squeeze(variance, axes) + if x.dtype == dtypes.float16: + return (math_ops.cast(mean, dtypes.float16), + math_ops.cast(variance, dtypes.float16)) + else: + return (mean, variance) diff --git a/centernet/utils/weight_utils/test_load_weights.py b/centernet/utils/weight_utils/test_load_weights.py index fd41318f7..e875b91f9 100644 --- a/centernet/utils/weight_utils/test_load_weights.py +++ b/centernet/utils/weight_utils/test_load_weights.py @@ -16,9 +16,6 @@ model, loss = build_centernet(input_specs=input_specs, task_config=config, l2_regularization=0) - - - # Test for running validation step on pretrained model # # Test for loading extremenet backbone weights From 1281546ce858ae7cd1c42ab3d6d64c0664ae5ecb Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 13 Apr 2021 11:03:51 -0400 Subject: [PATCH 092/132] tpu experiment config --- centernet/configs/centernet.py | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 23c2f3432..0a2e75ca7 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -321,3 +321,70 @@ def centernet_custom() -> cfg.ExperimentConfig: ]) return config + +@exp_factory.register_config_factory('centernet_tpu') +def centernet_tpu() -> cfg.ExperimentConfig: + """COCO object detection with CenterNet.""" + train_batch_size = 1 + eval_batch_size = 1 + base_default = 1200000 + num_batches = 1200000 * 64 / train_batch_size + + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'), + task=CenterNetTask( + model=CenterNet(), + train_data=DataConfig( # input_path=os.path.join( + # COCO_INPUT_PATH_BASE, 'train*'), + is_training=True, + global_batch_size=train_batch_size, + parser=Parser(), + shuffle_buffer_size=10000), + validation_data=DataConfig( + # input_path=os.path.join(COCO_INPUT_PATH_BASE, + # 'val*'), + is_training=False, + global_batch_size=eval_batch_size, + shuffle_buffer_size=10000)), + trainer=cfg.TrainerConfig( + steps_per_loop=2000, + summary_interval=8000, + checkpoint_interval=10000, + train_steps=num_batches, + validation_steps=1000, + validation_interval=10, + optimizer_config=optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [ + int(400000 / base_default * num_batches), + int(450000 / base_default * num_batches) + ], + 'values': [ + 0.00261 * train_batch_size / 64, + 0.000261 * train_batch_size / 64, + 0.0000261 * train_batch_size / 64 + ] + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 1000 * 64 // num_batches, + 'warmup_learning_rate': 0 + } + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + + return config From 46ad107843612507fb0dcff714b3fddbd45e254a Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 13 Apr 2021 15:43:40 +0000 Subject: [PATCH 093/132] tpu stuff --- centernet/configs/experiments/centernet-eval-tpu.yaml | 1 + centernet/modeling/CenterNet.py | 2 +- centernet/train.py | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index af3f93028..9ad26da51 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -1,6 +1,7 @@ runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' + tpu: 'node-8' task: model: base: diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 2b9edbab6..5fc3ad933 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -94,7 +94,7 @@ def build_centernet(input_specs, task_config, l2_regularization): model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) - # test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.float32) + # test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.bfloat16) # model(test_input) model.build(input_specs.shape) model.summary() diff --git a/centernet/train.py b/centernet/train.py index 5860accfa..1e9a812cd 100755 --- a/centernet/train.py +++ b/centernet/train.py @@ -54,7 +54,9 @@ nohup python3 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov4- --config_file=yolo/configs/experiments/yolov4-1gpu.yaml >> yolov4-1gpu.log & tail -f yolov4-1gpu.log nohup python3.8 -m yolo.train --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov3-1gpu_mosaic --config_file=yolo/configs/experiments/yolov3-1gpu.yaml >> yolov3-1gpu.log & tail -f yolov3-1gpu.log evalaute CenterNet: -nohup python -m centernet.train --mode=train_and_eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval.yaml >> centernet-eval.log & tail -f centernet-eval.log +nohup python -m centernet.train --mode=train_and_eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval-tpu.yaml >> centernet-eval.log & tail -f centernet-eval.log +evalaute CenterNet on TPU: +nohup python -m centernet.train --mode=eval --tpu=node-8 --experiment=centernet_tpu --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval-tpu.yaml >> centernet-eval.log & tail -f centernet-eval.log """ From c7098d052c737ab9566055ce44e68d66552b56f2 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 13 Apr 2021 15:57:57 +0000 Subject: [PATCH 094/132] getting cannot build error --- centernet/modeling/CenterNet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 5fc3ad933..7963411a9 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -93,7 +93,6 @@ def build_centernet(input_specs, task_config, l2_regularization): model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) - # test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.bfloat16) # model(test_input) model.build(input_specs.shape) From 24e516a83cc78cc47ab2e8fee9c5963750b24046 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 14 Apr 2021 00:15:12 +0000 Subject: [PATCH 095/132] testing --- .../experiments/.centernet-eval-tpu.yaml.swp | Bin 0 -> 4096 bytes .../configs/experiments/centernet-eval-tpu.yaml | 4 ++-- centernet/modeling/layers/detection_generator.py | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 centernet/configs/experiments/.centernet-eval-tpu.yaml.swp diff --git a/centernet/configs/experiments/.centernet-eval-tpu.yaml.swp b/centernet/configs/experiments/.centernet-eval-tpu.yaml.swp new file mode 100644 index 0000000000000000000000000000000000000000..9bf06aacba92061c683f70f8c333657c528aa8dd GIT binary patch literal 4096 zcmYc?2=nw+u+TGP00IF9h825D6Ihga7^0XN7?KlnjY?AUit~%oa`MYT3b2Fxl*EEe z>{2MAb*KjFrxY0XasyiAo1c=JQ>>qynpcuql$TnfpPZkUmYH6x zpIT9nT9la!6i1TP1(~8-Qc$W_nV6e{V&5odGz3ON067GB8H|k#4L~BwN{R}?LZQf# iqwLWT7!85Z5Eu=C(GVC7fzc2c4S~@R7!8489s&ULC^gvt literal 0 HcmV?d00001 diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index 9ad26da51..332408678 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -26,7 +26,7 @@ task: input_path: 'gs://tensorflow2/coco_records/val/2017*' global_batch_size: 8 tfds_download: true - dtype: float16 + dtype: bfloat16 is_training: false drop_remainder: true parser: @@ -38,7 +38,7 @@ task: gaussian_rad: -1 gaussian_iou: 0.7 output_dims: 128 - dtype: float32 + dtype: bfloat32 shuffle_buffer_size: 10000 subtasks: detection: diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 81ae80e60..7072a6627 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -288,10 +288,14 @@ class prediction for each box. def call(self, inputs): # Get heatmaps from decoded outputs via final hourglass stack output - ct_heatmaps = inputs['ct_heatmaps'][-1] - ct_sizes = inputs['ct_size'][-1] - ct_offsets = inputs['ct_offset'][-1] + all_ct_heatmaps = inputs['ct_heatmaps'] + all_ct_sizes = inputs['ct_size'] + all_ct_offsets = inputs['ct_offset'] + ct_heatmaps = all_ct_heatmaps[-1] + ct_sizes = all_ct_sizes[-1] + ct_offsets = all_ct_offsets[-1] + shape = tf.shape(ct_heatmaps) batch_size, height, width, num_channels = shape[0], shape[1], shape[2], shape[3] From 25afe15f961e67c3e8f5b051601d5b03f01a8f21 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 14 Apr 2021 16:04:17 +0000 Subject: [PATCH 096/132] casting in det gen --- .centernet-eval.log.swp | Bin 0 -> 24576 bytes .../experiments/.centernet-eval-tpu.yaml.swp | Bin 4096 -> 0 bytes .../experiments/centernet-eval-tpu.yaml | 2 +- centernet/modeling/CenterNet.py | 8 ++++---- .../modeling/decoders/centernet_decoder.py | 4 ++++ .../modeling/layers/detection_generator.py | 19 ++++++++++++------ 6 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 .centernet-eval.log.swp delete mode 100644 centernet/configs/experiments/.centernet-eval-tpu.yaml.swp diff --git a/.centernet-eval.log.swp b/.centernet-eval.log.swp new file mode 100644 index 0000000000000000000000000000000000000000..b022e2cf675d1010963a7fcfaf8e91d1ac1c7619 GIT binary patch literal 24576 zcmeI436vaV5r78*A`ldU2zX$V01a_>?%A20CrU^NZ-E3PiG(My>Fw^F>1^;NytH{Z} zXBqz+^7&Z*)?0>esJ>T_&qaPH<1Zr*{>}qs)-wKAs4#xk2{7|g_PR3tCzCZHm zIH0I~e?~qNc@N`%V)pwP|0AV7v&i>EemLXLAeWHW8UGE_Ka25SBi|48Z(#ga z$Z32%$@njke-QPze^>a1+W$0i`n(V0Pa)q0^<~DNME)Vge~f&0--w+2yPxqJkduGYrbqp|9yzT)dog|;a$0{5Wc*s>1`T{ytN_V@ybk;D?I0(C91S9aya{KIyFtzc=>_Qkc>~T{ zw}7kxIToY{@)n#y?*cg)WGP4o$TW~g;cUGUMBD>R2YCwaMXm+$LFR(I43pq4kkdep z1o;5S3lO*)L41(?LAC~25BInifK)&_L0*9vtpiyLBI0!n$heoZsY~|zbWhQpLECav zT`o)WRm(Fv5}PV7p@QnGif+1~Aj=b$Rz1aQ)D3sQ^o&W`P;_Cz6FQoqEO&YpTdf&j zZlPmO(7dRss_U-dd6Ures*S!r%TRjZ#c&nRJWkllj^~|MRb3r&Mxfkcw=rJ3TH+}; zJVU9d4bL-G8?4ppsJ?0(Q?2?=!>>1d2!BnumC2SSsb@E8ie{;vX9&IS^d!}4>X0H- zP22NTTMLvYZ`7=*6+;;WgN4j^v21lP)TwTk&S*g0U>J91Ch6U%p6N8eU^-bGM^Ub; zE{sViHzMo)a9s=#8B&r|KbIYM8knBqyQ*nVYOKhnE&8{v`XU5!#Ve;$&FrUaN^>+v zaShFJbuR@ONvUi)Q=C1n{fg!ELW)zb>T6XR+2ch>Hf*&Q=7QfBnN84~3D<{cTryqb zwo7-NI=E7U(C9&2CzwXEZa^wl4FUw38h%VVElpWVkrq)wRyThN}bT<9WHx~ zUSIY41%pWj-K}8@N94L|=%(f?xzQN6&Q%$97)67ws5tJhQZszl)J9v^TCnMf%$LN8 zod{NMJ6L%q={B@W?N^K;=qYG>zsMxCXb$GaZdbE~5^B4yBbJ?v(V6a=SVt_`qoETq z$fPG;?PXX26~79*rC4^-$^682g55-M`uaS>7gomBko!yvW-SyLk;7cdRx_JY>Qy~4 z;zwxOiYXH59=RscGlL?qH;EM!x9m@gohI0Bq};NxnX3(B{^-$d-eL0zTtLwJmEr zZBxk&W0{o)9T(=LQgu9Ew0o?9Aj5fU6WYQUj6U5M zf&y)a4%uzkiU9{8*BS=i=BVggv1(Q7;mm^wX-*?>m=BF$sj915urNk?PgEk-F{mV% z>a9IX4EoXV6k*CW>Xxag(3-YNYUYrkE3n9FrU#pXGSN;R3>zhw4p5z}55o~1Hf3(V z#W^u!q3BrMfCr8tOEhiFKIioh+_G-;sSOL(1#wIl z=Vd6}-bdn21Jnko6crB)A`Ow1G>7p(%G`q zBQ+0`aF(pqVQ#po<))v4;42yoMcoj0T}e$V=QGQoMqkn(93nYhT6)+VDa2lf1gArx z((F3a^dy*g62vr-YIts{*R)ghVZZ9wqInVwf`(;C;(kt-Msw>Ee*O61`9Iv=23HTh z(ycAM;-42bejNEBtX%y3Ka3FleLH^s9}W;2Kk@VbaD42A^Co`&9~~d@^Z#&s(DS$W z`G0hL#Lxf3@j>fp{QN&UKH}&9;rO8EOY!sn==g}A|A*rve*Pbhi}?9}bbiFo|D*fs zJN5bheqpyJCj0!~ukRh)|9>`gelt5-oc5<}cMc1Z zy;aEfMEwI8Ux|Dca+UEB|Mk7Fmeg&zn1YK;F^A}9YBF@7v^^8Y-> z73AdqV~l?WIr;w<-n_{Fr}OvOko*4%;|epJ-Cov1gy;t0eMh$9e3AdbNQZv;f8RF( z#xT(^B@!xwe?%5zE?~eg*=jJYz+}5I=D0kRZ!c z@~fsN)l?`_Rqcuat%9!%sjuNTT%$wsoKam#9dJzXJ20u?iGQJ3tu>_3+F~+WEQ}RW z=6(Ts5mI%1#}yShZZj6*bCB^K?XG+|S4ifIUEQT(TWEQ1X(qc&IugeM=EjEkr!a(C zdJC&8l!~SHDj_}88;0>!6Th%fMeeK^3mMUEvuDqi<_xH&C4M?2%?px9%aRhlnd67a zyN0)K3VzO44%4nvn2uadr$i<-yoAZ6WymvC<{q)6obC`Qr{Zmg1n<-PC3+UlKfK(` zU@cFP)zK+A8l7D3ZHvz0BbF$Oj$X9*pamj8%}-Miqo%b@*lvqdvGn$hBWP{87=&nw zV#BSqM$3}ii IqY<_CABb5O=>Px# literal 0 HcmV?d00001 diff --git a/centernet/configs/experiments/.centernet-eval-tpu.yaml.swp b/centernet/configs/experiments/.centernet-eval-tpu.yaml.swp deleted file mode 100644 index 9bf06aacba92061c683f70f8c333657c528aa8dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmYc?2=nw+u+TGP00IF9h825D6Ihga7^0XN7?KlnjY?AUit~%oa`MYT3b2Fxl*EEe z>{2MAb*KjFrxY0XasyiAo1c=JQ>>qynpcuql$TnfpPZkUmYH6x zpIT9nT9la!6i1TP1(~8-Qc$W_nV6e{V&5odGz3ON067GB8H|k#4L~BwN{R}?LZQf# iqwLWT7!85Z5Eu=C(GVC7fzc2c4S~@R7!8489s&ULC^gvt diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index 332408678..b98c6d0e0 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -38,7 +38,7 @@ task: gaussian_rad: -1 gaussian_iou: 0.7 output_dims: 128 - dtype: bfloat32 + dtype: bfloat16 shuffle_buffer_size: 10000 subtasks: detection: diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 7963411a9..016acbd07 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -93,10 +93,10 @@ def build_centernet(input_specs, task_config, l2_regularization): model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) - # test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.bfloat16) - # model(test_input) - model.build(input_specs.shape) - model.summary() + test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.bfloat16) + model(test_input) + # model.build(input_specs.shape) + # model.summary() # TODO: uncommend when filter is implemented # losses = filter.losses diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index fe0c1d2c0..73b4b9d98 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -51,6 +51,10 @@ def __init__(self, } super().__init__(inputs=inputs, outputs=outputs, name='CenterNetDecoder') + + @property + def output_specs(self): + return self._output_specs def get_config(self): layer_config = { diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 7072a6627..238c6f7e6 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -97,11 +97,11 @@ def process_heatmap(self, # Zero out everything that is not a peak. feature_map_peaks = ( - feature_map * tf.cast(feature_map_peak_mask, tf.float32)) + feature_map * tf.cast(feature_map_peak_mask, tf.bfloat16)) # Zero out peaks whose scores do not exceed threshold valid_peaks_mask = feature_map_peaks > center_thresh - feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.float32) + feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.bfloat16) return feature_map_peaks @@ -271,9 +271,16 @@ class prediction for each box. y_offsets = offsets[..., 0] x_offsets = offsets[..., 1] + + # Casting + y_indices = tf.cast(y_indices, dtype=tf.bfloat16) + x_indices = tf.cast(x_indices, dtype=tf.bfloat16) + + y_offsets = tf.cast(y_offsets, dtype=tf.bfloat16) + x_offsets = tf.cast(x_offsets, dtype=tf.bfloat16) - y_indices = tf.cast(y_indices, dtype=tf.float32) - x_indices = tf.cast(x_indices, dtype=tf.float32) + heights = tf.cast(heights, dtype=tf.bfloat16) + widths = tf.cast(widths, dtype=tf.bfloat16) detection_classes = channel_indices + self._class_offset @@ -307,13 +314,13 @@ def call(self, inputs): # Get top scores along with their x, y, and class scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, batch_size, width, num_channels, k=self._max_detections) - + # Parse the score and indices into bounding boxes boxes, classes, scores, num_det = self.get_boxes(scores, y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) # Normalize bounding boxes - boxes = boxes / tf.cast(height, tf.float32) + boxes = boxes / tf.cast(height, tf.bfloat16) # Apply nms if self._use_nms: From fe48982754341422ed2bcf0eed8d0560fe62f0fe Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 14 Apr 2021 16:23:48 -0400 Subject: [PATCH 097/132] added cast option to config --- centernet/configs/centernet.py | 1 + .../experiments/centernet-eval-tpu.yaml | 1 + centernet/modeling/CenterNet.py | 3 ++- centernet/modeling/backbones/hourglass.py | 6 ++--- .../modeling/layers/detection_generator.py | 26 ++++++++++++------- centernet/train.py | 2 +- 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 0a2e75ca7..0a95340b2 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -157,6 +157,7 @@ class CenterNetLayer(hyperparams.Config): center_thresh: float = 0.1 iou_thresh: float = 0.4 class_offset: int = 1 + dtype: str = 'float32' @dataclasses.dataclass class CenterNetDetection(hyperparams.Config): diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index b98c6d0e0..598bfa41d 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -22,6 +22,7 @@ task: center_thresh: 0.1 iou_thresh: 0.4 class_offset: 1 + dtype: bloat16 validation_data: input_path: 'gs://tensorflow2/coco_records/val/2017*' global_batch_size: 8 diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 016acbd07..e6be14f03 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -74,7 +74,8 @@ def build_centernet_filter(model_config): use_nms=model_config.filter.use_nms, center_thresh=model_config.filter.center_thresh, iou_thresh=model_config.filter.iou_thresh, - class_offset=model_config.filter.class_offset) + class_offset=model_config.filter.class_offset, + dtype=model_config.filter.dtype) def build_centernet_head(model_config): return None diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 1e21305d1..8571e0d90 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -132,10 +132,8 @@ def get_config(self): def output_specs(self): return self._output_specs -# @factory.register_backbone_builder('hourglass') - - -@register.backbone('hourglass', cfg.Hourglass) +# @register.backbone('hourglass', cfg.Hourglass) +@factory.register_backbone_builder('hourglass') def build_hourglass( input_specs: tf.keras.layers.InputSpec, model_config, diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 238c6f7e6..79da8b3ae 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -24,6 +24,7 @@ def __init__(self, center_thresh=0.1, iou_thresh=0.4, class_offset=1, + dtype='float32' **kwargs): """ Args: @@ -62,6 +63,13 @@ def __init__(self, self._iou_thresh = iou_thresh self._class_offset = class_offset + + if dtype == 'float16': + self._dtype = tf.float16 + elif dtype == 'bfloat16': + self._dtype = tf.bfloat16 + elif dtype == 'float32': + self._dtype = tf.float32 def process_heatmap(self, feature_map, @@ -97,11 +105,11 @@ def process_heatmap(self, # Zero out everything that is not a peak. feature_map_peaks = ( - feature_map * tf.cast(feature_map_peak_mask, tf.bfloat16)) + feature_map * tf.cast(feature_map_peak_mask, self._dtype)) # Zero out peaks whose scores do not exceed threshold valid_peaks_mask = feature_map_peaks > center_thresh - feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, tf.bfloat16) + feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, self._dtype) return feature_map_peaks @@ -273,14 +281,14 @@ class prediction for each box. x_offsets = offsets[..., 1] # Casting - y_indices = tf.cast(y_indices, dtype=tf.bfloat16) - x_indices = tf.cast(x_indices, dtype=tf.bfloat16) + y_indices = tf.cast(y_indices, dtype=self._dtype) + x_indices = tf.cast(x_indices, dtype=self._dtype) - y_offsets = tf.cast(y_offsets, dtype=tf.bfloat16) - x_offsets = tf.cast(x_offsets, dtype=tf.bfloat16) + y_offsets = tf.cast(y_offsets, dtype=self._dtype) + x_offsets = tf.cast(x_offsets, dtype=self._dtype) - heights = tf.cast(heights, dtype=tf.bfloat16) - widths = tf.cast(widths, dtype=tf.bfloat16) + heights = tf.cast(heights, dtype=self._dtype) + widths = tf.cast(widths, dtype=self._dtype) detection_classes = channel_indices + self._class_offset @@ -320,7 +328,7 @@ def call(self, inputs): y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) # Normalize bounding boxes - boxes = boxes / tf.cast(height, tf.bfloat16) + boxes = boxes / tf.cast(height, self._dtype) # Apply nms if self._use_nms: diff --git a/centernet/train.py b/centernet/train.py index 1e9a812cd..c36ded527 100755 --- a/centernet/train.py +++ b/centernet/train.py @@ -56,7 +56,7 @@ evalaute CenterNet: nohup python -m centernet.train --mode=train_and_eval --experiment=centernet_custom --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval-tpu.yaml >> centernet-eval.log & tail -f centernet-eval.log evalaute CenterNet on TPU: -nohup python -m centernet.train --mode=eval --tpu=node-8 --experiment=centernet_tpu --model_dir=../checkpoints/centernet- --config_file=centernet/configs/experiments/centernet-eval-tpu.yaml >> centernet-eval.log & tail -f centernet-eval.log +nohup python3 -m centernet.train --mode=eval --tpu=node-8 --experiment=centernet_tpu --model_dir=gs://tensorflow2/centernet-eval-tpu --config_file=centernet/configs/experiments/centernet-eval-tpu.yaml > centernet-eval.log & tail -f centernet-eval.log """ From d0ca63e3eae32fa9b22815e79b77b0e163904cc2 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 15 Apr 2021 23:15:38 +0000 Subject: [PATCH 098/132] fix model build on tpu --- .centernet-eval.log.swp | Bin 24576 -> 0 bytes centernet/modeling/CenterNet.py | 6 +- centernet/modeling/CenterNet_test.py | 4 -- centernet/modeling/backbones/hourglass.py | 4 +- .../modeling/decoders/centernet_decoder.py | 1 - .../decoders/centernet_decoder_test.py | 18 +++--- .../modeling/layers/detection_generator.py | 53 +++++++++--------- 7 files changed, 41 insertions(+), 45 deletions(-) delete mode 100644 .centernet-eval.log.swp diff --git a/.centernet-eval.log.swp b/.centernet-eval.log.swp deleted file mode 100644 index b022e2cf675d1010963a7fcfaf8e91d1ac1c7619..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24576 zcmeI436vaV5r78*A`ldU2zX$V01a_>?%A20CrU^NZ-E3PiG(My>Fw^F>1^;NytH{Z} zXBqz+^7&Z*)?0>esJ>T_&qaPH<1Zr*{>}qs)-wKAs4#xk2{7|g_PR3tCzCZHm zIH0I~e?~qNc@N`%V)pwP|0AV7v&i>EemLXLAeWHW8UGE_Ka25SBi|48Z(#ga z$Z32%$@njke-QPze^>a1+W$0i`n(V0Pa)q0^<~DNME)Vge~f&0--w+2yPxqJkduGYrbqp|9yzT)dog|;a$0{5Wc*s>1`T{ytN_V@ybk;D?I0(C91S9aya{KIyFtzc=>_Qkc>~T{ zw}7kxIToY{@)n#y?*cg)WGP4o$TW~g;cUGUMBD>R2YCwaMXm+$LFR(I43pq4kkdep z1o;5S3lO*)L41(?LAC~25BInifK)&_L0*9vtpiyLBI0!n$heoZsY~|zbWhQpLECav zT`o)WRm(Fv5}PV7p@QnGif+1~Aj=b$Rz1aQ)D3sQ^o&W`P;_Cz6FQoqEO&YpTdf&j zZlPmO(7dRss_U-dd6Ures*S!r%TRjZ#c&nRJWkllj^~|MRb3r&Mxfkcw=rJ3TH+}; zJVU9d4bL-G8?4ppsJ?0(Q?2?=!>>1d2!BnumC2SSsb@E8ie{;vX9&IS^d!}4>X0H- zP22NTTMLvYZ`7=*6+;;WgN4j^v21lP)TwTk&S*g0U>J91Ch6U%p6N8eU^-bGM^Ub; zE{sViHzMo)a9s=#8B&r|KbIYM8knBqyQ*nVYOKhnE&8{v`XU5!#Ve;$&FrUaN^>+v zaShFJbuR@ONvUi)Q=C1n{fg!ELW)zb>T6XR+2ch>Hf*&Q=7QfBnN84~3D<{cTryqb zwo7-NI=E7U(C9&2CzwXEZa^wl4FUw38h%VVElpWVkrq)wRyThN}bT<9WHx~ zUSIY41%pWj-K}8@N94L|=%(f?xzQN6&Q%$97)67ws5tJhQZszl)J9v^TCnMf%$LN8 zod{NMJ6L%q={B@W?N^K;=qYG>zsMxCXb$GaZdbE~5^B4yBbJ?v(V6a=SVt_`qoETq z$fPG;?PXX26~79*rC4^-$^682g55-M`uaS>7gomBko!yvW-SyLk;7cdRx_JY>Qy~4 z;zwxOiYXH59=RscGlL?qH;EM!x9m@gohI0Bq};NxnX3(B{^-$d-eL0zTtLwJmEr zZBxk&W0{o)9T(=LQgu9Ew0o?9Aj5fU6WYQUj6U5M zf&y)a4%uzkiU9{8*BS=i=BVggv1(Q7;mm^wX-*?>m=BF$sj915urNk?PgEk-F{mV% z>a9IX4EoXV6k*CW>Xxag(3-YNYUYrkE3n9FrU#pXGSN;R3>zhw4p5z}55o~1Hf3(V z#W^u!q3BrMfCr8tOEhiFKIioh+_G-;sSOL(1#wIl z=Vd6}-bdn21Jnko6crB)A`Ow1G>7p(%G`q zBQ+0`aF(pqVQ#po<))v4;42yoMcoj0T}e$V=QGQoMqkn(93nYhT6)+VDa2lf1gArx z((F3a^dy*g62vr-YIts{*R)ghVZZ9wqInVwf`(;C;(kt-Msw>Ee*O61`9Iv=23HTh z(ycAM;-42bejNEBtX%y3Ka3FleLH^s9}W;2Kk@VbaD42A^Co`&9~~d@^Z#&s(DS$W z`G0hL#Lxf3@j>fp{QN&UKH}&9;rO8EOY!sn==g}A|A*rve*Pbhi}?9}bbiFo|D*fs zJN5bheqpyJCj0!~ukRh)|9>`gelt5-oc5<}cMc1Z zy;aEfMEwI8Ux|Dca+UEB|Mk7Fmeg&zn1YK;F^A}9YBF@7v^^8Y-> z73AdqV~l?WIr;w<-n_{Fr}OvOko*4%;|epJ-Cov1gy;t0eMh$9e3AdbNQZv;f8RF( z#xT(^B@!xwe?%5zE?~eg*=jJYz+}5I=D0kRZ!c z@~fsN)l?`_Rqcuat%9!%sjuNTT%$wsoKam#9dJzXJ20u?iGQJ3tu>_3+F~+WEQ}RW z=6(Ts5mI%1#}yShZZj6*bCB^K?XG+|S4ifIUEQT(TWEQ1X(qc&IugeM=EjEkr!a(C zdJC&8l!~SHDj_}88;0>!6Th%fMeeK^3mMUEvuDqi<_xH&C4M?2%?px9%aRhlnd67a zyN0)K3VzO44%4nvn2uadr$i<-yoAZ6WymvC<{q)6obC`Qr{Zmg1n<-PC3+UlKfK(` zU@cFP)zK+A8l7D3ZHvz0BbF$Oj$X9*pamj8%}-Miqo%b@*lvqdvGn$hBWP{87=&nw zV#BSqM$3}ii IqY<_CABb5O=>Px# diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index e6be14f03..702f29a9c 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -94,10 +94,8 @@ def build_centernet(input_specs, task_config, l2_regularization): model = CenterNet(backbone=backbone, decoder=decoder, head=head, filter=filter) - test_input = tf.zeros(shape=(1, 512, 512, 3), dtype=tf.bfloat16) - model(test_input) - # model.build(input_specs.shape) - # model.summary() + model.build(input_specs.shape) + model.summary() # TODO: uncommend when filter is implemented # losses = filter.losses diff --git a/centernet/modeling/CenterNet_test.py b/centernet/modeling/CenterNet_test.py index 9996eefce..fc0c544e0 100644 --- a/centernet/modeling/CenterNet_test.py +++ b/centernet/modeling/CenterNet_test.py @@ -28,9 +28,5 @@ def testBuildCenterNet(self): self.assertEqual(outputs['raw_output']['ct_offset'][0].shape, (5, 128, 128, 2)) self.assertEqual(outputs['raw_output']['ct_size'][0].shape, (5, 128, 128, 2)) - model.summary() - - - if __name__ == '__main__': tf.test.main() diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 8571e0d90..ceaa3fc72 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -132,8 +132,8 @@ def get_config(self): def output_specs(self): return self._output_specs -# @register.backbone('hourglass', cfg.Hourglass) -@factory.register_backbone_builder('hourglass') +# @factory.register_backbone_builder('hourglass') +@register.backbone('hourglass', cfg.Hourglass) def build_hourglass( input_specs: tf.keras.layers.InputSpec, model_config, diff --git a/centernet/modeling/decoders/centernet_decoder.py b/centernet/modeling/decoders/centernet_decoder.py index 73b4b9d98..29de964c0 100644 --- a/centernet/modeling/decoders/centernet_decoder.py +++ b/centernet/modeling/decoders/centernet_decoder.py @@ -32,7 +32,6 @@ def __init__(self, self._heatmap_bias = heatmap_bias self._num_inputs = num_inputs - inputs = [tf.keras.layers.Input(shape=value[1:]) for value in self._input_specs] outputs = dict() diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index e4dc819f3..cc7139a0f 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -4,14 +4,15 @@ from centernet.configs import centernet as cfg from centernet.modeling.decoders import centernet_decoder +from centernet.modeling.CenterNet import build_centernet_decoder class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): def test_create_decoder(self): - decoder = centernet_decoder.build_centernet_decoder( + decoder = build_centernet_decoder( task_config=cfg.CenterNetTask(), - input_specs=(None, 128, 128, 256), + input_specs=[(None, 128, 128, 256), (None, 128, 128, 256)], num_inputs=2) config = decoder.get_config() @@ -19,9 +20,9 @@ def test_create_decoder(self): self.assertEqual(config['heatmap_bias'], -2.19) def test_decoder_shape(self): - decoder = centernet_decoder.build_centernet_decoder( - task_config=cfg.CenterNetTask(), - input_specs=(2, 128, 128, 256), + decoder = build_centernet_decoder( + task_config=cfg.CenterNetTask(), + input_specs=[(None, 128, 128, 256), (None, 128, 128, 256)], num_inputs=2) # Output shape tests @@ -33,9 +34,10 @@ def test_decoder_shape(self): self.assertEqual(outputs['ct_size'][0].shape, (2, 128, 128, 2)) # Weight initialization tests - hm_bias_vector = np.asarray(decoder.layers[1].weights[-1]) - off_bias_vector = np.asarray(decoder.layers[2].weights[-1]) - size_bias_vector = np.asarray(decoder.layers[3].weights[-1]) + tf.print("\n\n{}\n\n".format(decoder.layers)) + hm_bias_vector = np.asarray(decoder.layers[2].weights[-1]) + off_bias_vector = np.asarray(decoder.layers[4].weights[-1]) + size_bias_vector = np.asarray(decoder.layers[6].weights[-1]) self.assertArrayNear(hm_bias_vector, np.repeat(-2.19, repeats=90), err=1.00e-6) diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 79da8b3ae..298a2bc7a 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -24,7 +24,7 @@ def __init__(self, center_thresh=0.1, iou_thresh=0.4, class_offset=1, - dtype='float32' + dtype='float32', **kwargs): """ Args: @@ -63,13 +63,6 @@ def __init__(self, self._iou_thresh = iou_thresh self._class_offset = class_offset - - if dtype == 'float16': - self._dtype = tf.float16 - elif dtype == 'bfloat16': - self._dtype = tf.bfloat16 - elif dtype == 'float32': - self._dtype = tf.float32 def process_heatmap(self, feature_map, @@ -92,6 +85,8 @@ def process_heatmap(self, A Tensor with the same shape as the input but with non-valid center prediction locations masked out. """ + + # all dtypes in this function are float32 or bool (for masks) feature_map = tf.math.sigmoid(feature_map) if not kernel_size or kernel_size == 1: feature_map_peaks = feature_map @@ -102,15 +97,16 @@ def process_heatmap(self, feature_map_peak_mask = tf.math.abs( feature_map - feature_map_max_pool) < self._peak_error - + # Zero out everything that is not a peak. feature_map_peaks = ( - feature_map * tf.cast(feature_map_peak_mask, self._dtype)) - + feature_map * tf.cast(feature_map_peak_mask, feature_map.dtype)) + # Zero out peaks whose scores do not exceed threshold valid_peaks_mask = feature_map_peaks > center_thresh - feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, self._dtype) + feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, feature_map_peaks.dtype) + return feature_map_peaks def get_row_col_channel_indices_from_flattened_indices(self, @@ -137,6 +133,8 @@ def get_row_col_channel_indices_from_flattened_indices(self, """ # Avoid using mod operator to make the ops more easy to be compatible with # different environments, e.g. WASM. + + # all inputs and outputs are dtype int32 row_indices = (indices // num_channels) // num_cols col_indices = (indices // num_channels) - row_indices * num_cols channel_indices_temp = indices // num_channels @@ -259,36 +257,33 @@ class prediction for each box. """ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use # tf_gather_nd instead and here we prepare the indices for that. + + # combined indices dtype=int32 combined_indices = tf.stack([ self.multi_range(batch_size, value_repetitions=num_boxes), tf.reshape(y_indices, [-1]), tf.reshape(x_indices, [-1]) ], axis=1) + - # Get the heights and widths of center points new_height_width = tf.gather_nd(height_width_predictions, combined_indices) new_height_width = tf.reshape(new_height_width, [batch_size, num_boxes, -1]) height_width = tf.maximum(new_height_width, 0) + # height and widths dtype=float32 heights = height_width[..., 0] widths = height_width[..., 1] # Get the offsets of center points new_offsets = tf.gather_nd(offset_predictions, combined_indices) offsets = tf.reshape(new_offsets, [batch_size, num_boxes, -1]) - + + # offsets are dtype=float32 y_offsets = offsets[..., 0] x_offsets = offsets[..., 1] - # Casting - y_indices = tf.cast(y_indices, dtype=self._dtype) - x_indices = tf.cast(x_indices, dtype=self._dtype) - - y_offsets = tf.cast(y_offsets, dtype=self._dtype) - x_offsets = tf.cast(x_offsets, dtype=self._dtype) - - heights = tf.cast(heights, dtype=self._dtype) - widths = tf.cast(widths, dtype=self._dtype) + y_indices = tf.cast(y_indices, dtype=heights.dtype) + x_indices = tf.cast(x_indices, dtype=widths.dtype) detection_classes = channel_indices + self._class_offset @@ -307,28 +302,34 @@ def call(self, inputs): all_ct_sizes = inputs['ct_size'] all_ct_offsets = inputs['ct_offset'] + # Heatmaps below are all dtype=float32 ct_heatmaps = all_ct_heatmaps[-1] ct_sizes = all_ct_sizes[-1] ct_offsets = all_ct_offsets[-1] - + shape = tf.shape(ct_heatmaps) - batch_size, height, width, num_channels = shape[0], shape[1], shape[2], shape[3] + # Values below from shape array are all dtype=int32 + batch_size, height, width, num_channels = shape[0], shape[1], shape[2], shape[3] + + # Resulting peaks from process_heatmap are dtype=float32 # Process heatmaps using 3x3 max pool and applying sigmoid peaks = self.process_heatmap(ct_heatmaps, kernel_size=self._peak_extract_kernel_size, center_thresh=self._center_thresh) + # scores are dtype=float32, y, x, and channel indices are dtype=int32 # Get top scores along with their x, y, and class scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, batch_size, width, num_channels, k=self._max_detections) + # boxes and scores are dtype=float32, classes and num_dets are dtype=int32 # Parse the score and indices into bounding boxes boxes, classes, scores, num_det = self.get_boxes(scores, y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) # Normalize bounding boxes - boxes = boxes / tf.cast(height, self._dtype) + boxes = boxes / tf.cast(height, boxes.dtype) # Apply nms if self._use_nms: From 4b2fb2d2aec7f15fc9c3da75956d7499c2aae075 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 16 Apr 2021 06:26:16 +0000 Subject: [PATCH 099/132] model build & inputs on TPU --- centernet/dataloaders/centernet_input.py | 82 +++++----- centernet/dataloaders/centernet_input_test.py | 11 +- .../penalty_reduced_logistic_focal_loss.py | 1 - centernet/modeling/layers/nn_blocks.py | 9 +- centernet/ops/preprocessing_ops.py | 4 +- centernet/tasks/centernet.py | 21 ++- centernet/train_vm.py | 150 ++++++++++++++++++ 7 files changed, 225 insertions(+), 53 deletions(-) create mode 100644 centernet/train_vm.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 71c7f0986..2786925ea 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -68,9 +68,7 @@ def __init__(self, ) def _build_heatmap_and_regressed_features(self, - boxes, - classes, - num_objects, + labels, output_size=[128, 128], input_size=[512, 512]): """ Generates the ground truth labels for centernet. @@ -80,12 +78,13 @@ def _build_heatmap_and_regressed_features(self, generated. Args: - boxes: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension - corresponds to the top left x, top left y, bottom right x, and - bottom left y coordinates of the bounding box - classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each - box, given in the same order as the boxes - num_objects: A `Tensor` or int that gives the number of objects + labels: A dictionary of COCO ground truth labels with at minimum the following fields: + bbox: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension + corresponds to the top left x, top left y, bottom right x, and + bottom left y coordinates of the bounding box + classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each + box, given in the same order as the boxes + num_detections: A `Tensor` or int that gives the number of objects in the image output_size: A `list` of length 2 containing the desired output height and width of the heatmaps input_size: A `list` of length 2 the expected input height and width of @@ -120,32 +119,38 @@ def _build_heatmap_and_regressed_features(self, These are used to extract the regressed box features from the prediction when computing the loss """ - boxes = tf.cast(boxes, dtype=tf.float32) - classes = tf.cast(classes, dtype=tf.float32) - input_h, input_w = input_size - output_h, output_w = output_size - - # We will transpose these at the end - tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=tf.float32) - br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=tf.float32) - ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=tf.float32) - - tl_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - br_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - ct_offset = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) - size = tf.zeros((self._max_num_instances, 2), dtype=tf.float32) + # boxes and classes are cast to self._dtype already from build_label + boxes = labels['bbox'] + classes = labels['classes'] + input_size = tf.cast(input_size, self._dtype) + output_size = tf.cast(output_size, self._dtype) + input_h, input_w = input_size[0], input_size[1] + output_h, output_w = output_size[0], output_size[1] + + # We will transpose the heatmaps at the end + tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + + # Maps for offset and size predictions + tl_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + br_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + ct_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + size = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + + # Masks for valid boxes box_mask = tf.zeros((self._max_num_instances), dtype=tf.int32) box_indices = tf.zeros((self._max_num_instances, 2), dtype=tf.int32) # Scaling factor for determining center/corners - width_ratio = tf.cast(output_w / input_w, tf.float32) - height_ratio = tf.cast(output_h / input_h, tf.float32) + width_ratio = output_w / input_w + height_ratio = output_h / input_h - num_boxes = tf.shape(boxes)[0] + num_objects = labels['num_detections'] - height = 0.0 - width = 0.0 + height = tf.cast(0.0, self._dtype) + width = tf.cast(0.0, self._dtype) for tag_ind in tf.range(num_objects): box = boxes[tag_ind] obj_class = classes[tag_ind] - 1 # TODO: See if subtracting 1 from the class like the paper is unnecessary @@ -158,6 +163,7 @@ def _build_heatmap_and_regressed_features(self, ) # Scale center and corner locations + # These should be dtype=float32 fxtl = (xtl * width_ratio) fytl = (ytl * height_ratio) fxbr = (xbr * width_ratio) @@ -166,6 +172,7 @@ def _build_heatmap_and_regressed_features(self, fyct = (yct * height_ratio) # Fit center and corners onto the output image + # These should be dtype=float32 xtl = tf.math.floor(fxtl) ytl = tf.math.floor(fytl) xbr = tf.math.floor(fxbr) @@ -175,6 +182,7 @@ def _build_heatmap_and_regressed_features(self, # Splat gaussian at for the center/corner heatmaps if self._use_gaussian_bump: + # Check: do we need to normalize these boxes? width = box[3] - box[1] height = box[2] - box[0] @@ -183,18 +191,21 @@ def _build_heatmap_and_regressed_features(self, if self._gaussian_rad == -1: radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - radius = tf.math.maximum(0.0, tf.math.floor(radius)) + radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) else: radius = self._gaussian_rad - + tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[obj_class, xtl, ytl, radius]]) br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[obj_class, xbr, ybr, radius]]) ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[obj_class, xct, yct, radius]], scaling_factor=5) else: - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, [[obj_class, ytl, xtl]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, [[obj_class, ybr, xbr]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, [[obj_class, yct, xct]], [1]) + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, + [[tf.cast(obj_class, tf.int32), tf.cast(ytl, tf.int32), tf.cast(xtl, tf.int32)]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, + [[tf.cast(obj_class, tf.int32), tf.cast(ybr, tf.int32), tf.cast(xbr, tf.int32)]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, + [[tf.cast(obj_class, tf.int32), tf.cast(yct, tf.int32), tf.cast(xct, tf.int32)]], [1]) # Add box offset and size to the ground truth tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) @@ -327,9 +338,8 @@ def _build_label(self, image, boxes, classes, width, height, info, data, } heatmap_feature_labels = self._build_heatmap_and_regressed_features( - boxes=boxes, classes=classes, num_objects=num_detections, - output_size=[self._output_dims, self._output_dims], - input_size=[self._image_w, self._image_w] + labels, output_size=[self._output_dims, self._output_dims], + input_size=[self._image_h, self._image_w] ) labels.update(heatmap_feature_labels) return image, labels diff --git a/centernet/dataloaders/centernet_input_test.py b/centernet/dataloaders/centernet_input_test.py index fc43aeaa4..7876e9692 100755 --- a/centernet/dataloaders/centernet_input_test.py +++ b/centernet/dataloaders/centernet_input_test.py @@ -7,9 +7,12 @@ class CenterNetInputTest(tf.test.TestCase, parameterized.TestCase): def check_labels_correct(self, boxes, classes, output_size, input_size): parser = CenterNetParser() - labels = parser._build_labels( - boxes=tf.constant(boxes, dtype=tf.float32), - classes=tf.constant(classes, dtype=tf.float32), + labels = parser._build_heatmap_and_regressed_features( + labels = { + 'bbox': tf.constant(boxes, dtype=tf.float32), + 'num_detections': len(boxes), + 'classes': tf.constant(classes, dtype=tf.float32) + }, output_size=output_size, input_size=input_size) tl_heatmaps = labels['tl_heatmaps'] @@ -75,7 +78,7 @@ def test_generate_heatmap_no_scale(self): (100, 300, 150, 370), (15, 100, 200, 170), ] - classes = (0, 1, 2) + classes = (1, 2, 3) sizes = [512, 512] self.check_labels_correct(boxes=boxes, classes=classes, diff --git a/centernet/losses/penalty_reduced_logistic_focal_loss.py b/centernet/losses/penalty_reduced_logistic_focal_loss.py index 78dda198c..858788be2 100644 --- a/centernet/losses/penalty_reduced_logistic_focal_loss.py +++ b/centernet/losses/penalty_reduced_logistic_focal_loss.py @@ -51,7 +51,6 @@ def call(self, y_true, y_pred): prediction_tensor = tf.clip_by_value(tf.sigmoid(y_pred), self._sigmoid_clip_value, 1 - self._sigmoid_clip_value) - positive_loss = (tf.math.pow((1 - prediction_tensor), self._alpha)* tf.math.log(prediction_tensor)) negative_loss = (tf.math.pow((1 - target_tensor), self._beta)* diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index a8edab423..8fad8d8a8 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -3,7 +3,7 @@ from official.modeling import tf_utils from official.vision.beta.modeling.layers import \ nn_blocks as official_nn_blocks -from yolo.modeling.layers import subnormalization +from centernet.modeling.layers import subnormalization TPU_BASE = True @@ -308,13 +308,6 @@ def call(self, x): hg_output = self.decoder_block(inner_outputs) return self.upsample_layer(hg_output) + encoded_outputs - # x_pre_pooled = self.encoder_block1(x) - # x_side = self.encoder_block2(x_pre_xpooled) - # x_pooled = self.pool(x_pre_pooled) - # inner_output = self.inner_hg(x_pooled) - # hg_output = self.decoder_block(inner_output) - # return self.upsample_layer(hg_output) + x_side - def get_config(self): layer_config = { 'channel_dims_per_stage': self._channel_dims_per_stage, diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index d9fc4891e..66558fa7e 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -17,7 +17,7 @@ def _smallest_positive_root(a, b, c) -> tf.Tensor: root1 = (-b - discriminant_sqrt) / (2 * a) root2 = (-b + discriminant_sqrt) / (2 * a) - return tf.where(tf.less(discriminant, 0), LARGE_NUM, (-b + discriminant_sqrt) / (2)) + return tf.where(tf.less(discriminant, 0), tf.cast(LARGE_NUM, b.dtype), (-b + discriminant_sqrt) / (2)) # return tf.where(tf.less(discriminant, 0), LARGE_NUM, tf.where(tf.less(root1, 0), root2, root1)) def gaussian_radius(det_size, min_overlap=0.7) -> int: @@ -179,7 +179,7 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): masked_gaussian = masked_gaussian_ta.stack() heatmap_mask = heatmap_mask_ta.stack() heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) - heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, masked_gaussian * scaling_factor) + heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, tf.cast(masked_gaussian * scaling_factor, heatmap.dtype)) # print('after ',heatmap) return heatmap diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 2a53e999a..0fd9bdc6c 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -27,6 +27,7 @@ def __init__(self, params, logging_dir: str = None): self._metrics = [] def build_inputs(self, params, input_context=None): + print("\nIn build_inputs\n") """Build input dataset.""" decoder = self.get_decoder(params) model = self.task_config.model @@ -51,6 +52,7 @@ def build_inputs(self, params, input_context=None): return dataset def build_model(self): + print("\nIn build_model\n") """get an instance of CenterNet""" from centernet.modeling.CenterNet import build_centernet params = self.task_config.train_data @@ -67,6 +69,7 @@ def build_model(self): return model def initialize(self, model: tf.keras.Model): + print("\nIn initialize\n") """Initializes CenterNet model by loading pretrained weights """ if self.task_config.load_odapi_weights and self.task_config.load_extremenet_weights: raise ValueError('Only 1 of odapi or extremenet weights should be loaded') @@ -114,6 +117,7 @@ def get_decoder(self, params): def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): + print("\nIn build_losses\n") total_loss = 0.0 loss = 0.0 @@ -123,7 +127,7 @@ def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): object_center_loss_fn = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) - # Set up box indicies so that they have a batch element as well + # Set up box indices so that they have a batch element as well box_indices = loss_ops.add_batch_to_indices(labels['box_indices']) box_mask = tf.cast(labels['box_mask'], dtype=tf.float32) @@ -134,10 +138,12 @@ def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): true_flattened_ct_heatmap = loss_ops._flatten_spatial_dimensions( labels['ct_heatmaps']) + true_flattened_ct_heatmap = tf.cast(true_flattened_ct_heatmap, tf.float32) total_center_loss = 0.0 for ct_heatmap in pred_ct_heatmap_list: pred_flattened_ct_heatmap = loss_ops._flatten_spatial_dimensions( ct_heatmap) + pred_flattened_ct_heatmap = tf.cast(pred_flattened_ct_heatmap, tf.float32) total_center_loss += object_center_loss_fn( pred_flattened_ct_heatmap, true_flattened_ct_heatmap) @@ -147,10 +153,12 @@ def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): # Calculate scale loss pred_scale_list = outputs['ct_size'] true_scale = labels['size'] + true_scale = tf.cast(true_scale, tf.float32) total_scale_loss = 0.0 for scale_map in pred_scale_list: pred_scale = loss_ops.get_batch_predictions_from_indices(scale_map, box_indices) + pred_scale = tf.cast(pred_scale, tf.float32) # Only apply loss for boxes that appear in the ground truth total_scale_loss += tf.reduce_sum(localization_loss_fn(pred_scale, true_scale), axis=-1) * box_mask @@ -160,16 +168,19 @@ def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): # Calculate offset loss pred_offset_list = outputs['ct_offset'] true_offset = labels['ct_offset'] + true_offset = tf.cast(true_offset, tf.float32) total_offset_loss = 0.0 for offset_map in pred_offset_list: pred_offset = loss_ops.get_batch_predictions_from_indices(offset_map, box_indices) + pred_offset = tf.cast(pred_offset, tf.float32) # Only apply loss for boxes that appear in the ground truth total_offset_loss += tf.reduce_sum(localization_loss_fn(pred_offset, true_offset), axis=-1) * box_mask offset_loss = tf.reduce_sum(total_offset_loss) / float(len(pred_offset_list) * num_boxes) metric_dict['ct_offset_loss'] = offset_loss - + + # Aggregate and finalize loss loss_weights = self.task_config.losses.detection total_loss = (center_loss + loss_weights.scale_weight * scale_loss + @@ -179,6 +190,7 @@ def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): return total_loss, metric_dict def build_metrics(self, training=True): + print("\nIn build_metrics\n") metrics = [] metric_names = self._metric_names @@ -195,6 +207,7 @@ def build_metrics(self, training=True): return metrics def train_step(self, inputs, model, optimizer, metrics=None): + print("\nIn train_step\n") # get the data point image, label = inputs @@ -236,10 +249,12 @@ def train_step(self, inputs, model, optimizer, metrics=None): def validation_step(self, inputs, model, metrics=None): # get the data point + print("\nIn validation step\n") image, label = inputs y_pred = model(image, training=False) y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) + loss, loss_metrics = self.build_losses(y_pred['raw_output'], label) logs = {self.loss: loss_metrics['total_loss']} @@ -273,6 +288,7 @@ def validation_step(self, inputs, model, metrics=None): return logs def aggregate_logs(self, state=None, step_outputs=None): + print("\nIn aggregate_logs\n") if not state: self.coco_metric.reset_states() state = self.coco_metric @@ -281,4 +297,5 @@ def aggregate_logs(self, state=None, step_outputs=None): return state def reduce_aggregated_logs(self, aggregated_logs): + print("\nIn reduce_aggregate_logs\n") return self.coco_metric.result() diff --git a/centernet/train_vm.py b/centernet/train_vm.py new file mode 100644 index 000000000..2fc405bcb --- /dev/null +++ b/centernet/train_vm.py @@ -0,0 +1,150 @@ +# Lint as: python3 +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorFlow Model Garden Vision training driver.""" +from yolo.utils.run_utils import prep_gpu + +from absl import app +from absl import flags +import gin +import sys + +from official.core import train_utils +# pylint: disable=unused-import +from centernet.common import registry_imports +# pylint: enable=unused-import +from official.common import distribute_utils +from official.common import flags as tfm_flags +from official.core import task_factory +from official.core import train_lib +from official.modeling import performance +import tensorflow as tf + +FLAGS = flags.FLAGS +""" +get the cache file: +scp -i cache.zip purdue@:~/ +tensorboard: +on the vm: +nohup tensorboard --logdir ../checkpoints/yolov4-model --port 6006 >> temp.log +on your device: +ssh -i -N -f -L localhost:16006:localhost:6006 purdue@ +get the checkpoint from device: +scp -i purdue@:/checkpoints/.zip . +train darknet: +python3.8 -m yolo.train_vm --mode=train_and_eval --experiment=darknet_classification --model_dir=../checkpoints/darknet53 --config_file=yolo/configs/experiments/darknet/darknet53.yaml +python3 -m yolo.train_vm --mode=train_and_eval --experiment=darknet_classification --model_dir=../checkpoints/dilated_darknet53 --config_file=yolo/configs/experiments/dilated_darknet53.yaml +finetune darknet: +nohup python3 -m yolo.train_vm --mode=train_and_eval --experiment=darknet_classification --model_dir=../checkpoints/darknet53_remap_fn --config_file=yolo/configs/experiments/darknet53_leaky_fn_tune.yaml >> darknet53.log & tail -f darknet53.log +train yolo-v4: +nohup python3 -m yolo.train_vm --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov4-model --config_file=yolo/configs/experiments/yolov4.yaml >> yolov4.log & tail -f yolov4.log +nohup python3 -m yolo.train_vm --mode=train_and_eval --experiment=yolo_custom --model_dir=../checkpoints/yolov4- --config_file=yolo/configs/experiments/yolov4-1gpu.yaml >> yolov4-1gpu.log & tail -f yolov4-1gpu.log +evalaute CenterNet on TPU: +nohup python3 -m centernet.train_vm --mode=eval --tpu=node-8 --experiment=centernet_tpu --model_dir=gs://tensorflow2/centernet-eval-tpu --config_file=centernet/configs/experiments/centernet-eval-tpu.yaml > centernet-eval.log & tail -f centernet-eval.log +""" + + +def subdivison_adjustment(params): + tf.config.set_soft_device_placement(True) + if hasattr(params.task.model, + 'subdivisions') and params.task.model.subdivisions > 1: + print('adjustment is needed') + subdivisons = params.task.model.subdivisions + params.task.train_data.global_batch_size //= subdivisons + # params.task.validation_data.global_batch_size //= subdivisons + params.trainer.train_steps *= subdivisons + # params.trainer.validation_steps = subdivisons + params.trainer.validation_interval = (params.trainer.validation_interval // + subdivisons) * subdivisons + params.trainer.checkpoint_interval = (params.trainer.checkpoint_interval // + subdivisons) * subdivisons + params.trainer.steps_per_loop = (params.trainer.steps_per_loop // + subdivisons) * subdivisons + params.trainer.summary_interval = (params.trainer.summary_interval // + subdivisons) * subdivisons + + if params.trainer.optimizer_config.learning_rate.type == 'stepwise': + bounds = params.trainer.optimizer_config.learning_rate.stepwise.boundaries + params.trainer.optimizer_config.learning_rate.stepwise.boundaries = [ + subdivisons * bound for bound in bounds + ] + + if params.trainer.optimizer_config.learning_rate.type == 'polynomial': + params.trainer.optimizer_config.learning_rate.polynomial.decay_steps *= subdivisons + + if params.trainer.optimizer_config.optimizer.type == 'sgd': + print(params.trainer.optimizer_config.optimizer.type) + params.trainer.optimizer_config.optimizer.type = 'sgd_accum' + params.trainer.optimizer_config.optimizer.sgd_accum.accumulation_steps = subdivisons + params.trainer.optimizer_config.optimizer.sgd_accum.momentum = params.trainer.optimizer_config.optimizer.sgd.momentum + params.trainer.optimizer_config.optimizer.sgd_accum.decay = params.trainer.optimizer_config.optimizer.sgd.decay + + if params.trainer.optimizer_config.warmup.type == 'linear': + params.trainer.optimizer_config.warmup.linear.warmup_steps *= subdivisons + + print(params.as_dict()) + # sys.exit() + return params + + +def main(_): + gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params) + print(FLAGS.experiment) + params = train_utils.parse_configuration(FLAGS) + + params = subdivison_adjustment(params) + model_dir = FLAGS.model_dir + if 'train' in FLAGS.mode and model_dir != None: + # Pure eval modes do not output yaml files. Otherwise continuous eval job + # may race against the train job for writing the same file. + train_utils.serialize_config(params, model_dir) + + # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16' + # can have significant impact on model speeds by utilizing float16 in case of + # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when + # dtype is float16 + if params.runtime.mixed_precision_dtype: + performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype, + params.runtime.loss_scale) + if params.runtime.worker_hosts != '' and params.runtime.worker_hosts is not None: + num_workers = distribute_utils.configure_cluster( + worker_hosts=params.runtime.worker_hosts, + task_index=params.runtime.task_index) + print(num_workers) + distribution_strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=params.runtime.distribution_strategy, + all_reduce_alg=params.runtime.all_reduce_alg, + num_gpus=params.runtime.num_gpus, + tpu_address=params.runtime.tpu) + with distribution_strategy.scope(): + task = task_factory.get_task(params.task, logging_dir=model_dir) + + train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode=FLAGS.mode, + params=params, + model_dir=model_dir) + + +if __name__ == '__main__': + import datetime + + a = datetime.datetime.now() + tfm_flags.define_flags() + app.run(main) + b = datetime.datetime.now() + + print('\n\n\n\n\n\n\n {b - a}') From 09c9d053ded93329a7acae0826a2d0f7271b7d5e Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 16 Apr 2021 16:16:09 +0000 Subject: [PATCH 100/132] testing --- centernet/configs/centernet.py | 1 + .../experiments/centernet-eval-tpu.yaml | 2 +- centernet/tasks/centernet.py | 42 +++++++++++-------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 0a95340b2..3f8f359b1 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -158,6 +158,7 @@ class CenterNetLayer(hyperparams.Config): iou_thresh: float = 0.4 class_offset: int = 1 dtype: str = 'float32' + use_reduction_sum: bool = True @dataclasses.dataclass class CenterNetDetection(hyperparams.Config): diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index 598bfa41d..ca529a8b3 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -29,7 +29,7 @@ task: tfds_download: true dtype: bfloat16 is_training: false - drop_remainder: true + drop_remainder: false parser: image_h: 512 image_w: 512 diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 0fd9bdc6c..31dbf7b06 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -116,7 +116,7 @@ def get_decoder(self, params): return decoder - def build_losses(self, outputs, labels, num_replicas=1, aux_losses=None): + def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_losses=None): print("\nIn build_losses\n") total_loss = 0.0 loss = 0.0 @@ -248,14 +248,23 @@ def train_step(self, inputs, model, optimizer, metrics=None): return logs def validation_step(self, inputs, model, metrics=None): - # get the data point print("\nIn validation step\n") + # get the data point image, label = inputs + scale_replicas = tf.distribute.get_strategy().num_replicas_in_sync + if self._task_config.model.filter.use_reduction_sum: + num_replicas = 1 + else: + num_replicas = scale_replicas + y_pred = model(image, training=False) y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) - - loss, loss_metrics = self.build_losses(y_pred['raw_output'], label) + loss, loss_metrics = self.build_losses( + y_pred['raw_output'], + label, + num_replicas=num_replicas, + scale_replicas=1) logs = {self.loss: loss_metrics['total_loss']} image_shape = tf.shape(image)[1:-1] @@ -263,19 +272,19 @@ def validation_step(self, inputs, model, metrics=None): label['boxes'] = box_ops.denormalize_boxes( tf.cast(label['bbox'], tf.float32), image_shape) del label['bbox'] - + coco_model_outputs = { - 'detection_boxes': - box_ops.denormalize_boxes( - tf.cast(y_pred['bbox'], tf.float32), image_shape), - 'detection_scores': - y_pred['confidence'], - 'detection_classes': - y_pred['classes'], - 'num_detections': - y_pred['num_dets'], - 'source_id': - label['source_id'], + 'detection_boxes': + box_ops.denormalize_boxes( + tf.cast(y_pred['bbox'], tf.float32), image_shape), + 'detection_scores': + y_pred['confidence'], + 'detection_classes': + y_pred['classes'], + 'num_detections': + y_pred['num_dets'], + 'source_id': + label['source_id'], } logs.update({self.coco_metric.name: (label, coco_model_outputs)}) @@ -286,7 +295,6 @@ def validation_step(self, inputs, model, metrics=None): logs.update({m.name: m.result()}) return logs - def aggregate_logs(self, state=None, step_outputs=None): print("\nIn aggregate_logs\n") if not state: From f0b2d0cc91d32f7a9951adbba588a1378f4c4618 Mon Sep 17 00:00:00 2001 From: David Li Date: Sat, 17 Apr 2021 22:09:07 -0400 Subject: [PATCH 101/132] removed loops from gt builder, need to test speed --- centernet/dataloaders/centernet_input.py | 363 +++++++++++++----- .../decoders/centernet_decoder_test.py | 2 +- .../modeling/layers/detection_generator.py | 2 +- centernet/modeling/layers/nn_blocks.py | 2 +- centernet/ops/preprocessing_ops.py | 144 ++++--- centernet/ops/preprocessing_ops2.py | 50 +++ centernet/train_vm.py | 13 +- 7 files changed, 394 insertions(+), 182 deletions(-) create mode 100644 centernet/ops/preprocessing_ops2.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 2786925ea..d51e88a8c 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -52,7 +52,7 @@ def __init__(self, self._num_classes = num_classes self._max_num_instances = max_num_instances self._gaussian_iou = gaussian_iou - self._use_gaussian_bump = True + self._use_gaussian_bump = use_gaussian_bump self._gaussian_rad = -1 self._output_dims = output_dims @@ -67,6 +67,176 @@ def __init__(self, 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' ) + # def _build_heatmap_and_regressed_features(self, + # labels, + # output_size=[128, 128], + # input_size=[512, 512]): + # """ Generates the ground truth labels for centernet. + + # Ground truth labels are generated by splatting gaussians on heatmaps for + # corners and centers. Regressed features (offsets and sizes) are also + # generated. + + # Args: + # labels: A dictionary of COCO ground truth labels with at minimum the following fields: + # bbox: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension + # corresponds to the top left x, top left y, bottom right x, and + # bottom left y coordinates of the bounding box + # classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each + # box, given in the same order as the boxes + # num_detections: A `Tensor` or int that gives the number of objects in the image + # output_size: A `list` of length 2 containing the desired output height + # and width of the heatmaps + # input_size: A `list` of length 2 the expected input height and width of + # the image + # Returns: + # Dictionary of labels with the following fields: + # 'tl_heatmaps': A `Tensor` of shape [output_h, output_w, num_classes], + # heatmap with splatted gaussians centered at the positions and channels + # corresponding to the top left location and class of the object + # 'br_heatmaps': `Tensor` of shape [output_h, output_w, num_classes], + # heatmap with splatted gaussians centered at the positions and channels + # corresponding to the bottom right location and class of the object + # 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], + # heatmap with splatted gaussians centered at the positions and channels + # corresponding to the center location and class of the object + # 'tl_offset': `Tensor` of shape [max_num_instances, 2], where the first + # num_boxes entries contain the x-offset and y-offset of the top-left + # corner of an object. All other entires are 0 + # 'br_offset': `Tensor` of shape [max_num_instances, 2], where the first + # num_boxes entries contain the x-offset and y-offset of the + # bottom-right corner of an object. All other entires are 0 + # 'ct_offset': `Tensor` of shape [max_num_instances, 2], where the first + # num_boxes entries contain the x-offset and y-offset of the center of + # an object. All other entires are 0 + # 'size': `Tensor` of shape [max_num_instances, 2], where the first + # num_boxes entries contain the width and height of an object. All + # other entires are 0 + # 'box_mask': `Tensor` of shape [max_num_instances], where the first + # num_boxes entries are 1. All other entires are 0 + # 'box_indices': `Tensor` of shape [max_num_instances, 2], where the first + # num_boxes entries contain the y-center and x-center of a valid box. + # These are used to extract the regressed box features from the + # prediction when computing the loss + # """ + + # # boxes and classes are cast to self._dtype already from build_label + # boxes = labels['bbox'] + # classes = labels['classes'] + # input_size = tf.cast(input_size, self._dtype) + # output_size = tf.cast(output_size, self._dtype) + # input_h, input_w = input_size[0], input_size[1] + # output_h, output_w = output_size[0], output_size[1] + + # # We will transpose the heatmaps at the end + # tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + # br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + # ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + + # # Maps for offset and size predictions + # tl_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + # br_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + # ct_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + # size = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + + # # Masks for valid boxes + # box_mask = tf.zeros((self._max_num_instances), dtype=tf.int32) + # box_indices = tf.zeros((self._max_num_instances, 2), dtype=tf.int32) + + # # Scaling factor for determining center/corners + # width_ratio = output_w / input_w + # height_ratio = output_h / input_h + + # num_objects = labels['num_detections'] + + # height = tf.cast(0.0, self._dtype) + # width = tf.cast(0.0, self._dtype) + # for tag_ind in tf.range(num_objects): + # box = boxes[tag_ind] + # obj_class = classes[tag_ind] - 1 # TODO: See if subtracting 1 from the class like the paper is unnecessary + + # ytl, xtl, ybr, xbr = box[0], box[1], box[2], box[3] + + # xct, yct = ( + # (xtl + xbr) / 2, + # (ytl + ybr) / 2 + # ) + + # # Scale center and corner locations + # # These should be dtype=float32 + # fxtl = (xtl * width_ratio) + # fytl = (ytl * height_ratio) + # fxbr = (xbr * width_ratio) + # fybr = (ybr * height_ratio) + # fxct = (xct * width_ratio) + # fyct = (yct * height_ratio) + + # # Fit center and corners onto the output image + # # These should be dtype=float32 + # xtl = tf.math.floor(fxtl) + # ytl = tf.math.floor(fytl) + # xbr = tf.math.floor(fxbr) + # ybr = tf.math.floor(fybr) + # xct = tf.math.floor(fxct) + # yct = tf.math.floor(fyct) + + # # Splat gaussian at for the center/corner heatmaps + # if self._use_gaussian_bump: + # # Check: do we need to normalize these boxes? + # width = box[3] - box[1] + # height = box[2] - box[0] + + # width = tf.math.ceil(width * width_ratio) + # height = tf.math.ceil(height * height_ratio) + + # if self._gaussian_rad == -1: + # radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) + # radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) + # else: + # radius = self._gaussian_rad + + # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[obj_class, xtl, ytl, radius]]) + # br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[obj_class, xbr, ybr, radius]]) + # ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[obj_class, xct, yct, radius]], scaling_factor=5) + + # else: + # tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, + # [[tf.cast(obj_class, tf.int32), tf.cast(ytl, tf.int32), tf.cast(xtl, tf.int32)]], [1]) + # br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, + # [[tf.cast(obj_class, tf.int32), tf.cast(ybr, tf.int32), tf.cast(xbr, tf.int32)]], [1]) + # ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, + # [[tf.cast(obj_class, tf.int32), tf.cast(yct, tf.int32), tf.cast(xct, tf.int32)]], [1]) + + # # Add box offset and size to the ground truth + # tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + # br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + # ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) + # size = tf.tensor_scatter_nd_update(size, [[tag_ind, 0], [tag_ind, 1]], [width, height]) + + # # Initialy the mask is zeros, but each valid box needs to be unmasked + # box_mask = tf.tensor_scatter_nd_update(box_mask, [[tag_ind]], [1]) + + # # Contains the y and x coordinate of the box center in the heatmap + # box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1]], [yct, xct]) + + # # Make heatmaps of shape [height, width, num_classes] + # tl_heatmaps = tf.transpose(tl_heatmaps, perm=[1, 2, 0]) + # br_heatmaps = tf.transpose(br_heatmaps, perm=[1, 2, 0]) + # ct_heatmaps = tf.transpose(ct_heatmaps, perm=[1, 2, 0]) + + # labels = { + # 'tl_heatmaps': tl_heatmaps, + # 'br_heatmaps': br_heatmaps, + # 'ct_heatmaps': ct_heatmaps, + # 'tl_offset': tl_offset, + # 'br_offset': br_offset, + # 'ct_offset': ct_offset, + # 'size': size, + # 'box_mask': box_mask, + # 'box_indices': box_indices + # } + # return labels + def _build_heatmap_and_regressed_features(self, labels, output_size=[128, 128], @@ -118,20 +288,21 @@ def _build_heatmap_and_regressed_features(self, num_boxes entries contain the y-center and x-center of a valid box. These are used to extract the regressed box features from the prediction when computing the loss - """ - - # boxes and classes are cast to self._dtype already from build_label + """ + boxes = labels['bbox'] - classes = labels['classes'] + classes = labels['classes'] - 1 + num_objects = labels['num_detections'] + input_size = tf.cast(input_size, self._dtype) output_size = tf.cast(output_size, self._dtype) input_h, input_w = input_size[0], input_size[1] output_h, output_w = output_size[0], output_size[1] - # We will transpose the heatmaps at the end - tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + # Heatmaps which we will transpose at the end + tl_heatmap = tf.zeros((output_h, output_w, self._num_classes), dtype=self._dtype) + br_heatmap = tf.zeros((output_h, output_w, self._num_classes), dtype=self._dtype) + ct_heatmap = tf.zeros((output_h, output_w, self._num_classes), dtype=self._dtype) # Maps for offset and size predictions tl_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) @@ -146,88 +317,93 @@ def _build_heatmap_and_regressed_features(self, # Scaling factor for determining center/corners width_ratio = output_w / input_w height_ratio = output_h / input_h + + # Original box coordinates + ytl, ybr = boxes[..., 0], boxes[..., 2] + xtl, xbr = boxes[..., 1], boxes[..., 3] + yct = (ytl + ybr) / 2 + xct = (xtl + xbr) / 2 + + # Scaled box coordinates + fxtl = xtl * width_ratio + fytl = ytl * height_ratio + fxbr = xbr * width_ratio + fybr = ybr * height_ratio + fxct = xct * width_ratio + fyct = yct * height_ratio + + # Scaled box coordinate positions on heatmaps + xtl = tf.math.floor(fxtl) + ytl = tf.math.floor(fytl) + xbr = tf.math.floor(fxbr) + ybr = tf.math.floor(fybr) + xct = tf.math.floor(fxct) + yct = tf.math.floor(fyct) + + box_widths = boxes[..., 3] - boxes[..., 1] + box_heights = boxes[..., 2] - boxes[..., 0] - num_objects = labels['num_detections'] - - height = tf.cast(0.0, self._dtype) - width = tf.cast(0.0, self._dtype) - for tag_ind in tf.range(num_objects): - box = boxes[tag_ind] - obj_class = classes[tag_ind] - 1 # TODO: See if subtracting 1 from the class like the paper is unnecessary - - ytl, xtl, ybr, xbr = box[0], box[1], box[2], box[3] + box_widths = tf.math.ceil(box_widths * width_ratio) + box_heights = tf.math.ceil(box_heights * height_ratio) + box_widths_heights = tf.stack([box_widths, box_heights], axis=-1) - xct, yct = ( - (xtl + xbr) / 2, - (ytl + ybr) / 2 - ) + if self._use_gaussian_bump: + if self._gaussian_rad == -1: + radius = tf.map_fn(fn=lambda x: preprocessing_ops.gaussian_radius(x), elems=box_widths_heights) + radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) + else: + radius = tf.constant([self._gaussian_rad] * num_objects, dtype=self._dtype) + + tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) + br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) + ct_blobs = tf.stack([classes, xct, yct, radius], axis=-1) + + tl_gaussians = tf.map_fn(fn = lambda x: preprocessing_ops.draw_gaussian(tl_heatmap, x), elems=tl_blobs) + br_gaussians = tf.map_fn(fn = lambda x: preprocessing_ops.draw_gaussian(br_heatmap, x), elems=br_blobs) + ct_gaussians = tf.map_fn(fn = lambda x: preprocessing_ops.draw_gaussian(ct_heatmap, x, 5), elems=ct_blobs) + + tl_heatmap = tf.math.reduce_max(tl_gaussians, axis=0) + br_heatmap = tf.math.reduce_max(br_gaussians, axis=0) + ct_heatmap = tf.math.reduce_max(ct_gaussians, axis=0) + + else: + tl_hm_update_indices = tf.cast(tf.stack([ytl, xtl, classes], axis=-1), tf.int32) + br_hm_update_indices = tf.cast(tf.stack([ybr, xbr, classes], axis=-1), tf.int32) + ct_hm_update_indices = tf.cast(tf.stack([yct, xct, classes], axis=-1), tf.int32) + + tl_heatmap = tf.tensor_scatter_nd_update(tl_heatmap, + tl_hm_update_indices, [1] * num_objects) + br_heatmap = tf.tensor_scatter_nd_update(br_heatmap, + br_hm_update_indices, [1] * num_objects) + ct_heatmap = tf.tensor_scatter_nd_update(ct_heatmap, + ct_hm_update_indices, [1] * num_objects) + + update_indices = preprocessing_ops.cartesian_product(tf.range(num_objects), tf.range(2)) + update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) - # Scale center and corner locations - # These should be dtype=float32 - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - # Fit center and corners onto the output image - # These should be dtype=float32 - xtl = tf.math.floor(fxtl) - ytl = tf.math.floor(fytl) - xbr = tf.math.floor(fxbr) - ybr = tf.math.floor(fybr) - xct = tf.math.floor(fxct) - yct = tf.math.floor(fyct) - - # Splat gaussian at for the center/corner heatmaps - if self._use_gaussian_bump: - # Check: do we need to normalize these boxes? - width = box[3] - box[1] - height = box[2] - box[0] - - width = tf.math.ceil(width * width_ratio) - height = tf.math.ceil(height * height_ratio) - - if self._gaussian_rad == -1: - radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) - else: - radius = self._gaussian_rad - - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[obj_class, xtl, ytl, radius]]) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[obj_class, xbr, ybr, radius]]) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[obj_class, xct, yct, radius]], scaling_factor=5) + tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) + br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) + ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) - else: - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, - [[tf.cast(obj_class, tf.int32), tf.cast(ytl, tf.int32), tf.cast(xtl, tf.int32)]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, - [[tf.cast(obj_class, tf.int32), tf.cast(ybr, tf.int32), tf.cast(xbr, tf.int32)]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, - [[tf.cast(obj_class, tf.int32), tf.cast(yct, tf.int32), tf.cast(xct, tf.int32)]], [1]) - - # Add box offset and size to the ground truth - tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - size = tf.tensor_scatter_nd_update(size, [[tag_ind, 0], [tag_ind, 1]], [width, height]) + tl_offset = tf.tensor_scatter_nd_update(tl_offset, update_indices, tl_offset_values) + br_offset = tf.tensor_scatter_nd_update(br_offset, update_indices, br_offset_values) + ct_offset = tf.tensor_scatter_nd_update(ct_offset, update_indices, ct_offset_values) - # Initialy the mask is zeros, but each valid box needs to be unmasked - box_mask = tf.tensor_scatter_nd_update(box_mask, [[tag_ind]], [1]) + size = tf.tensor_scatter_nd_update(size, update_indices, box_widths_heights) - # Contains the y and x coordinate of the box center in the heatmap - box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1]], [yct, xct]) + # Initialy the mask is zeros, but each valid box needs to be unmasked + mask_indices = tf.expand_dims(tf.range(num_objects), -1) + mask_values = tf.repeat(1, num_objects) + box_mask = tf.tensor_scatter_nd_update(box_mask, mask_indices, mask_values) - # Make heatmaps of shape [height, width, num_classes] - tl_heatmaps = tf.transpose(tl_heatmaps, perm=[1, 2, 0]) - br_heatmaps = tf.transpose(br_heatmaps, perm=[1, 2, 0]) - ct_heatmaps = tf.transpose(ct_heatmaps, perm=[1, 2, 0]) + # Contains the y and x coordinate of the box center in the heatmap + box_index_values = tf.cast(tf.stack([yct, xct], axis=-1), dtype=tf.int32) + box_indices = tf.tensor_scatter_nd_update(box_indices, update_indices, box_index_values) labels = { - 'tl_heatmaps': tl_heatmaps, - 'br_heatmaps': br_heatmaps, - 'ct_heatmaps': ct_heatmaps, + 'tl_heatmaps': tl_heatmap, + 'br_heatmaps': br_heatmap, + 'ct_heatmaps': ct_heatmap, 'tl_offset': tl_offset, 'br_offset': br_offset, 'ct_offset': ct_offset, @@ -355,23 +531,24 @@ def postprocess_fn(self, is_training): # This code is for visualization import matplotlib.pyplot as plt boxes = [ - (10, 300, 15, 370), - (100, 300, 150, 370), - (200, 100, 15, 170), + (10, 300, 15, 370), # center (y, x) = (12, 335) + (100, 300, 150, 370), # center (y, x) = (125, 335) + (15, 100, 200, 170), # center (y, x) = (107, 135) ] - classes = (0, 1, 2) + classes = (1, 1, 1) - labels = CenterNetParser()._build_labels( - tf.constant(boxes, dtype=tf.float32), - tf.constant(classes, dtype=tf.float32), - [512, 512], [512, 512] + labels = CenterNetParser(use_gaussian_bump=True)._build_heatmap_and_regressed_features( + labels = { + 'bbox': tf.constant(boxes, dtype=tf.float32), + 'num_detections': len(boxes), + 'classes': tf.constant(classes, dtype=tf.float32) + }, + output_size=[512, 512], input_size=[512, 512] ) tl_heatmaps = labels['tl_heatmaps'] br_heatmaps = labels['br_heatmaps'] ct_heatmaps = labels['ct_heatmaps'] - # plt.imshow(ct_heatmaps[0, ...]) - plt.imshow(ct_heatmaps[..., 1]) - # plt.imshow(ct_heatmaps[2, ...]) + plt.imshow(ct_heatmaps[..., 0]) plt.show() diff --git a/centernet/modeling/decoders/centernet_decoder_test.py b/centernet/modeling/decoders/centernet_decoder_test.py index cc7139a0f..67b6a0ce1 100644 --- a/centernet/modeling/decoders/centernet_decoder_test.py +++ b/centernet/modeling/decoders/centernet_decoder_test.py @@ -3,8 +3,8 @@ from absl.testing import parameterized from centernet.configs import centernet as cfg -from centernet.modeling.decoders import centernet_decoder from centernet.modeling.CenterNet import build_centernet_decoder +from centernet.modeling.decoders import centernet_decoder class CenterNetDecoderTest(tf.test.TestCase, parameterized.TestCase): diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 298a2bc7a..138cc1a19 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -347,5 +347,5 @@ def call(self, inputs): 'bbox': boxes, 'classes': classes, 'confidence': scores, - 'num_dets': num_det + 'num_dets': num_dets } diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 8fad8d8a8..c0495064a 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -1,9 +1,9 @@ import tensorflow as tf +from centernet.modeling.layers import subnormalization from official.modeling import tf_utils from official.vision.beta.modeling.layers import \ nn_blocks as official_nn_blocks -from centernet.modeling.layers import subnormalization TPU_BASE = True diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 66558fa7e..54e11001e 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -124,96 +124,84 @@ def write_all(ta, index, values): return ta, index + i # scaling_factor doesn't do anything right now -def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): - """ - Draws a gaussian heatmap around a center point given a radius. - Params: - heatmap (tf.Tensor): heatmap placeholder to fill - blobs (tf.Tensor): a tensor whose last dimension is 4 integers for - the category of the object, center (x, y), and for radius of the - gaussian - scaling_factor (int): scaling factor for gaussian - """ - blobs = tf.cast(blobs, tf.int32) - category = blobs[..., 0] - x = blobs[..., 1] - y = blobs[..., 2] - radius = blobs[..., 3] - num_boxes = tf.shape(radius)[0] - - diameter = 2 * radius + 1 - - heatmap_shape = tf.shape(heatmap) - height, width = heatmap_shape[-2], heatmap_shape[-1] - - left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) - top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) - - # print('heatmap ',heatmap) - # print(len(heatmap)) - # print('category ',category) - - # TODO: make sure this replicates original functionality - # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] - # masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] - # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - update_count = tf.reduce_sum((bottom + top) * (right + left)) - masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) - heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((3,)), size=update_count) - i = 0 - for j in range(num_boxes): - cat = category[j] - X = x[j] - Y = y[j] - R = radius[j] - l = left[j] - r = right[j] - t = top[j] - b = bottom[j] - - gaussian = _gaussian_penalty(R, dtype=dtype) - masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) - heatmap_mask_instance = cartesian_product([cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) - masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) - heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) - masked_gaussian = masked_gaussian_ta.stack() - heatmap_mask = heatmap_mask_ta.stack() - heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) - heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, tf.cast(masked_gaussian * scaling_factor, heatmap.dtype)) - # print('after ',heatmap) - return heatmap - -# def draw_gaussian(heatmap, category, center, radius, scaling_factor=1): +# def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): # """ # Draws a gaussian heatmap around a center point given a radius. # Params: # heatmap (tf.Tensor): heatmap placeholder to fill -# center (int): integer for center of gaussian -# radius (int): integer for radius of gaussian +# blobs (tf.Tensor): a tensor whose last dimension is 4 integers for +# the category of the object, center (x, y), and for radius of the +# gaussian # scaling_factor (int): scaling factor for gaussian # """ +# blobs = tf.cast(blobs, tf.int32) +# category = blobs[..., 0] +# x = blobs[..., 1] +# y = blobs[..., 2] +# radius = blobs[..., 3] +# num_boxes = tf.shape(radius)[0] # diameter = 2 * radius + 1 -# gaussian = _gaussian_penalty(radius) -# x, y = center +# heatmap_shape = tf.shape(heatmap) +# height, width = heatmap_shape[-2], heatmap_shape[-1] + +# left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) +# top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) + +# # print('heatmap ',heatmap) +# # print(len(heatmap)) +# # print('category ',category) + +# # TODO: make sure this replicates original functionality +# # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] +# # masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] +# # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) +# update_count = tf.reduce_sum((bottom + top) * (right + left)) +# masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) +# heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((3,)), size=update_count) +# i = 0 +# for j in range(num_boxes): +# cat = category[j] +# X = x[j] +# Y = y[j] +# R = radius[j] +# l = left[j] +# r = right[j] +# t = top[j] +# b = bottom[j] + +# gaussian = _gaussian_penalty(R, dtype=dtype) +# masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) +# heatmap_mask_instance = cartesian_product([cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) +# masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) +# heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) +# masked_gaussian = masked_gaussian_ta.stack() +# heatmap_mask = heatmap_mask_ta.stack() +# heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) +# heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, tf.cast(masked_gaussian * scaling_factor, heatmap.dtype)) +# # print('after ',heatmap) +# return heatmap + +@tf.function +def draw_gaussian(heatmap, blob, scaling_factor=1): + # dummy heatmap to draw onto + heatmap_shape = tf.shape(heatmap) + gaussian_heatmap = tf.zeros(shape=heatmap_shape, dtype=heatmap.dtype) -# height, width = heatmap.shape[0:2] + blob = tf.cast(blob, tf.int32) + obj_class, x, y, radius = blob[0], blob[1], blob[2], blob[3] -# left, right = min(x, radius), min(width - x, radius + 1) -# top, bottom = min(y, radius), min(height - y, radius + 1) + height, width = heatmap_shape[0], heatmap_shape[1] -# print('heatmap ',heatmap) -# print(len(heatmap)) -# print('category ',category) + left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) + top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) -# heatmap_category = heatmap[0][category, ...] + gaussian = _gaussian_penalty(radius=radius, dtype=heatmap.dtype) + gaussian = gaussian[radius-top:radius+bottom, radius-left:radius+right] + gaussian = tf.reshape(gaussian, [-1]) -# print('heatmap_category ',heatmap_category) + heatmap_indices = cartesian_product(tf.range(y - top, y + bottom), tf.range(x - left, x + right), [obj_class]) + gaussian_heatmap = tf.tensor_scatter_nd_update(gaussian_heatmap, heatmap_indices, gaussian * scaling_factor) -# masked_heatmap = heatmap_category[y - top:y + bottom, x - left:x + right] -# masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] -# # TODO: make sure this replicates original functionality -# # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) -# masked_heatmap = tf.math.maximum(masked_heatmap, masked_gaussian * scaling_factor) -# return masked_heatmap + return gaussian_heatmap diff --git a/centernet/ops/preprocessing_ops2.py b/centernet/ops/preprocessing_ops2.py new file mode 100644 index 000000000..b56bf15ea --- /dev/null +++ b/centernet/ops/preprocessing_ops2.py @@ -0,0 +1,50 @@ +def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): + """ + Draws a gaussian heatmap around a center point given a radius. + Params: + heatmap (tf.Tensor): heatmap placeholder to fill + blobs (tf.Tensor): a tensor whose last dimension is 4 integers for + the category of the object, center (x, y), and for radius of the + gaussian + scaling_factor (int): scaling factor for gaussian + """ + blobs = tf.cast(blobs, tf.int32) + category = blobs[..., 0] + x = blobs[..., 1] + y = blobs[..., 2] + radius = blobs[..., 3] + + diameter = 2 * radius + 1 + + num_boxes = tf.shape(radius)[0] + + heatmap_shape = tf.shape(heatmap) + height, width = heatmap_shape[-2], heatmap_shape[-1] + + left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) + top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) + + update_count = tf.reduce_sum((bottom + top) * (right + left)) + masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) + heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((3,)), size=update_count) + i = 0 + for j in range(num_boxes): + cat = category[j] + X = x[j] + Y = y[j] + R = radius[j] + l = left[j] + r = right[j] + t = top[j] + b = bottom[j] + + gaussian = _gaussian_penalty(R, dtype=dtype) + masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) + heatmap_mask_instance = cartesian_product([cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) + masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) + heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) + masked_gaussian = masked_gaussian_ta.stack() + heatmap_mask = heatmap_mask_ta.stack() + heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) + heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, tf.cast(masked_gaussian * scaling_factor, heatmap.dtype)) + return heatmap diff --git a/centernet/train_vm.py b/centernet/train_vm.py index 2fc405bcb..f067caac5 100644 --- a/centernet/train_vm.py +++ b/centernet/train_vm.py @@ -14,23 +14,20 @@ # limitations under the License. # ============================================================================== """TensorFlow Model Garden Vision training driver.""" -from yolo.utils.run_utils import prep_gpu +import sys -from absl import app -from absl import flags import gin -import sys +import tensorflow as tf +from absl import app, flags -from official.core import train_utils # pylint: disable=unused-import from centernet.common import registry_imports # pylint: enable=unused-import from official.common import distribute_utils from official.common import flags as tfm_flags -from official.core import task_factory -from official.core import train_lib +from official.core import task_factory, train_lib, train_utils from official.modeling import performance -import tensorflow as tf +from yolo.utils.run_utils import prep_gpu FLAGS = flags.FLAGS """ From ee8c96d955b7cf33927d39b752cccc96ed772607 Mon Sep 17 00:00:00 2001 From: David Li Date: Sat, 17 Apr 2021 23:39:30 -0400 Subject: [PATCH 102/132] new hm builder ~8x faster on GPU, todo test on TPU --- centernet/dataloaders/centernet_input.py | 358 ++++++++++++----------- centernet/ops/preprocessing_ops.py | 117 ++++---- 2 files changed, 249 insertions(+), 226 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index d51e88a8c..38f588775 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -1,3 +1,5 @@ +import time + import tensorflow as tf from centernet.ops import preprocessing_ops @@ -67,175 +69,175 @@ def __init__(self, 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' ) - # def _build_heatmap_and_regressed_features(self, - # labels, - # output_size=[128, 128], - # input_size=[512, 512]): - # """ Generates the ground truth labels for centernet. + def _build_heatmap_and_regressed_features1(self, + labels, + output_size=[128, 128], + input_size=[512, 512]): + """ Generates the ground truth labels for centernet. - # Ground truth labels are generated by splatting gaussians on heatmaps for - # corners and centers. Regressed features (offsets and sizes) are also - # generated. - - # Args: - # labels: A dictionary of COCO ground truth labels with at minimum the following fields: - # bbox: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension - # corresponds to the top left x, top left y, bottom right x, and - # bottom left y coordinates of the bounding box - # classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each - # box, given in the same order as the boxes - # num_detections: A `Tensor` or int that gives the number of objects in the image - # output_size: A `list` of length 2 containing the desired output height - # and width of the heatmaps - # input_size: A `list` of length 2 the expected input height and width of - # the image - # Returns: - # Dictionary of labels with the following fields: - # 'tl_heatmaps': A `Tensor` of shape [output_h, output_w, num_classes], - # heatmap with splatted gaussians centered at the positions and channels - # corresponding to the top left location and class of the object - # 'br_heatmaps': `Tensor` of shape [output_h, output_w, num_classes], - # heatmap with splatted gaussians centered at the positions and channels - # corresponding to the bottom right location and class of the object - # 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], - # heatmap with splatted gaussians centered at the positions and channels - # corresponding to the center location and class of the object - # 'tl_offset': `Tensor` of shape [max_num_instances, 2], where the first - # num_boxes entries contain the x-offset and y-offset of the top-left - # corner of an object. All other entires are 0 - # 'br_offset': `Tensor` of shape [max_num_instances, 2], where the first - # num_boxes entries contain the x-offset and y-offset of the - # bottom-right corner of an object. All other entires are 0 - # 'ct_offset': `Tensor` of shape [max_num_instances, 2], where the first - # num_boxes entries contain the x-offset and y-offset of the center of - # an object. All other entires are 0 - # 'size': `Tensor` of shape [max_num_instances, 2], where the first - # num_boxes entries contain the width and height of an object. All - # other entires are 0 - # 'box_mask': `Tensor` of shape [max_num_instances], where the first - # num_boxes entries are 1. All other entires are 0 - # 'box_indices': `Tensor` of shape [max_num_instances, 2], where the first - # num_boxes entries contain the y-center and x-center of a valid box. - # These are used to extract the regressed box features from the - # prediction when computing the loss - # """ - - # # boxes and classes are cast to self._dtype already from build_label - # boxes = labels['bbox'] - # classes = labels['classes'] - # input_size = tf.cast(input_size, self._dtype) - # output_size = tf.cast(output_size, self._dtype) - # input_h, input_w = input_size[0], input_size[1] - # output_h, output_w = output_size[0], output_size[1] + Ground truth labels are generated by splatting gaussians on heatmaps for + corners and centers. Regressed features (offsets and sizes) are also + generated. + + Args: + labels: A dictionary of COCO ground truth labels with at minimum the following fields: + bbox: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension + corresponds to the top left x, top left y, bottom right x, and + bottom left y coordinates of the bounding box + classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each + box, given in the same order as the boxes + num_detections: A `Tensor` or int that gives the number of objects in the image + output_size: A `list` of length 2 containing the desired output height + and width of the heatmaps + input_size: A `list` of length 2 the expected input height and width of + the image + Returns: + Dictionary of labels with the following fields: + 'tl_heatmaps': A `Tensor` of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the top left location and class of the object + 'br_heatmaps': `Tensor` of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the bottom right location and class of the object + 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the center location and class of the object + 'tl_offset': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the top-left + corner of an object. All other entires are 0 + 'br_offset': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the + bottom-right corner of an object. All other entires are 0 + 'ct_offset': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the center of + an object. All other entires are 0 + 'size': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the width and height of an object. All + other entires are 0 + 'box_mask': `Tensor` of shape [max_num_instances], where the first + num_boxes entries are 1. All other entires are 0 + 'box_indices': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the y-center and x-center of a valid box. + These are used to extract the regressed box features from the + prediction when computing the loss + """ + + # boxes and classes are cast to self._dtype already from build_label + boxes = labels['bbox'] + classes = labels['classes'] + input_size = tf.cast(input_size, self._dtype) + output_size = tf.cast(output_size, self._dtype) + input_h, input_w = input_size[0], input_size[1] + output_h, output_w = output_size[0], output_size[1] - # # We will transpose the heatmaps at the end - # tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - # br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - # ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + # We will transpose the heatmaps at the end + tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) + ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - # # Maps for offset and size predictions - # tl_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - # br_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - # ct_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - # size = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + # Maps for offset and size predictions + tl_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + br_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + ct_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) + size = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - # # Masks for valid boxes - # box_mask = tf.zeros((self._max_num_instances), dtype=tf.int32) - # box_indices = tf.zeros((self._max_num_instances, 2), dtype=tf.int32) - - # # Scaling factor for determining center/corners - # width_ratio = output_w / input_w - # height_ratio = output_h / input_h - - # num_objects = labels['num_detections'] - - # height = tf.cast(0.0, self._dtype) - # width = tf.cast(0.0, self._dtype) - # for tag_ind in tf.range(num_objects): - # box = boxes[tag_ind] - # obj_class = classes[tag_ind] - 1 # TODO: See if subtracting 1 from the class like the paper is unnecessary - - # ytl, xtl, ybr, xbr = box[0], box[1], box[2], box[3] - - # xct, yct = ( - # (xtl + xbr) / 2, - # (ytl + ybr) / 2 - # ) - - # # Scale center and corner locations - # # These should be dtype=float32 - # fxtl = (xtl * width_ratio) - # fytl = (ytl * height_ratio) - # fxbr = (xbr * width_ratio) - # fybr = (ybr * height_ratio) - # fxct = (xct * width_ratio) - # fyct = (yct * height_ratio) - - # # Fit center and corners onto the output image - # # These should be dtype=float32 - # xtl = tf.math.floor(fxtl) - # ytl = tf.math.floor(fytl) - # xbr = tf.math.floor(fxbr) - # ybr = tf.math.floor(fybr) - # xct = tf.math.floor(fxct) - # yct = tf.math.floor(fyct) + # Masks for valid boxes + box_mask = tf.zeros((self._max_num_instances), dtype=tf.int32) + box_indices = tf.zeros((self._max_num_instances, 2), dtype=tf.int32) + + # Scaling factor for determining center/corners + width_ratio = output_w / input_w + height_ratio = output_h / input_h + + num_objects = labels['num_detections'] + + height = tf.cast(0.0, self._dtype) + width = tf.cast(0.0, self._dtype) + for tag_ind in tf.range(num_objects): + box = boxes[tag_ind] + obj_class = classes[tag_ind] - 1 # TODO: See if subtracting 1 from the class like the paper is unnecessary + + ytl, xtl, ybr, xbr = box[0], box[1], box[2], box[3] + + xct, yct = ( + (xtl + xbr) / 2, + (ytl + ybr) / 2 + ) + + # Scale center and corner locations + # These should be dtype=float32 + fxtl = (xtl * width_ratio) + fytl = (ytl * height_ratio) + fxbr = (xbr * width_ratio) + fybr = (ybr * height_ratio) + fxct = (xct * width_ratio) + fyct = (yct * height_ratio) + + # Fit center and corners onto the output image + # These should be dtype=float32 + xtl = tf.math.floor(fxtl) + ytl = tf.math.floor(fytl) + xbr = tf.math.floor(fxbr) + ybr = tf.math.floor(fybr) + xct = tf.math.floor(fxct) + yct = tf.math.floor(fyct) - # # Splat gaussian at for the center/corner heatmaps - # if self._use_gaussian_bump: - # # Check: do we need to normalize these boxes? - # width = box[3] - box[1] - # height = box[2] - box[0] - - # width = tf.math.ceil(width * width_ratio) - # height = tf.math.ceil(height * height_ratio) - - # if self._gaussian_rad == -1: - # radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - # radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) - # else: - # radius = self._gaussian_rad + # Splat gaussian at for the center/corner heatmaps + if self._use_gaussian_bump: + # Check: do we need to normalize these boxes? + width = box[3] - box[1] + height = box[2] - box[0] + + width = tf.math.ceil(width * width_ratio) + height = tf.math.ceil(height * height_ratio) + + if self._gaussian_rad == -1: + radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) + radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) + else: + radius = self._gaussian_rad - # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[obj_class, xtl, ytl, radius]]) - # br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[obj_class, xbr, ybr, radius]]) - # ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[obj_class, xct, yct, radius]], scaling_factor=5) - - # else: - # tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, - # [[tf.cast(obj_class, tf.int32), tf.cast(ytl, tf.int32), tf.cast(xtl, tf.int32)]], [1]) - # br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, - # [[tf.cast(obj_class, tf.int32), tf.cast(ybr, tf.int32), tf.cast(xbr, tf.int32)]], [1]) - # ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, - # [[tf.cast(obj_class, tf.int32), tf.cast(yct, tf.int32), tf.cast(xct, tf.int32)]], [1]) + tl_heatmaps = preprocessing_ops.draw_gaussian1(tl_heatmaps, [[obj_class, xtl, ytl, radius]]) + br_heatmaps = preprocessing_ops.draw_gaussian1(br_heatmaps, [[obj_class, xbr, ybr, radius]]) + ct_heatmaps = preprocessing_ops.draw_gaussian1(ct_heatmaps, [[obj_class, xct, yct, radius]], scaling_factor=5) + + else: + tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, + [[tf.cast(obj_class, tf.int32), tf.cast(ytl, tf.int32), tf.cast(xtl, tf.int32)]], [1]) + br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, + [[tf.cast(obj_class, tf.int32), tf.cast(ybr, tf.int32), tf.cast(xbr, tf.int32)]], [1]) + ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, + [[tf.cast(obj_class, tf.int32), tf.cast(yct, tf.int32), tf.cast(xct, tf.int32)]], [1]) - # # Add box offset and size to the ground truth - # tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - # br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - # ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - # size = tf.tensor_scatter_nd_update(size, [[tag_ind, 0], [tag_ind, 1]], [width, height]) - - # # Initialy the mask is zeros, but each valid box needs to be unmasked - # box_mask = tf.tensor_scatter_nd_update(box_mask, [[tag_ind]], [1]) - - # # Contains the y and x coordinate of the box center in the heatmap - # box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1]], [yct, xct]) - - # # Make heatmaps of shape [height, width, num_classes] - # tl_heatmaps = tf.transpose(tl_heatmaps, perm=[1, 2, 0]) - # br_heatmaps = tf.transpose(br_heatmaps, perm=[1, 2, 0]) - # ct_heatmaps = tf.transpose(ct_heatmaps, perm=[1, 2, 0]) - - # labels = { - # 'tl_heatmaps': tl_heatmaps, - # 'br_heatmaps': br_heatmaps, - # 'ct_heatmaps': ct_heatmaps, - # 'tl_offset': tl_offset, - # 'br_offset': br_offset, - # 'ct_offset': ct_offset, - # 'size': size, - # 'box_mask': box_mask, - # 'box_indices': box_indices - # } - # return labels + # Add box offset and size to the ground truth + tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) + br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) + ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) + size = tf.tensor_scatter_nd_update(size, [[tag_ind, 0], [tag_ind, 1]], [width, height]) + + # Initialy the mask is zeros, but each valid box needs to be unmasked + box_mask = tf.tensor_scatter_nd_update(box_mask, [[tag_ind]], [1]) + + # Contains the y and x coordinate of the box center in the heatmap + box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1]], [yct, xct]) + + # Make heatmaps of shape [height, width, num_classes] + tl_heatmaps = tf.transpose(tl_heatmaps, perm=[1, 2, 0]) + br_heatmaps = tf.transpose(br_heatmaps, perm=[1, 2, 0]) + ct_heatmaps = tf.transpose(ct_heatmaps, perm=[1, 2, 0]) + + labels = { + 'tl_heatmaps': tl_heatmaps, + 'br_heatmaps': br_heatmaps, + 'ct_heatmaps': ct_heatmaps, + 'tl_offset': tl_offset, + 'br_offset': br_offset, + 'ct_offset': ct_offset, + 'size': size, + 'box_mask': box_mask, + 'box_indices': box_indices + } + return labels def _build_heatmap_and_regressed_features(self, labels, @@ -538,7 +540,24 @@ def postprocess_fn(self, is_training): classes = (1, 1, 1) - labels = CenterNetParser(use_gaussian_bump=True)._build_heatmap_and_regressed_features( + parser = CenterNetParser() + + print("testing original build heatmaps function: ") + a = time.time() + labels1 = parser._build_heatmap_and_regressed_features1( + labels = { + 'bbox': tf.constant(boxes, dtype=tf.float32), + 'num_detections': len(boxes), + 'classes': tf.constant(classes, dtype=tf.float32) + }, + output_size=[512, 512], input_size=[512, 512] + ) + b = time.time() + print("Time taken: {} ms", (b-a) * 1000) + + print("testing new build heatmaps function: ") + a = time.time() + labels1 = parser._build_heatmap_and_regressed_features( labels = { 'bbox': tf.constant(boxes, dtype=tf.float32), 'num_detections': len(boxes), @@ -546,9 +565,12 @@ def postprocess_fn(self, is_training): }, output_size=[512, 512], input_size=[512, 512] ) - tl_heatmaps = labels['tl_heatmaps'] - br_heatmaps = labels['br_heatmaps'] - ct_heatmaps = labels['ct_heatmaps'] + b = time.time() + print("Time taken: {} ms", (b-a) * 1000) + + # tl_heatmaps = labels['tl_heatmaps'] + # br_heatmaps = labels['br_heatmaps'] + # ct_heatmaps = labels['ct_heatmaps'] - plt.imshow(ct_heatmaps[..., 0]) - plt.show() + # plt.imshow(ct_heatmaps[..., 0]) + # plt.show() diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 54e11001e..b92cbc9f5 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -124,64 +124,65 @@ def write_all(ta, index, values): return ta, index + i # scaling_factor doesn't do anything right now -# def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): -# """ -# Draws a gaussian heatmap around a center point given a radius. -# Params: -# heatmap (tf.Tensor): heatmap placeholder to fill -# blobs (tf.Tensor): a tensor whose last dimension is 4 integers for -# the category of the object, center (x, y), and for radius of the -# gaussian -# scaling_factor (int): scaling factor for gaussian -# """ -# blobs = tf.cast(blobs, tf.int32) -# category = blobs[..., 0] -# x = blobs[..., 1] -# y = blobs[..., 2] -# radius = blobs[..., 3] -# num_boxes = tf.shape(radius)[0] - -# diameter = 2 * radius + 1 - -# heatmap_shape = tf.shape(heatmap) -# height, width = heatmap_shape[-2], heatmap_shape[-1] - -# left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) -# top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) - -# # print('heatmap ',heatmap) -# # print(len(heatmap)) -# # print('category ',category) - -# # TODO: make sure this replicates original functionality -# # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] -# # masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] -# # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) -# update_count = tf.reduce_sum((bottom + top) * (right + left)) -# masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) -# heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((3,)), size=update_count) -# i = 0 -# for j in range(num_boxes): -# cat = category[j] -# X = x[j] -# Y = y[j] -# R = radius[j] -# l = left[j] -# r = right[j] -# t = top[j] -# b = bottom[j] - -# gaussian = _gaussian_penalty(R, dtype=dtype) -# masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) -# heatmap_mask_instance = cartesian_product([cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) -# masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) -# heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) -# masked_gaussian = masked_gaussian_ta.stack() -# heatmap_mask = heatmap_mask_ta.stack() -# heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) -# heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, tf.cast(masked_gaussian * scaling_factor, heatmap.dtype)) -# # print('after ',heatmap) -# return heatmap +@tf.function +def draw_gaussian1(heatmap, blobs, scaling_factor=1, dtype=tf.float32): + """ + Draws a gaussian heatmap around a center point given a radius. + Params: + heatmap (tf.Tensor): heatmap placeholder to fill + blobs (tf.Tensor): a tensor whose last dimension is 4 integers for + the category of the object, center (x, y), and for radius of the + gaussian + scaling_factor (int): scaling factor for gaussian + """ + blobs = tf.cast(blobs, tf.int32) + category = blobs[..., 0] + x = blobs[..., 1] + y = blobs[..., 2] + radius = blobs[..., 3] + num_boxes = tf.shape(radius)[0] + + diameter = 2 * radius + 1 + + heatmap_shape = tf.shape(heatmap) + height, width = heatmap_shape[-2], heatmap_shape[-1] + + left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) + top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) + + # print('heatmap ',heatmap) + # print(len(heatmap)) + # print('category ',category) + + # TODO: make sure this replicates original functionality + # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] + # masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] + # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) + update_count = tf.reduce_sum((bottom + top) * (right + left)) + masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) + heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((3,)), size=update_count) + i = 0 + for j in range(num_boxes): + cat = category[j] + X = x[j] + Y = y[j] + R = radius[j] + l = left[j] + r = right[j] + t = top[j] + b = bottom[j] + + gaussian = _gaussian_penalty(R, dtype=dtype) + masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) + heatmap_mask_instance = cartesian_product([cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) + masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) + heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) + masked_gaussian = masked_gaussian_ta.stack() + heatmap_mask = heatmap_mask_ta.stack() + heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) + heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, tf.cast(masked_gaussian * scaling_factor, heatmap.dtype)) + # print('after ',heatmap) + return heatmap @tf.function def draw_gaussian(heatmap, blob, scaling_factor=1): From 9135403399ae4018fbfd9c588355e177e8354a2f Mon Sep 17 00:00:00 2001 From: David Li Date: Sun, 18 Apr 2021 00:25:56 -0400 Subject: [PATCH 103/132] added documentation, also is now 16x faster --- centernet/dataloaders/centernet_input.py | 104 ++++++++++++++--------- centernet/ops/preprocessing_ops.py | 38 ++++++--- 2 files changed, 94 insertions(+), 48 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 38f588775..7e9ee0ba2 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -292,31 +292,17 @@ def _build_heatmap_and_regressed_features(self, prediction when computing the loss """ + # Get relevant bounding box and class information from labels boxes = labels['bbox'] classes = labels['classes'] - 1 num_objects = labels['num_detections'] + # Compute scaling factors for center/corner positions on heatmap input_size = tf.cast(input_size, self._dtype) output_size = tf.cast(output_size, self._dtype) input_h, input_w = input_size[0], input_size[1] output_h, output_w = output_size[0], output_size[1] - - # Heatmaps which we will transpose at the end - tl_heatmap = tf.zeros((output_h, output_w, self._num_classes), dtype=self._dtype) - br_heatmap = tf.zeros((output_h, output_w, self._num_classes), dtype=self._dtype) - ct_heatmap = tf.zeros((output_h, output_w, self._num_classes), dtype=self._dtype) - - # Maps for offset and size predictions - tl_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - br_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - ct_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - size = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - - # Masks for valid boxes - box_mask = tf.zeros((self._max_num_instances), dtype=tf.int32) - box_indices = tf.zeros((self._max_num_instances, 2), dtype=tf.int32) - # Scaling factor for determining center/corners width_ratio = output_w / input_w height_ratio = output_h / input_h @@ -326,7 +312,7 @@ def _build_heatmap_and_regressed_features(self, yct = (ytl + ybr) / 2 xct = (xtl + xbr) / 2 - # Scaled box coordinates + # Scaled box coordinates (could be decimal) fxtl = xtl * width_ratio fytl = ytl * height_ratio fxbr = xbr * width_ratio @@ -334,7 +320,7 @@ def _build_heatmap_and_regressed_features(self, fxct = xct * width_ratio fyct = yct * height_ratio - # Scaled box coordinate positions on heatmaps + # Floor the scaled box coordinates to be placed on heatmaps xtl = tf.math.floor(fxtl) ytl = tf.math.floor(fytl) xbr = tf.math.floor(fxbr) @@ -342,6 +328,7 @@ def _build_heatmap_and_regressed_features(self, xct = tf.math.floor(fxct) yct = tf.math.floor(fyct) + # Get the scaled box dimensions for computing the gaussian radius box_widths = boxes[..., 3] - boxes[..., 1] box_heights = boxes[..., 2] - boxes[..., 0] @@ -349,29 +336,62 @@ def _build_heatmap_and_regressed_features(self, box_heights = tf.math.ceil(box_heights * height_ratio) box_widths_heights = tf.stack([box_widths, box_heights], axis=-1) + # Center/corner heatmaps + tl_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) + br_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) + ct_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) + + # Maps for offset and size features for each instance of a box + tl_offset = tf.zeros((self._max_num_instances, 2), self._dtype) + br_offset = tf.zeros((self._max_num_instances, 2), self._dtype) + ct_offset = tf.zeros((self._max_num_instances, 2), self._dtype) + size = tf.zeros((self._max_num_instances, 2), self._dtype) + + # Mask for valid box instances and their center indices in the heatmap + box_mask = tf.zeros((self._max_num_instances), tf.int32) + box_indices = tf.zeros((self._max_num_instances, 2), tf.int32) + if self._use_gaussian_bump: + # Need to gaussians around the centers and corners of the objects + + # First compute the desired gaussian radius if self._gaussian_rad == -1: - radius = tf.map_fn(fn=lambda x: preprocessing_ops.gaussian_radius(x), elems=box_widths_heights) - radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) + radius = tf.map_fn(fn=lambda x: preprocessing_ops.gaussian_radius(x), + elems=box_widths_heights) + radius = tf.math.maximum(tf.math.floor(radius), + tf.cast(0.0, radius.dtype)) else: - radius = tf.constant([self._gaussian_rad] * num_objects, dtype=self._dtype) - + radius = tf.constant([self._gaussian_rad] * num_objects, self._dtype) + + # These blobs contain information needed to draw the gaussian tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) ct_blobs = tf.stack([classes, xct, yct, radius], axis=-1) - - tl_gaussians = tf.map_fn(fn = lambda x: preprocessing_ops.draw_gaussian(tl_heatmap, x), elems=tl_blobs) - br_gaussians = tf.map_fn(fn = lambda x: preprocessing_ops.draw_gaussian(br_heatmap, x), elems=br_blobs) - ct_gaussians = tf.map_fn(fn = lambda x: preprocessing_ops.draw_gaussian(ct_heatmap, x, 5), elems=ct_blobs) - + + # Get individual gaussian contributions from each bounding box + tl_gaussians = tf.map_fn( + fn=lambda x: preprocessing_ops.draw_gaussian( + tf.shape(tl_heatmap), x, self._dtype), elems=tl_blobs) + br_gaussians = tf.map_fn( + fn=lambda x: preprocessing_ops.draw_gaussian( + tf.shape(br_heatmap), x, self._dtype), elems=br_blobs) + ct_gaussians = tf.map_fn( + fn=lambda x: preprocessing_ops.draw_gaussian( + tf.shape(ct_heatmap), x, self._dtype), elems=ct_blobs) + + # Combine contributions into single heatmaps tl_heatmap = tf.math.reduce_max(tl_gaussians, axis=0) br_heatmap = tf.math.reduce_max(br_gaussians, axis=0) ct_heatmap = tf.math.reduce_max(ct_gaussians, axis=0) - else: - tl_hm_update_indices = tf.cast(tf.stack([ytl, xtl, classes], axis=-1), tf.int32) - br_hm_update_indices = tf.cast(tf.stack([ybr, xbr, classes], axis=-1), tf.int32) - ct_hm_update_indices = tf.cast(tf.stack([yct, xct, classes], axis=-1), tf.int32) + else: + # Instead of a gaussian, insert 1s in the center and corner heatmaps + tl_hm_update_indices = tf.cast( + tf.stack([ytl, xtl, classes], axis=-1), tf.int32) + br_hm_update_indices = tf.cast( + tf.stack([ybr, xbr, classes], axis=-1), tf.int32) + ct_hm_update_indices = tf.cast( + tf.stack([yct, xct, classes], axis=-1), tf.int32) tl_heatmap = tf.tensor_scatter_nd_update(tl_heatmap, tl_hm_update_indices, [1] * num_objects) @@ -380,27 +400,35 @@ def _build_heatmap_and_regressed_features(self, ct_heatmap = tf.tensor_scatter_nd_update(ct_heatmap, ct_hm_update_indices, [1] * num_objects) - update_indices = preprocessing_ops.cartesian_product(tf.range(num_objects), tf.range(2)) + # Indicies used to update offsets and sizes for valid box instances + update_indices = preprocessing_ops.cartesian_product( + tf.range(num_objects), tf.range(2)) update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) - tl_offset = tf.tensor_scatter_nd_update(tl_offset, update_indices, tl_offset_values) - br_offset = tf.tensor_scatter_nd_update(br_offset, update_indices, br_offset_values) - ct_offset = tf.tensor_scatter_nd_update(ct_offset, update_indices, ct_offset_values) + # Write the offsets of each box instance + tl_offset = tf.tensor_scatter_nd_update( + tl_offset, update_indices, tl_offset_values) + br_offset = tf.tensor_scatter_nd_update( + br_offset, update_indices, br_offset_values) + ct_offset = tf.tensor_scatter_nd_update( + ct_offset, update_indices, ct_offset_values) + # Write the size of each bounding box size = tf.tensor_scatter_nd_update(size, update_indices, box_widths_heights) - # Initialy the mask is zeros, but each valid box needs to be unmasked + # Initially the mask is zeros, so now we unmask each valid box instance mask_indices = tf.expand_dims(tf.range(num_objects), -1) mask_values = tf.repeat(1, num_objects) box_mask = tf.tensor_scatter_nd_update(box_mask, mask_indices, mask_values) - # Contains the y and x coordinate of the box center in the heatmap + # Write the y and x coordinate of each box center in the heatmap box_index_values = tf.cast(tf.stack([yct, xct], axis=-1), dtype=tf.int32) - box_indices = tf.tensor_scatter_nd_update(box_indices, update_indices, box_index_values) + box_indices = tf.tensor_scatter_nd_update( + box_indices, update_indices, box_index_values) labels = { 'tl_heatmaps': tl_heatmap, diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index b92cbc9f5..30ab24d30 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -185,24 +185,42 @@ def draw_gaussian1(heatmap, blobs, scaling_factor=1, dtype=tf.float32): return heatmap @tf.function -def draw_gaussian(heatmap, blob, scaling_factor=1): - # dummy heatmap to draw onto - heatmap_shape = tf.shape(heatmap) - gaussian_heatmap = tf.zeros(shape=heatmap_shape, dtype=heatmap.dtype) +def draw_gaussian(hm_shape, blob, dtype, scaling_factor=1): + """ Draws an instance of a 2D gaussian on a heatmap. + + A heatmap with shape hm_shape and of type dtype is generated with + a gaussian with a given center, radius, and scaling factor + + Args: + hm_shape: A `list` of `Tensor` of shape [3] that gives the height, width, + and number of channels in the heatmap + blob: A `Tensor` of shape [4] that gives the channel number, x, y, and + radius for the desired gaussian to be drawn onto + dtype: The desired type of the heatmap + scaling_factor: A `int` that can be used to scale the magnitude of the + gaussian + Returns: + A `Tensor` with shape hm_shape and type dtype with a 2D gaussian + """ + gaussian_heatmap = tf.zeros(shape=hm_shape, dtype=dtype) blob = tf.cast(blob, tf.int32) obj_class, x, y, radius = blob[0], blob[1], blob[2], blob[3] - height, width = heatmap_shape[0], heatmap_shape[1] + height, width = hm_shape[0], hm_shape[1] - left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) - top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) + left = tf.math.minimum(x, radius) + right = tf.math.minimum(width - x, radius + 1) + top = tf.math.minimum(y, radius) + bottom = tf.math.minimum(height - y, radius + 1) - gaussian = _gaussian_penalty(radius=radius, dtype=heatmap.dtype) + gaussian = _gaussian_penalty(radius=radius, dtype=dtype) gaussian = gaussian[radius-top:radius+bottom, radius-left:radius+right] gaussian = tf.reshape(gaussian, [-1]) - heatmap_indices = cartesian_product(tf.range(y - top, y + bottom), tf.range(x - left, x + right), [obj_class]) - gaussian_heatmap = tf.tensor_scatter_nd_update(gaussian_heatmap, heatmap_indices, gaussian * scaling_factor) + heatmap_indices = cartesian_product( + tf.range(y - top, y + bottom), tf.range(x - left, x + right), [obj_class]) + gaussian_heatmap = tf.tensor_scatter_nd_update( + gaussian_heatmap, heatmap_indices, gaussian * scaling_factor) return gaussian_heatmap From 529b792a66397282c7959478e8e57397dff33530 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 19 Apr 2021 12:31:11 -0400 Subject: [PATCH 104/132] testing on tpu --- centernet/dataloaders/centernet_input.py | 2 +- centernet/tasks/centernet_test.py | 40 ++++++++---------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 7e9ee0ba2..68a86d636 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -400,7 +400,7 @@ def _build_heatmap_and_regressed_features(self, ct_heatmap = tf.tensor_scatter_nd_update(ct_heatmap, ct_hm_update_indices, [1] * num_objects) - # Indicies used to update offsets and sizes for valid box instances + # Indices used to update offsets and sizes for valid box instances update_indices = preprocessing_ops.cartesian_product( tf.range(num_objects), tf.range(2)) update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index e53f9e6ac..25e42a99a 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -1,18 +1,9 @@ -import dataclasses - -import numpy as np import tensorflow as tf -import tensorflow_datasets as tfds from absl.testing import parameterized import orbit -from centernet.configs import centernet as exp_cfg -from centernet.dataloaders.centernet_input import CenterNetParser -from centernet.tasks.centernet import CenterNetTask -from centernet.utils.weight_utils.load_weights import ( - get_model_weights_as_dict, load_weights_model) -from official.modeling import hyperparams -from official.vision.beta.configs import backbones +from centernet.tasks import centernet +from official.core import exp_factory CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' @@ -33,27 +24,22 @@ class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): # model.summary() - def testCenterNetValidation(self): - model_config = exp_cfg.CenterNet(input_size=[512, 512, 3]) - config = exp_cfg.CenterNetTask(model=model_config) - - task = CenterNetTask(config) + @parameterized.parameters(("centernet_tpu",)) + def testCenterNetValidation(self, config_name): + config = exp_factory.get_exp_config(config_name) + + task = centernet.CenterNetTask(config.task) model = task.build_model() - task.initialize(model) metrics = task.build_metrics(training=False) strategy = tf.distribute.get_strategy() - - dataset = orbit.utils.make_distributed_dataset(strategy, task.build_inputs, - config.validation_data) - # tf.print("Dataset: ") - # tf.print(dataset) - # iterator = iter(dataset) - # tf.print("Getting logs: ") - # logs = task.validation_step(next(iterator), model, metrics=metrics) - # tf.print("logs: ") - # tf.print(logs) + dataset = orbit.utils.make_distributed_dataset(strategy, task.build_inputs, + config.task.train_data) + iterator = iter(dataset) + opt_factory = optimization.OptimizerFactory(config.trainer.optimizer_config) + logs = task.validation_step(next(iterator), model, metrics=metrics) + self.assertIn("loss", logs) if __name__ == '__main__': From e7f4f779050acf9cea359f567ed649924e64379a Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 19 Apr 2021 17:37:58 +0000 Subject: [PATCH 105/132] dataset iterator test --- centernet/dataloaders/centernet_input.py | 9 +++++---- centernet/ops/preprocessing_ops.py | 2 +- centernet/tasks/centernet_test.py | 6 ++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 68a86d636..58153132f 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -293,9 +293,9 @@ def _build_heatmap_and_regressed_features(self, """ # Get relevant bounding box and class information from labels - boxes = labels['bbox'] - classes = labels['classes'] - 1 num_objects = labels['num_detections'] + boxes = labels['bbox'][:num_objects] + classes = labels['classes'][:num_objects] - 1 # Compute scaling factors for center/corner positions on heatmap input_size = tf.cast(input_size, self._dtype) @@ -404,11 +404,12 @@ def _build_heatmap_and_regressed_features(self, update_indices = preprocessing_ops.cartesian_product( tf.range(num_objects), tf.range(2)) update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) - + + tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) - + # Write the offsets of each box instance tl_offset = tf.tensor_scatter_nd_update( tl_offset, update_indices, tl_offset_values) diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 30ab24d30..63205d234 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -33,7 +33,7 @@ def gaussian_radius(det_size, min_overlap=0.7) -> int: Returns: int representing desired gaussian radius """ - height, width = det_size + height, width = det_size[0], det_size[1] # Case where detected box is offset from ground truth and no box completely # contains the other. diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index 25e42a99a..2116e4725 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -5,8 +5,6 @@ from centernet.tasks import centernet from official.core import exp_factory -CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' - class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): # def testCenterNetTask(self): @@ -34,11 +32,11 @@ def testCenterNetValidation(self, config_name): strategy = tf.distribute.get_strategy() dataset = orbit.utils.make_distributed_dataset(strategy, task.build_inputs, - config.task.train_data) + config.task.validation_data) iterator = iter(dataset) - opt_factory = optimization.OptimizerFactory(config.trainer.optimizer_config) logs = task.validation_step(next(iterator), model, metrics=metrics) + print(logs) self.assertIn("loss", logs) From a5f016d5e30ce9aee6666807e1713e565c5a3067 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 19 Apr 2021 14:06:13 -0400 Subject: [PATCH 106/132] class offset, checking if parser outputs are valid --- centernet/dataloaders/centernet_input.py | 29 ++++++++++------- centernet/dataloaders/centernet_input_test.py | 31 ++++++++++++++++--- centernet/tasks/centernet_test.py | 1 + 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 58153132f..6b7607c80 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -33,6 +33,7 @@ def __init__(self, gaussian_rad: int = -1, gaussian_iou: float = 0.7, output_dims: int = 128, + class_offset: int = 1, dtype: str = 'float32'): """Initializes parameters for parsing annotations in the dataset. Args: @@ -48,6 +49,7 @@ def __init__(self, gaussian_iou: A `float` number for the minimum desired IOU used when determining the gaussian radius of center locations in the heatmap. output_dims: A `Tensor` or `int` for output dimensions of the heatmap. + class_offset: A `int` for subtracting a value from the ground truth classes """ self._image_w = image_w self._image_h = image_h @@ -57,6 +59,7 @@ def __init__(self, self._use_gaussian_bump = use_gaussian_bump self._gaussian_rad = -1 self._output_dims = output_dims + self._class_offset = class_offset if dtype == 'float16': self._dtype = tf.float16 @@ -293,9 +296,10 @@ def _build_heatmap_and_regressed_features(self, """ # Get relevant bounding box and class information from labels + # only keep the first num_objects boxes and classes num_objects = labels['num_detections'] boxes = labels['bbox'][:num_objects] - classes = labels['classes'][:num_objects] - 1 + classes = labels['classes'][:num_objects] - self._class_offset # Compute scaling factors for center/corner positions on heatmap input_size = tf.cast(input_size, self._dtype) @@ -561,13 +565,16 @@ def postprocess_fn(self, is_training): if __name__ == '__main__': # This code is for visualization import matplotlib.pyplot as plt - boxes = [ + boxes = tf.constant([ (10, 300, 15, 370), # center (y, x) = (12, 335) (100, 300, 150, 370), # center (y, x) = (125, 335) (15, 100, 200, 170), # center (y, x) = (107, 135) - ] + ], dtype=tf.float32) - classes = (1, 1, 1) + classes = tf.constant((1, 1, 1), dtype=tf.float32) + + boxes = pad_max_instances(boxes, 128, 0) + classes = pad_max_instances(classes, 128, -1) parser = CenterNetParser() @@ -575,9 +582,9 @@ def postprocess_fn(self, is_training): a = time.time() labels1 = parser._build_heatmap_and_regressed_features1( labels = { - 'bbox': tf.constant(boxes, dtype=tf.float32), - 'num_detections': len(boxes), - 'classes': tf.constant(classes, dtype=tf.float32) + 'bbox': boxes, + 'num_detections': 3, + 'classes': classes }, output_size=[512, 512], input_size=[512, 512] ) @@ -586,11 +593,11 @@ def postprocess_fn(self, is_training): print("testing new build heatmaps function: ") a = time.time() - labels1 = parser._build_heatmap_and_regressed_features( + labels = parser._build_heatmap_and_regressed_features( labels = { - 'bbox': tf.constant(boxes, dtype=tf.float32), - 'num_detections': len(boxes), - 'classes': tf.constant(classes, dtype=tf.float32) + 'bbox': boxes, + 'num_detections': 3, + 'classes': classes }, output_size=[512, 512], input_size=[512, 512] ) diff --git a/centernet/dataloaders/centernet_input_test.py b/centernet/dataloaders/centernet_input_test.py index 7876e9692..1a9ad66a0 100755 --- a/centernet/dataloaders/centernet_input_test.py +++ b/centernet/dataloaders/centernet_input_test.py @@ -4,14 +4,35 @@ from centernet.dataloaders.centernet_input import CenterNetParser +def pad_max_instances(value, instances, pad_value=0, pad_axis=0): + shape = tf.shape(value) + if pad_axis < 0: + pad_axis = tf.shape(shape)[0] + pad_axis + dim1 = shape[pad_axis] + take = tf.math.reduce_min([instances, dim1]) + value, _ = tf.split( + value, [take, -1], axis=pad_axis) # value[:instances, ...] + pad = tf.convert_to_tensor([tf.math.reduce_max([instances - dim1, 0])]) + nshape = tf.concat([shape[:pad_axis], pad, shape[(pad_axis + 1):]], axis=0) + pad_tensor = tf.fill(nshape, tf.cast(pad_value, dtype=value.dtype)) + value = tf.concat([value, pad_tensor], axis=pad_axis) + return value + class CenterNetInputTest(tf.test.TestCase, parameterized.TestCase): def check_labels_correct(self, boxes, classes, output_size, input_size): parser = CenterNetParser() + num_dets = len(boxes) + boxes = tf.constant(boxes, dtype=tf.float32) + classes = tf.constant(classes, dtype=tf.float32) + + boxes = pad_max_instances(boxes, 128, 0) + classes = pad_max_instances(classes, 128, -1) + labels = parser._build_heatmap_and_regressed_features( labels = { - 'bbox': tf.constant(boxes, dtype=tf.float32), - 'num_detections': len(boxes), - 'classes': tf.constant(classes, dtype=tf.float32) + 'bbox': boxes, + 'num_detections': num_dets, + 'classes': classes }, output_size=output_size, input_size=input_size) @@ -43,7 +64,9 @@ def check_labels_correct(self, boxes, classes, output_size, input_size): self.assertEqual(box_mask.shape, (parser._max_num_instances)) self.assertEqual(box_indices.shape, (parser._max_num_instances, 2)) - # Not checking heatmaps, but we can visually validate them + self.assertAllInRange(tl_heatmaps, 0, 1) + self.assertAllInRange(br_heatmaps, 0, 1) + self.assertAllInRange(ct_heatmaps, 0, 1) for i in range(len(boxes)): # Check sizes diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index 2116e4725..431257010 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -5,6 +5,7 @@ from centernet.tasks import centernet from official.core import exp_factory + class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): # def testCenterNetTask(self): From b92ba42c067afdecfed564307bf3244dde44bf15 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 19 Apr 2021 18:45:22 +0000 Subject: [PATCH 107/132] works on tpu --- centernet/dataloaders/centernet_input.py | 49 ++++++++++++++---------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 6b7607c80..5aed286e3 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -565,6 +565,13 @@ def postprocess_fn(self, is_training): if __name__ == '__main__': # This code is for visualization import matplotlib.pyplot as plt + + + resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='node-8') + tf.config.experimental_connect_to_cluster(resolver) + tf.tpu.experimental.initialize_tpu_system(resolver) + print("All devices: ", tf.config.list_logical_devices('TPU')) + boxes = tf.constant([ (10, 300, 15, 370), # center (y, x) = (12, 335) (100, 300, 150, 370), # center (y, x) = (125, 335) @@ -578,29 +585,31 @@ def postprocess_fn(self, is_training): parser = CenterNetParser() - print("testing original build heatmaps function: ") - a = time.time() - labels1 = parser._build_heatmap_and_regressed_features1( - labels = { - 'bbox': boxes, - 'num_detections': 3, - 'classes': classes - }, - output_size=[512, 512], input_size=[512, 512] - ) - b = time.time() - print("Time taken: {} ms", (b-a) * 1000) + # print("testing original build heatmaps function: ") + # a = time.time() + # with tf.device('/TPU:0'): + # labels1 = parser._build_heatmap_and_regressed_features1( + # labels = { + # 'bbox': boxes, + # 'num_detections': 3, + # 'classes': classes + # }, + # output_size=[512, 512], input_size=[512, 512] + # ) + # b = time.time() + # print("Time taken: {} ms", (b-a) * 1000) print("testing new build heatmaps function: ") a = time.time() - labels = parser._build_heatmap_and_regressed_features( - labels = { - 'bbox': boxes, - 'num_detections': 3, - 'classes': classes - }, - output_size=[512, 512], input_size=[512, 512] - ) + with tf.device('/TPU:0'): + labels = parser._build_heatmap_and_regressed_features( + labels = { + 'bbox': boxes, + 'num_detections': 3, + 'classes': classes + }, + output_size=[512, 512], input_size=[512, 512] + ) b = time.time() print("Time taken: {} ms", (b-a) * 1000) From 1ba96baf73a41a9bea887c4099eb64465c6c95cb Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 19 Apr 2021 16:10:18 -0400 Subject: [PATCH 108/132] fixed box widths --- centernet/dataloaders/centernet_input.py | 26 +++++++------- centernet/dataloaders/centernet_input_test.py | 36 ++++++++++++++++--- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 6b7607c80..26df952a3 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -316,7 +316,7 @@ def _build_heatmap_and_regressed_features(self, yct = (ytl + ybr) / 2 xct = (xtl + xbr) / 2 - # Scaled box coordinates (could be decimal) + # Scaled box coordinates (could be floating point) fxtl = xtl * width_ratio fytl = ytl * height_ratio fxbr = xbr * width_ratio @@ -331,15 +331,23 @@ def _build_heatmap_and_regressed_features(self, ybr = tf.math.floor(fybr) xct = tf.math.floor(fxct) yct = tf.math.floor(fyct) + + # Offset computations to make up for discretization error + # used for offset maps + tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) + br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) + ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) # Get the scaled box dimensions for computing the gaussian radius box_widths = boxes[..., 3] - boxes[..., 1] box_heights = boxes[..., 2] - boxes[..., 0] - box_widths = tf.math.ceil(box_widths * width_ratio) - box_heights = tf.math.ceil(box_heights * height_ratio) - box_widths_heights = tf.stack([box_widths, box_heights], axis=-1) + box_widths = box_widths * width_ratio + box_heights = box_heights * height_ratio + # Used for size map + box_widths_heights = tf.stack([box_widths, box_heights], axis=-1) + # Center/corner heatmaps tl_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) br_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) @@ -361,12 +369,11 @@ def _build_heatmap_and_regressed_features(self, # First compute the desired gaussian radius if self._gaussian_rad == -1: radius = tf.map_fn(fn=lambda x: preprocessing_ops.gaussian_radius(x), - elems=box_widths_heights) + elems=tf.math.ceil(box_widths_heights)) radius = tf.math.maximum(tf.math.floor(radius), - tf.cast(0.0, radius.dtype)) + tf.cast(1.0, radius.dtype)) else: radius = tf.constant([self._gaussian_rad] * num_objects, self._dtype) - # These blobs contain information needed to draw the gaussian tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) @@ -409,11 +416,6 @@ def _build_heatmap_and_regressed_features(self, tf.range(num_objects), tf.range(2)) update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) - - tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) - br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) - ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) - # Write the offsets of each box instance tl_offset = tf.tensor_scatter_nd_update( tl_offset, update_indices, tl_offset_values) diff --git a/centernet/dataloaders/centernet_input_test.py b/centernet/dataloaders/centernet_input_test.py index 1a9ad66a0..020be0272 100755 --- a/centernet/dataloaders/centernet_input_test.py +++ b/centernet/dataloaders/centernet_input_test.py @@ -52,9 +52,9 @@ def check_labels_correct(self, boxes, classes, output_size, input_size): width_ratio = output_size[1] / input_size[1] # Shape checks - self.assertEqual(tl_heatmaps.shape, (512, 512, 90)) - self.assertEqual(br_heatmaps.shape, (512, 512, 90)) - self.assertEqual(ct_heatmaps.shape, (512, 512, 90)) + self.assertEqual(tl_heatmaps.shape, (output_size[0], output_size[1], 90)) + self.assertEqual(br_heatmaps.shape, (output_size[0], output_size[1], 90)) + self.assertEqual(ct_heatmaps.shape, (output_size[0], output_size[1], 90)) self.assertEqual(tl_offset.shape, (parser._max_num_instances, 2)) self.assertEqual(br_offset.shape, (parser._max_num_instances, 2)) @@ -70,7 +70,9 @@ def check_labels_correct(self, boxes, classes, output_size, input_size): for i in range(len(boxes)): # Check sizes - self.assertAllEqual(size[i], [boxes[i][3] - boxes[i][1], boxes[i][2] - boxes[i][0]]) + self.assertAllEqual(size[i], + [(boxes[i][3] - boxes[i][1]) * width_ratio, + (boxes[i][2] - boxes[i][0]) * height_ratio]) # Check box indices y = tf.math.floor((boxes[i][0] + boxes[i][2]) / 2 * height_ratio) @@ -106,6 +108,32 @@ def test_generate_heatmap_no_scale(self): self.check_labels_correct(boxes=boxes, classes=classes, output_size=sizes, input_size=sizes) + + def test_generate_heatmap_scale_1(self): + boxes = [ + (10, 300, 15, 370), + (100, 300, 150, 370), + (15, 100, 200, 170), + ] + classes = (1, 2, 3) + output_size = [128, 128] + input_size = [512, 512] + + self.check_labels_correct(boxes=boxes, classes=classes, + output_size=output_size, input_size=input_size) + + def test_generate_heatmap_scale_2(self): + boxes = [ + (10, 300, 15, 370), + (100, 300, 150, 370), + (15, 100, 200, 170), + ] + classes = (1, 2, 3) + output_size = [128, 128] + input_size = [1024, 1024] + + self.check_labels_correct(boxes=boxes, classes=classes, + output_size=output_size, input_size=input_size) if __name__ == '__main__': tf.test.main() From 0021a4396ee89ab9d37b62e145bcbe62a1ce56be Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 20 Apr 2021 06:03:48 +0000 Subject: [PATCH 109/132] getting stuck in standard_runner loop --- .../experiments/centernet-eval-tpu.yaml | 6 ++--- centernet/modeling/CenterNet.py | 3 +-- .../modeling/layers/detection_generator.py | 1 - centernet/tasks/centernet.py | 20 +++++---------- centernet/tasks/centernet_test.py | 11 ++++++-- official/core/.base_trainer.py.swp | Bin 0 -> 28672 bytes official/core/base_trainer.py | 12 ++++++++- official/core/train_lib.py | 4 ++- orbit/controller.py | 3 ++- orbit/standard_runner.py | 24 ++++++++++++++++-- 10 files changed, 56 insertions(+), 28 deletions(-) create mode 100644 official/core/.base_trainer.py.swp diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index ca529a8b3..0cca2f172 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -1,7 +1,6 @@ runtime: distribution_strategy: 'tpu' mixed_precision_dtype: 'bfloat16' - tpu: 'node-8' task: model: base: @@ -22,7 +21,6 @@ task: center_thresh: 0.1 iou_thresh: 0.4 class_offset: 1 - dtype: bloat16 validation_data: input_path: 'gs://tensorflow2/coco_records/val/2017*' global_batch_size: 8 @@ -40,7 +38,7 @@ task: gaussian_iou: 0.7 output_dims: 128 dtype: bfloat16 - shuffle_buffer_size: 10000 + shuffle_buffer_size: 100 subtasks: detection: use_centers: true @@ -63,7 +61,7 @@ task: load_extremenet_weights: false trainer: train_steps: 10000 # 160 epochs at 64 batchsize -> 500500 * 64/16 - validation_steps: 625 # 5063 #625 + validation_steps: -1 # 5063 #625 steps_per_loop: 10000 validation_interval: 1 summary_interval: 10000 diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 702f29a9c..90a561a84 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -74,8 +74,7 @@ def build_centernet_filter(model_config): use_nms=model_config.filter.use_nms, center_thresh=model_config.filter.center_thresh, iou_thresh=model_config.filter.iou_thresh, - class_offset=model_config.filter.class_offset, - dtype=model_config.filter.dtype) + class_offset=model_config.filter.class_offset) def build_centernet_head(model_config): return None diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 138cc1a19..8faa6ccc6 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -24,7 +24,6 @@ def __init__(self, center_thresh=0.1, iou_thresh=0.4, class_offset=1, - dtype='float32', **kwargs): """ Args: diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 31dbf7b06..2deb6baa2 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -27,7 +27,6 @@ def __init__(self, params, logging_dir: str = None): self._metrics = [] def build_inputs(self, params, input_context=None): - print("\nIn build_inputs\n") """Build input dataset.""" decoder = self.get_decoder(params) model = self.task_config.model @@ -52,7 +51,6 @@ def build_inputs(self, params, input_context=None): return dataset def build_model(self): - print("\nIn build_model\n") """get an instance of CenterNet""" from centernet.modeling.CenterNet import build_centernet params = self.task_config.train_data @@ -69,7 +67,6 @@ def build_model(self): return model def initialize(self, model: tf.keras.Model): - print("\nIn initialize\n") """Initializes CenterNet model by loading pretrained weights """ if self.task_config.load_odapi_weights and self.task_config.load_extremenet_weights: raise ValueError('Only 1 of odapi or extremenet weights should be loaded') @@ -117,7 +114,6 @@ def get_decoder(self, params): def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_losses=None): - print("\nIn build_losses\n") total_loss = 0.0 loss = 0.0 @@ -190,7 +186,6 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo return total_loss, metric_dict def build_metrics(self, training=True): - print("\nIn build_metrics\n") metrics = [] metric_names = self._metric_names @@ -207,7 +202,6 @@ def build_metrics(self, training=True): return metrics def train_step(self, inputs, model, optimizer, metrics=None): - print("\nIn train_step\n") # get the data point image, label = inputs @@ -248,7 +242,6 @@ def train_step(self, inputs, model, optimizer, metrics=None): return logs def validation_step(self, inputs, model, metrics=None): - print("\nIn validation step\n") # get the data point image, label = inputs @@ -257,7 +250,7 @@ def validation_step(self, inputs, model, metrics=None): num_replicas = 1 else: num_replicas = scale_replicas - + y_pred = model(image, training=False) y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) loss, loss_metrics = self.build_losses( @@ -289,14 +282,14 @@ def validation_step(self, inputs, model, metrics=None): logs.update({self.coco_metric.name: (label, coco_model_outputs)}) - if metrics: - for m in metrics: - m.update_state(loss_metrics[m.name]) - logs.update({m.name: m.result()}) + # if metrics: + # for m in metrics: + # m.update_state(loss_metrics[m.name]) + # logs.update({m.name: m.result()}) return logs + def aggregate_logs(self, state=None, step_outputs=None): - print("\nIn aggregate_logs\n") if not state: self.coco_metric.reset_states() state = self.coco_metric @@ -305,5 +298,4 @@ def aggregate_logs(self, state=None, step_outputs=None): return state def reduce_aggregated_logs(self, aggregated_logs): - print("\nIn reduce_aggregate_logs\n") return self.coco_metric.result() diff --git a/centernet/tasks/centernet_test.py b/centernet/tasks/centernet_test.py index 431257010..d3450685c 100644 --- a/centernet/tasks/centernet_test.py +++ b/centernet/tasks/centernet_test.py @@ -25,6 +25,10 @@ class CenterNetTaskTest(parameterized.TestCase, tf.test.TestCase): @parameterized.parameters(("centernet_tpu",)) def testCenterNetValidation(self, config_name): + resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='node-8') + tf.config.experimental_connect_to_cluster(resolver) + tf.tpu.experimental.initialize_tpu_system(resolver) + config = exp_factory.get_exp_config(config_name) task = centernet.CenterNetTask(config.task) @@ -36,8 +40,11 @@ def testCenterNetValidation(self, config_name): config.task.validation_data) iterator = iter(dataset) - logs = task.validation_step(next(iterator), model, metrics=metrics) - print(logs) + + with strategy.scope(): + logs = task.validation_step(next(iterator), model, metrics=metrics) + logs = task.validation_step(next(iterator), model, metrics=metrics) + self.assertIn("loss", logs) diff --git a/official/core/.base_trainer.py.swp b/official/core/.base_trainer.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..cfe0ca10f9b6e741d8525664d991ebf5e8b8650b GIT binary patch literal 28672 zcmeI4e~cu>b;rjLLIOAhN0Bj(;<{ZC@4;U0-4P|u!V=bZ2U+0mqTK~TI6T`s(=$8V z%=FORb9)OhKkWQ+g#XAN0V3N95f>zj#IbD0&L5Cud3dA^{UtR-nw+yyd~II;rD};$|sKBxAwMO z7gd&CTdCB-&i*Xwrg73}$D5L2^e?W5z1HYg1F827RyY@(2;0>xO0!nCIk&0}T~9-Y z<9gIi=i)}ARcnRqxmuh=bL(LmRkI{)b)zKcZ4J~rd6;lu!hthzpr1ANyydl(tM>2P zD> zV8Ve32PPbtaA3lL2?r(|m~ddife8mD9GGz6Ti`%g*JJ<6$Hm^}{NLOEe{pf8^5@__ za5cCDTnzr{2P&0+01ty@a6b4LM*U;px4^yN9&io#+C`PhN5BWc?}Jm|A&`OuL|_j% z2fPNnaABqLo8VFK2>2Cn2l#n#F?i_$+6Vs#o&%485L^Y$122DnrSf6$09XN2U>Eq} z_f;y-gQvhdz%Aea*bnxBZ=7GL{4RJB{5HtIHQ;J62mTf3$Fty1!7qYMa68xq&IcdG z`SCDlgC7HzfM;=dJO~~D_k#`yz!z|Id>(unJPCdotbsk?0`L-!m_G*Z01tvqFbmED zGDgpMhe>Iyjs7|kHZgBjHzK3|HLY%%h22^-YoeeTn9yX6;CPgTX`m4n`TL+^E6Bq1 z_`F#(Hq?TLV3mKiWvj*AMypB2u30;DvKJ+-PSni~XzE&?+oZjy)@p1K+F_zRm$8Ly zi;GD!ov&!)Q&Uq*-Bw1lN9&%5Or1{LH7?r2G~&b*(IK{TsVCZ!&cC&dsCK*;x4K!C z_TnUql7%C2H`+a~E#&!(k+73~m__quJ&xOE!K^0zXm&W4wto67aW89iAUK&9*5*%X zaMZupR;wGN!I9XO%Z|k0IBd)0$6#o zP=+|2ZiGoxAH#4;S8DArP2rb@;mpLzdMgW75VCrh)LrVV8B9K8xg94A#C6S9H?VZ~ zlW@HqA;iZ0rY>)oFf(Zu^~~kqHMp_T5Dt%TX;p zfkYy7(mgv58cEzKRW@%fPXnDMQlHWK%sK@-QEH~Nu6Epn(q^z3CSCr{XktmggdF5X z7#oz5J7G6$5}uS0(nQQsHBNL0tW?ifGN;YWaagz9?|>rH-^xb0j=s!Up;53=(Fv`|WzwwKg+u*limkmNdmrM*OaL-Pq!$#!9#0Sy}GWyxH7{y2e$%l{2Kn zm{Cc{$|JH9o;2NlXB{@i4LB*ivU6}*xa3zZQ1HQf`BsY;pc`9U->>`_?6qmYXq3DEL z>ru{a8G-Rr&8^po4xG6jo7VXnK?nbDsq&1iG2 zn_Pa3s_PLGQfE7?>DFBf!);OxD{F_@8MM-BH|&;VM_ZEa4oTOJ`m17Da1lY*na&!p zJyB`X0>hmp`*2c`IqzyQiO4@9bwrUecS>bS$=eF~wS+eI>Pu7nMl?Fl#-5S)s_olp zp1ahX&%Ss2u2pQ)j)}T?s^A6(@|TA#gSQnQ=HI-XCa2x==$_pOm=sAA)@4S>JS~RK zpl#G!C_H3U8tiP6#{B-ke46+h7)L#O-=ume{m_c6yVq#KKwcqpkwj{ORjsmx_F57i}i&<82+bNSBZ41uZZL^95%eKz+O@o0fm0>+# zbeZ8?X#BE0`e55YIVBuwbwr6WPOH%cV<@Y{<@#*7uEMsh6qgCD__{=2a1Dm*Y1KBk5hQ?_;5y}4Cm6Gw8U%}OK` z=1Ho)$+OkKTQ$>$Q++fO^bC|E(P#_{aHo}~B1THgw7G3*^=FRWvTANyTwY#0vbuC= z#T;EW2aX;&xU{--^axMao5dryo12!79K@n%(Ly4cJ*-Vp`IPcjR2PDnN^Cys{R~}l z8^s1zmNPI~?6^~lD`shB%Di=PWoc!WI&SHl(}0>GT6CMUCXE{znp;$vrXKliWLa;1Zf{ok<{Xn&IGE1eyma8uk(EPx z_6K{l$n9}o_Cyn}XXv@?#I?A`6MIXE$zew2TP~+)E`5;$k&{LXnT$@>qFyFsp;xj# zh#h7&VOqHo;guW5GNTriM*n8k+!`e^PzL(UjL5@W9J_S zw}YPp7lLQ7@$Uye4!()~{tfU+@P2R%90Akd6>RtC!8^egxDxyu_WCLC5V#S{fdE_t zK84NxC|CtQ4Ze(x{w44Spbqwe3&0E5=1+iCa1a~-7lJ3~i}dI1(x=I9!hs0~CLEY> z;QyWjvKc#@O${Vb$AYlQF5bi+S49s@XYg5OXwqUTSL1%x>u0I;cX+2vtJ{cYre>_h znO_Jqd(^Lo8C&nyqTL3~DiL)+)xJX~RW&H*ko~-@yG=^?V~}o!z3OJ9JG%2OZO?@}&N?HlLW%m1 zPZa-7*ghPefIo|9gs6sv9X{BIvagkTSF&{nO)HZODwZgK;%ddQgtHmjN@!;^`|U6> zjx_7Sm_{u@Ir7dY>z`W38BsRFY;7oTXi4}2Lra^lNK`C!PAY@Zd+C45Al8oOIh@^p0{lr9}K3 z&Ge2b%{OJPbcv;WzItmK1#0Z#C$tskHg8o$&lYoaK&Br#gYU45k&IVYU+$M!eq_v*}x8B4F5qn@j&@3GEsJDaNcM6cVK zKJ;t}$AvSL?)~?JGint*SSrVssH^8NiPT=jo!f#g(U--14|HA&8!@EBoZrNx6m{J$ zLn2RN{)$#DwY@ntYefy}Z(0*U*@zJysYU8pQ`VNUNyVBCdCD8G?5WR+YdN&WK_t!u zM_ong;+ktz>Rx)ssG5TT2`y@XiS!TitKZJt^oMyPYf!(j7Z9@xm(M_}b_nx!+~Yva zat345#iki<#J+J(UlC7+|K7K27Y9de+^VhX;8NEvzT=lbEasn0VoY?*HqFa1j*PS= zmBChDaK4IDkz2xtL~3B=mVp@iD_N?8yOG<*a5r{Y9`HH#kd~ycz6NbH;F0;!+zOa! z?vRWFi*{V&Uq`TYZ|#gldeSktS4fhR@3X1n2E<0CyjjUOCCHDVsV84GBi2W~r7!8cv`mwjPd#pK@!2PPbtaA3lL2?r(|m~ddife8mD z9GGxm!h!EL2gFd-&?jrP&sd|?`Ft@zHQ4d=eswXfcN_1bk*6_kYKdwfAVGcW8ZJ8Q zl~FTY4CPs(5uL|C0$zm3ozbF!M!DZyy{4KphN4%KnV9y|x0 z2B$y|tb!GA1GoZwl~{sLffvCiz?0x!Ah8B_fTLgz{3r1RFM>~kKLYOq?*%L1Jn(14 z7~Bn7;2<~veiFP1e1&*}zXg8-o&f7$9=rj3mY9S0gBV;37QilWKKMi84&DoX30w!x z1z#in;7`D>gFd((yh5zSm%&HCDeyRW5Hvvp90pf_OTmA0U*PY+hrmg&0+zuW!DZka z@Hxg!#_kaycL|Py-Qe@w7kF0g41mYMyTCly1$O+CDJj2K``TftROj2P;&|-ap5@Y^ zvsw%N!hE`QP*-Mxe~3SHKFHyvSxc+;n!TYSL?}OLIW}kO{ zY9{+x&$D#S3ULD-Ry|a!4mnpPS1u9O6PrJlV@uU#TlU^oq@?{h4{yajZFAS!c6C$J zEF96Gimdq<%vhYeHtL1x~|U?79ivF+69EGx0c8>OMm zPwqI{XiI>j%`t|CJc4-qrshsmT?T*tRpc*Dyhdz)1A>t6d^T16`__H$2GSi<35!qj z+a}B-4_kdCG^ats5%u=Wcksy0AIMoj^rsCz5T77mA!wSB*E=o5Kh}u2mFVU6mOoQ> z!OE1C6HFWR3TYfS)ii8@yCs}`bJW40=uti=(8Ndh+duiyvo>{R`IKuURe!4Hmk6P9 z;QEz|>s~0%tbM$^pY~F~cdK-1WT6)(VMoq1ZI4T@%9AyJt;>}j(x%&rmfn@iiU3qg z_YZuz;dEcdE>ONjj>485ZTV^Wc$(!FLw?XIUCw{oIdi%oc%3k^tgv+E(xFpq*4Wm$ z^=AWN4ruK4tX;d^l3}}0G+7Zc80*EFJ4n^TZz(UwYel}&=1?U1&B2)Kh==A}Gviat zQlYk@r%W22&Y9aiZW&Oiv?EWenwg+sosS&97gZC|K){C&gbg{^hwU6E5_$`T67}qF z=Ez0IDzEJ<*lZEkDo6V3g!Hoj5$9o6CC*!ec=NDad8M%;jefVLr^aaCT(PN}P-$Ay zIV2J7a`~oAmhh)|CW@E#XiLgMu7;Is+^?%wYtvwtyd_`6l4s*mx^cPEhFpfpBe#`h zfn}_Wl=f?YcO({0PpY+=WmpEvNdoQ1^Z?AEzTntb$ zur!Y(LUV{5j%|COj-54IB3I`5wuWVZi#!Z0aUuilm!?9IPZ#r@5nyQ^+jidR)3#hx zva?1@8?+OaQp(sWa!o_-&MK$M>cYTi=KH&5V-MO0(=f{tt$Wr?Rr3r}`4W~VjU`~9 z37rUzdj0>-ipe_UO38nBR>%ftxnlNVX9c}_?27uwv4zd(-xSwh-L(m`rYmVl)(B|| z*NR{!R%@;sEc*QRs-|8ARViV$bQdS=MJC%DuH5Rn$CF%akc%HvYX4t`LHLBb|L?!= z_gC2W?*w;&H-dA(N3qu*2k!wlfgb^1z%CcN{R7}p@Bml?v*2~$67VcG`90t!a3iGP4If~_wf8PKzRQ9kr*bJ>~emR z*)a`7P?&2&q9PNnGFr?ub-S$D)#W=TX42|uJ5aQL*0g^|=EYcoVYcx6?5MQb*>|5% z+pb-HRq3lpztGlIPn(lumbjtz-i zMJqPPihss%_qDPChOys#+admur7}ar^W^kE~{m>gEQwyV&{T-h!Wi zu3B|N59G*0aV=+z#2`x@UM(C~gMaUB5GgMECU^A~#(Z1hsC{jX65E{8BO{8-xLlJc z>i&65Z9)F&h>ERC{#6Cv_n9#bhQ_~a+i70E54uogk%M5J+pu*$il8xbuJSUW7+993 ze%5NI*wM!$6n$#4w+XARKe_QGW%W}Y!>gULy4SC!{Z1!Lw!A8$8{jbCQGVmiDmM*$ z?z0^@EApk0_5AhRJEQRAH5Bqyu4DA*JAocC_O`mS8hlsT_@O`JaBi27f($PlMG8`R zwu7yOOZ%EWV=FA^X9#xV>QT9~#*@|Gt*)=he!eq}yqC~Nww6TG6)PrE-iUnI;EZk9 z{<%taQhj;j7VV$Up479gUbH}9v3>yJs{NLC+xnok!b0&)AJ=GbIO4MhI<;&OOVVj` zPx(%&Ni&iJH&>0h;)=o)u5y0Y$s_-F;%U{f8R=&&*(UaVDZx$!+nu6y*b9@C^wNg| z{KysgES2PTuD-H8Ga{EZV`XUgtukTA&aoHfQ3{4L`%Tx9UFr>#l-IZ#pCsV!Qz=9K zr5OE830s$SO2l2h_OY*UXWKoQi`#6I`oR=b!xP-`EaK)SM3e9zGyZrX5n9@R14!&ilTWnLU_ki4pr^^-(C$B@@AH#PFy z^5qOMXWSPS!tPf7&ZG#7`yiM17#e>07aD9+13qWZZ&cc@b6APPP|YQ!W^K%^r#0oA zRF69?hBYySvP2|F*PUZZm^jR|Dx_%lisQfP$CGda% literal 0 HcmV?d00001 diff --git a/official/core/base_trainer.py b/official/core/base_trainer.py index 776d13497..054a497b9 100755 --- a/official/core/base_trainer.py +++ b/official/core/base_trainer.py @@ -299,29 +299,37 @@ def step_fn(inputs): def eval_begin(self): """Sets up metrics.""" + logging.info("(base_trainer.py - eval_begin) In eval_begin of base trainer") for metric in self.validation_metrics + [self.validation_loss]: metric.reset_states() # Swaps weights to test on weights moving average. if self.optimizer and isinstance( self.optimizer, optimization.ExponentialMovingAverage): self.optimizer.swap_weights() + logging.info("(base_trainer.py - eval_begin) Exitingeval_begin of base trainer") + def eval_step(self, iterator): """See base class.""" - + logging.info("(base_trainer.py - eval_step) In eval_step of base trainer") def step_fn(inputs): + logging.info("(base_trainer.py - eval_step - step_fn) Running task.validation_step()") logs = self.task.validation_step( inputs, model=self.model, metrics=self.validation_metrics) if self.task.loss in logs: self._validation_loss.update_state(logs[self.task.loss]) + logging.info("(base_trainer.py - eval_step - step_fn) Returning logs from task.validation_step()") return logs distributed_outputs = self.strategy.run(step_fn, args=(next(iterator),)) + logging.info("(base_trainer.py - eval_step) Got logs from step_fn") + logging.info("(base_trainer.py - eval_step) Exiting eval_step") return tf.nest.map_structure(self.strategy.experimental_local_results, distributed_outputs) def eval_end(self, aggregated_logs=None): """Processes evaluation results.""" + logging.info("(base_trainer.py - eval_end) In eval_end of base trainer") logs = {} for metric in self.validation_metrics: logs[metric.name] = metric.result() @@ -348,7 +356,9 @@ def eval_end(self, aggregated_logs=None): if self.optimizer and isinstance( self.optimizer, optimization.ExponentialMovingAverage): self.optimizer.swap_weights() + logging.info("(base_trainer.py - eval_end) Exiting eval_end of base trainer") return logs def eval_reduce(self, state=None, step_outputs=None): + logging.info("(base_trainer.py - eval_reduce) Starting and Exiting eval_reduce of base trainer") return self.task.aggregate_logs(state, step_outputs) diff --git a/official/core/train_lib.py b/official/core/train_lib.py index 4dd95879c..f7a07832e 100755 --- a/official/core/train_lib.py +++ b/official/core/train_lib.py @@ -55,7 +55,7 @@ def run_experiment(distribution_strategy: tf.distribute.Strategy, eval_logs: returns eval metrics logs when run_post_eval is set to True, otherwise, returns {}. """ - + with distribution_strategy.scope(): trainer = train_utils.create_trainer( params, @@ -101,7 +101,9 @@ def run_experiment(distribution_strategy: tf.distribute.Strategy, eval_steps=params.trainer.validation_steps, eval_interval=params.trainer.validation_interval) elif mode == 'eval': + logging.info('About to run evaluate from controller') controller.evaluate(steps=params.trainer.validation_steps) + logging.info('Done running evaluate from controller') elif mode == 'continuous_eval': def timeout_fn(): diff --git a/orbit/controller.py b/orbit/controller.py index 03764e997..e1f787b26 100755 --- a/orbit/controller.py +++ b/orbit/controller.py @@ -237,7 +237,6 @@ def evaluate(self, steps: int = -1) -> Optional[runner.Output]: ValueError: If `steps` is not a positive value or -1. """ self._require("evaluator", for_method="evaluate") - if steps > 0: steps_msg = f"running {steps} steps of evaluation..." elif steps == -1: @@ -251,7 +250,9 @@ def evaluate(self, steps: int = -1) -> Optional[runner.Output]: start = time.time() with self.eval_summary_manager.summary_writer().as_default(): steps_tensor = tf.convert_to_tensor(steps, dtype=tf.int32) + _log(f"(controller.py - evaluate) Running self.evaluator.evaluate()") eval_output = self.evaluator.evaluate(steps_tensor) + _log(f"(controller.py - evaluate) Done running self.evaluator.evaluate()") eval_output = tf.nest.map_structure(utils.get_value, eval_output or {}) elapsed = time.time() - start diff --git a/orbit/standard_runner.py b/orbit/standard_runner.py index 6da458920..19d753c18 100755 --- a/orbit/standard_runner.py +++ b/orbit/standard_runner.py @@ -43,6 +43,14 @@ from orbit.utils import loop_fns +from absl import logging + +def _log(message: str): + """Logs `message` to the `info` log, and also prints to stdout.""" + logging.info(message) + print(message) + + @dataclasses.dataclass(frozen=True) class StandardTrainerOptions: """Advanced options for `orbit.StandardTrainer`. @@ -292,27 +300,38 @@ def evaluate(self, num_steps: tf.Tensor) -> Optional[runner.Output]: ValueError: If `options.use_tf_while_loop` is `True` and `num_steps` is unspecified. """ + _log("(standard_runner.py - evaluate) In standard evaluator 'evaluate' function") if self._eval_options.use_tf_while_loop and num_steps == -1: + _log("Value error") raise ValueError("Looping until exhausted is not supported if " "`options.use_tf_while_loop` is `True`") + _log("(standard_runner.py - evaluate) About to run eval_begin()") outputs = self.eval_begin() # pylint: disable=assignment-from-no-return - + + _log("(standard_runner.py - evaluate) Done running eval_begin()") has_state = outputs is not None if self._eval_loop_fn is None: + _log("(standard_runner.py - evaluate) self._eval_loop_fn is None, about to run create_eval_loop_fn") self._eval_loop_fn = _create_eval_loop_fn( self.eval_step, has_state=has_state, options=self._eval_options) + _log("(standard_runner.py - evaluate) Done running create_eval_loop_fn") + _log("(standard_runner.py - evaluate) Convering eval_dataset into eval_iterator") eval_iter = tf.nest.map_structure(iter, self.eval_dataset) + _log("(standard_runner.py - evaluate) Done convering eval_dataset into eval_iterator") if self._eval_options.use_tf_while_loop and not has_state: self._eval_loop_fn(eval_iter, num_steps) else: + _log("(standard_runner.py - evaluate) Getting outputs from eval_loop_fn") outputs = self._eval_loop_fn( eval_iter, num_steps, state=outputs, reduce_fn=self.eval_reduce) - + _log("(standard_runner.py - evaluate) Done getting outputs") if outputs is None: + _log("(standard_runner.py - evaluate) outputs is None") return self.eval_end() else: + _log("(standard_runner.py - evaluate) outputs is not None") return self.eval_end(outputs) def eval_begin(self) -> Any: @@ -371,6 +390,7 @@ def eval_end(self, *args) -> Optional[runner.Output]: written to logs and as TensorBoard summaries. It can also be a nested dictionary, yielding a hierarchy of summary directories. """ + _log("eval_end()") pass def eval_reduce(self, From 716b4ec06b5c638158b7decf93aba61c78d9dc6b Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 20 Apr 2021 20:36:15 +0000 Subject: [PATCH 110/132] log statements --- .../configs/experiments/centernet-eval-tpu.yaml | 2 +- centernet/tasks/centernet.py | 13 ++++++++----- official/core/.base_trainer.py.swp | Bin 28672 -> 0 bytes official/core/base_trainer.py | 1 + orbit/utils/loop_fns.py | 1 + 5 files changed, 11 insertions(+), 6 deletions(-) delete mode 100644 official/core/.base_trainer.py.swp diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index 0cca2f172..3ee54e38a 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -61,7 +61,7 @@ task: load_extremenet_weights: false trainer: train_steps: 10000 # 160 epochs at 64 batchsize -> 500500 * 64/16 - validation_steps: -1 # 5063 #625 + validation_steps: 625 # 5063 #625 steps_per_loop: 10000 validation_interval: 1 summary_interval: 10000 diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 2deb6baa2..b54256029 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -243,6 +243,7 @@ def train_step(self, inputs, model, optimizer, metrics=None): def validation_step(self, inputs, model, metrics=None): # get the data point + print("(centernet_task) - validation step") image, label = inputs scale_replicas = tf.distribute.get_strategy().num_replicas_in_sync @@ -252,6 +253,7 @@ def validation_step(self, inputs, model, metrics=None): num_replicas = scale_replicas y_pred = model(image, training=False) + print(y_pred) y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) loss, loss_metrics = self.build_losses( y_pred['raw_output'], @@ -282,14 +284,15 @@ def validation_step(self, inputs, model, metrics=None): logs.update({self.coco_metric.name: (label, coco_model_outputs)}) - # if metrics: - # for m in metrics: - # m.update_state(loss_metrics[m.name]) - # logs.update({m.name: m.result()}) - + if metrics: + for m in metrics: + m.update_state(loss_metrics[m.name]) + logs.update({m.name: m.result()}) + print("(centernet_task) - validation_step, logs: ", logs) return logs def aggregate_logs(self, state=None, step_outputs=None): + print("(centernet_task) - aggregate_logs") if not state: self.coco_metric.reset_states() state = self.coco_metric diff --git a/official/core/.base_trainer.py.swp b/official/core/.base_trainer.py.swp deleted file mode 100644 index cfe0ca10f9b6e741d8525664d991ebf5e8b8650b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28672 zcmeI4e~cu>b;rjLLIOAhN0Bj(;<{ZC@4;U0-4P|u!V=bZ2U+0mqTK~TI6T`s(=$8V z%=FORb9)OhKkWQ+g#XAN0V3N95f>zj#IbD0&L5Cud3dA^{UtR-nw+yyd~II;rD};$|sKBxAwMO z7gd&CTdCB-&i*Xwrg73}$D5L2^e?W5z1HYg1F827RyY@(2;0>xO0!nCIk&0}T~9-Y z<9gIi=i)}ARcnRqxmuh=bL(LmRkI{)b)zKcZ4J~rd6;lu!hthzpr1ANyydl(tM>2P zD> zV8Ve32PPbtaA3lL2?r(|m~ddife8mD9GGz6Ti`%g*JJ<6$Hm^}{NLOEe{pf8^5@__ za5cCDTnzr{2P&0+01ty@a6b4LM*U;px4^yN9&io#+C`PhN5BWc?}Jm|A&`OuL|_j% z2fPNnaABqLo8VFK2>2Cn2l#n#F?i_$+6Vs#o&%485L^Y$122DnrSf6$09XN2U>Eq} z_f;y-gQvhdz%Aea*bnxBZ=7GL{4RJB{5HtIHQ;J62mTf3$Fty1!7qYMa68xq&IcdG z`SCDlgC7HzfM;=dJO~~D_k#`yz!z|Id>(unJPCdotbsk?0`L-!m_G*Z01tvqFbmED zGDgpMhe>Iyjs7|kHZgBjHzK3|HLY%%h22^-YoeeTn9yX6;CPgTX`m4n`TL+^E6Bq1 z_`F#(Hq?TLV3mKiWvj*AMypB2u30;DvKJ+-PSni~XzE&?+oZjy)@p1K+F_zRm$8Ly zi;GD!ov&!)Q&Uq*-Bw1lN9&%5Or1{LH7?r2G~&b*(IK{TsVCZ!&cC&dsCK*;x4K!C z_TnUql7%C2H`+a~E#&!(k+73~m__quJ&xOE!K^0zXm&W4wto67aW89iAUK&9*5*%X zaMZupR;wGN!I9XO%Z|k0IBd)0$6#o zP=+|2ZiGoxAH#4;S8DArP2rb@;mpLzdMgW75VCrh)LrVV8B9K8xg94A#C6S9H?VZ~ zlW@HqA;iZ0rY>)oFf(Zu^~~kqHMp_T5Dt%TX;p zfkYy7(mgv58cEzKRW@%fPXnDMQlHWK%sK@-QEH~Nu6Epn(q^z3CSCr{XktmggdF5X z7#oz5J7G6$5}uS0(nQQsHBNL0tW?ifGN;YWaagz9?|>rH-^xb0j=s!Up;53=(Fv`|WzwwKg+u*limkmNdmrM*OaL-Pq!$#!9#0Sy}GWyxH7{y2e$%l{2Kn zm{Cc{$|JH9o;2NlXB{@i4LB*ivU6}*xa3zZQ1HQf`BsY;pc`9U->>`_?6qmYXq3DEL z>ru{a8G-Rr&8^po4xG6jo7VXnK?nbDsq&1iG2 zn_Pa3s_PLGQfE7?>DFBf!);OxD{F_@8MM-BH|&;VM_ZEa4oTOJ`m17Da1lY*na&!p zJyB`X0>hmp`*2c`IqzyQiO4@9bwrUecS>bS$=eF~wS+eI>Pu7nMl?Fl#-5S)s_olp zp1ahX&%Ss2u2pQ)j)}T?s^A6(@|TA#gSQnQ=HI-XCa2x==$_pOm=sAA)@4S>JS~RK zpl#G!C_H3U8tiP6#{B-ke46+h7)L#O-=ume{m_c6yVq#KKwcqpkwj{ORjsmx_F57i}i&<82+bNSBZ41uZZL^95%eKz+O@o0fm0>+# zbeZ8?X#BE0`e55YIVBuwbwr6WPOH%cV<@Y{<@#*7uEMsh6qgCD__{=2a1Dm*Y1KBk5hQ?_;5y}4Cm6Gw8U%}OK` z=1Ho)$+OkKTQ$>$Q++fO^bC|E(P#_{aHo}~B1THgw7G3*^=FRWvTANyTwY#0vbuC= z#T;EW2aX;&xU{--^axMao5dryo12!79K@n%(Ly4cJ*-Vp`IPcjR2PDnN^Cys{R~}l z8^s1zmNPI~?6^~lD`shB%Di=PWoc!WI&SHl(}0>GT6CMUCXE{znp;$vrXKliWLa;1Zf{ok<{Xn&IGE1eyma8uk(EPx z_6K{l$n9}o_Cyn}XXv@?#I?A`6MIXE$zew2TP~+)E`5;$k&{LXnT$@>qFyFsp;xj# zh#h7&VOqHo;guW5GNTriM*n8k+!`e^PzL(UjL5@W9J_S zw}YPp7lLQ7@$Uye4!()~{tfU+@P2R%90Akd6>RtC!8^egxDxyu_WCLC5V#S{fdE_t zK84NxC|CtQ4Ze(x{w44Spbqwe3&0E5=1+iCa1a~-7lJ3~i}dI1(x=I9!hs0~CLEY> z;QyWjvKc#@O${Vb$AYlQF5bi+S49s@XYg5OXwqUTSL1%x>u0I;cX+2vtJ{cYre>_h znO_Jqd(^Lo8C&nyqTL3~DiL)+)xJX~RW&H*ko~-@yG=^?V~}o!z3OJ9JG%2OZO?@}&N?HlLW%m1 zPZa-7*ghPefIo|9gs6sv9X{BIvagkTSF&{nO)HZODwZgK;%ddQgtHmjN@!;^`|U6> zjx_7Sm_{u@Ir7dY>z`W38BsRFY;7oTXi4}2Lra^lNK`C!PAY@Zd+C45Al8oOIh@^p0{lr9}K3 z&Ge2b%{OJPbcv;WzItmK1#0Z#C$tskHg8o$&lYoaK&Br#gYU45k&IVYU+$M!eq_v*}x8B4F5qn@j&@3GEsJDaNcM6cVK zKJ;t}$AvSL?)~?JGint*SSrVssH^8NiPT=jo!f#g(U--14|HA&8!@EBoZrNx6m{J$ zLn2RN{)$#DwY@ntYefy}Z(0*U*@zJysYU8pQ`VNUNyVBCdCD8G?5WR+YdN&WK_t!u zM_ong;+ktz>Rx)ssG5TT2`y@XiS!TitKZJt^oMyPYf!(j7Z9@xm(M_}b_nx!+~Yva zat345#iki<#J+J(UlC7+|K7K27Y9de+^VhX;8NEvzT=lbEasn0VoY?*HqFa1j*PS= zmBChDaK4IDkz2xtL~3B=mVp@iD_N?8yOG<*a5r{Y9`HH#kd~ycz6NbH;F0;!+zOa! z?vRWFi*{V&Uq`TYZ|#gldeSktS4fhR@3X1n2E<0CyjjUOCCHDVsV84GBi2W~r7!8cv`mwjPd#pK@!2PPbtaA3lL2?r(|m~ddife8mD z9GGxm!h!EL2gFd-&?jrP&sd|?`Ft@zHQ4d=eswXfcN_1bk*6_kYKdwfAVGcW8ZJ8Q zl~FTY4CPs(5uL|C0$zm3ozbF!M!DZyy{4KphN4%KnV9y|x0 z2B$y|tb!GA1GoZwl~{sLffvCiz?0x!Ah8B_fTLgz{3r1RFM>~kKLYOq?*%L1Jn(14 z7~Bn7;2<~veiFP1e1&*}zXg8-o&f7$9=rj3mY9S0gBV;37QilWKKMi84&DoX30w!x z1z#in;7`D>gFd((yh5zSm%&HCDeyRW5Hvvp90pf_OTmA0U*PY+hrmg&0+zuW!DZka z@Hxg!#_kaycL|Py-Qe@w7kF0g41mYMyTCly1$O+CDJj2K``TftROj2P;&|-ap5@Y^ zvsw%N!hE`QP*-Mxe~3SHKFHyvSxc+;n!TYSL?}OLIW}kO{ zY9{+x&$D#S3ULD-Ry|a!4mnpPS1u9O6PrJlV@uU#TlU^oq@?{h4{yajZFAS!c6C$J zEF96Gimdq<%vhYeHtL1x~|U?79ivF+69EGx0c8>OMm zPwqI{XiI>j%`t|CJc4-qrshsmT?T*tRpc*Dyhdz)1A>t6d^T16`__H$2GSi<35!qj z+a}B-4_kdCG^ats5%u=Wcksy0AIMoj^rsCz5T77mA!wSB*E=o5Kh}u2mFVU6mOoQ> z!OE1C6HFWR3TYfS)ii8@yCs}`bJW40=uti=(8Ndh+duiyvo>{R`IKuURe!4Hmk6P9 z;QEz|>s~0%tbM$^pY~F~cdK-1WT6)(VMoq1ZI4T@%9AyJt;>}j(x%&rmfn@iiU3qg z_YZuz;dEcdE>ONjj>485ZTV^Wc$(!FLw?XIUCw{oIdi%oc%3k^tgv+E(xFpq*4Wm$ z^=AWN4ruK4tX;d^l3}}0G+7Zc80*EFJ4n^TZz(UwYel}&=1?U1&B2)Kh==A}Gviat zQlYk@r%W22&Y9aiZW&Oiv?EWenwg+sosS&97gZC|K){C&gbg{^hwU6E5_$`T67}qF z=Ez0IDzEJ<*lZEkDo6V3g!Hoj5$9o6CC*!ec=NDad8M%;jefVLr^aaCT(PN}P-$Ay zIV2J7a`~oAmhh)|CW@E#XiLgMu7;Is+^?%wYtvwtyd_`6l4s*mx^cPEhFpfpBe#`h zfn}_Wl=f?YcO({0PpY+=WmpEvNdoQ1^Z?AEzTntb$ zur!Y(LUV{5j%|COj-54IB3I`5wuWVZi#!Z0aUuilm!?9IPZ#r@5nyQ^+jidR)3#hx zva?1@8?+OaQp(sWa!o_-&MK$M>cYTi=KH&5V-MO0(=f{tt$Wr?Rr3r}`4W~VjU`~9 z37rUzdj0>-ipe_UO38nBR>%ftxnlNVX9c}_?27uwv4zd(-xSwh-L(m`rYmVl)(B|| z*NR{!R%@;sEc*QRs-|8ARViV$bQdS=MJC%DuH5Rn$CF%akc%HvYX4t`LHLBb|L?!= z_gC2W?*w;&H-dA(N3qu*2k!wlfgb^1z%CcN{R7}p@Bml?v*2~$67VcG`90t!a3iGP4If~_wf8PKzRQ9kr*bJ>~emR z*)a`7P?&2&q9PNnGFr?ub-S$D)#W=TX42|uJ5aQL*0g^|=EYcoVYcx6?5MQb*>|5% z+pb-HRq3lpztGlIPn(lumbjtz-i zMJqPPihss%_qDPChOys#+admur7}ar^W^kE~{m>gEQwyV&{T-h!Wi zu3B|N59G*0aV=+z#2`x@UM(C~gMaUB5GgMECU^A~#(Z1hsC{jX65E{8BO{8-xLlJc z>i&65Z9)F&h>ERC{#6Cv_n9#bhQ_~a+i70E54uogk%M5J+pu*$il8xbuJSUW7+993 ze%5NI*wM!$6n$#4w+XARKe_QGW%W}Y!>gULy4SC!{Z1!Lw!A8$8{jbCQGVmiDmM*$ z?z0^@EApk0_5AhRJEQRAH5Bqyu4DA*JAocC_O`mS8hlsT_@O`JaBi27f($PlMG8`R zwu7yOOZ%EWV=FA^X9#xV>QT9~#*@|Gt*)=he!eq}yqC~Nww6TG6)PrE-iUnI;EZk9 z{<%taQhj;j7VV$Up479gUbH}9v3>yJs{NLC+xnok!b0&)AJ=GbIO4MhI<;&OOVVj` zPx(%&Ni&iJH&>0h;)=o)u5y0Y$s_-F;%U{f8R=&&*(UaVDZx$!+nu6y*b9@C^wNg| z{KysgES2PTuD-H8Ga{EZV`XUgtukTA&aoHfQ3{4L`%Tx9UFr>#l-IZ#pCsV!Qz=9K zr5OE830s$SO2l2h_OY*UXWKoQi`#6I`oR=b!xP-`EaK)SM3e9zGyZrX5n9@R14!&ilTWnLU_ki4pr^^-(C$B@@AH#PFy z^5qOMXWSPS!tPf7&ZG#7`yiM17#e>07aD9+13qWZZ&cc@b6APPP|YQ!W^K%^r#0oA zRF69?hBYySvP2|F*PUZZm^jR|Dx_%lisQfP$CGda% diff --git a/official/core/base_trainer.py b/official/core/base_trainer.py index 054a497b9..8eaacc2b9 100755 --- a/official/core/base_trainer.py +++ b/official/core/base_trainer.py @@ -322,6 +322,7 @@ def step_fn(inputs): return logs distributed_outputs = self.strategy.run(step_fn, args=(next(iterator),)) + logging.info(f"(base trainer.py - eval_step) distributed_outputs are: {distributed_outputs}") logging.info("(base_trainer.py - eval_step) Got logs from step_fn") logging.info("(base_trainer.py - eval_step) Exiting eval_step") return tf.nest.map_structure(self.strategy.experimental_local_results, diff --git a/orbit/utils/loop_fns.py b/orbit/utils/loop_fns.py index 559485f37..80ccda7b4 100755 --- a/orbit/utils/loop_fns.py +++ b/orbit/utils/loop_fns.py @@ -71,6 +71,7 @@ def loop_fn(iterator, num_steps, state=None, reduce_fn=None): # async remote eager, we need to wrap the loop body in `async_scope`. with tf.experimental.async_scope(): while num_steps == -1 or step < num_steps: + print("(loop_fns.py) step: ", step) outputs = step_fn(iterator) if reduce_fn is not None: state = reduce_fn(state, outputs) From bf956801c947e1418617aaf4cd9db9aa861f4a87 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 20 Apr 2021 16:38:50 -0400 Subject: [PATCH 111/132] removed old ground truth builder --- centernet/dataloaders/centernet_input.py | 170 ----------------------- centernet/ops/preprocessing_ops.py | 61 -------- centernet/tasks/centernet.py | 8 +- orbit/standard_runner.py | 3 +- 4 files changed, 5 insertions(+), 237 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 8edf5f4af..5a47caf95 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -71,176 +71,6 @@ def __init__(self, raise Exception( 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' ) - - def _build_heatmap_and_regressed_features1(self, - labels, - output_size=[128, 128], - input_size=[512, 512]): - """ Generates the ground truth labels for centernet. - - Ground truth labels are generated by splatting gaussians on heatmaps for - corners and centers. Regressed features (offsets and sizes) are also - generated. - - Args: - labels: A dictionary of COCO ground truth labels with at minimum the following fields: - bbox: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension - corresponds to the top left x, top left y, bottom right x, and - bottom left y coordinates of the bounding box - classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each - box, given in the same order as the boxes - num_detections: A `Tensor` or int that gives the number of objects in the image - output_size: A `list` of length 2 containing the desired output height - and width of the heatmaps - input_size: A `list` of length 2 the expected input height and width of - the image - Returns: - Dictionary of labels with the following fields: - 'tl_heatmaps': A `Tensor` of shape [output_h, output_w, num_classes], - heatmap with splatted gaussians centered at the positions and channels - corresponding to the top left location and class of the object - 'br_heatmaps': `Tensor` of shape [output_h, output_w, num_classes], - heatmap with splatted gaussians centered at the positions and channels - corresponding to the bottom right location and class of the object - 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], - heatmap with splatted gaussians centered at the positions and channels - corresponding to the center location and class of the object - 'tl_offset': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the x-offset and y-offset of the top-left - corner of an object. All other entires are 0 - 'br_offset': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the x-offset and y-offset of the - bottom-right corner of an object. All other entires are 0 - 'ct_offset': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the x-offset and y-offset of the center of - an object. All other entires are 0 - 'size': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the width and height of an object. All - other entires are 0 - 'box_mask': `Tensor` of shape [max_num_instances], where the first - num_boxes entries are 1. All other entires are 0 - 'box_indices': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the y-center and x-center of a valid box. - These are used to extract the regressed box features from the - prediction when computing the loss - """ - - # boxes and classes are cast to self._dtype already from build_label - boxes = labels['bbox'] - classes = labels['classes'] - input_size = tf.cast(input_size, self._dtype) - output_size = tf.cast(output_size, self._dtype) - input_h, input_w = input_size[0], input_size[1] - output_h, output_w = output_size[0], output_size[1] - - # We will transpose the heatmaps at the end - tl_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - br_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - ct_heatmaps = tf.zeros((self._num_classes, output_h, output_w), dtype=self._dtype) - - # Maps for offset and size predictions - tl_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - br_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - ct_offset = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - size = tf.zeros((self._max_num_instances, 2), dtype=self._dtype) - - # Masks for valid boxes - box_mask = tf.zeros((self._max_num_instances), dtype=tf.int32) - box_indices = tf.zeros((self._max_num_instances, 2), dtype=tf.int32) - - # Scaling factor for determining center/corners - width_ratio = output_w / input_w - height_ratio = output_h / input_h - - num_objects = labels['num_detections'] - - height = tf.cast(0.0, self._dtype) - width = tf.cast(0.0, self._dtype) - for tag_ind in tf.range(num_objects): - box = boxes[tag_ind] - obj_class = classes[tag_ind] - 1 # TODO: See if subtracting 1 from the class like the paper is unnecessary - - ytl, xtl, ybr, xbr = box[0], box[1], box[2], box[3] - - xct, yct = ( - (xtl + xbr) / 2, - (ytl + ybr) / 2 - ) - - # Scale center and corner locations - # These should be dtype=float32 - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - # Fit center and corners onto the output image - # These should be dtype=float32 - xtl = tf.math.floor(fxtl) - ytl = tf.math.floor(fytl) - xbr = tf.math.floor(fxbr) - ybr = tf.math.floor(fybr) - xct = tf.math.floor(fxct) - yct = tf.math.floor(fyct) - - # Splat gaussian at for the center/corner heatmaps - if self._use_gaussian_bump: - # Check: do we need to normalize these boxes? - width = box[3] - box[1] - height = box[2] - box[0] - - width = tf.math.ceil(width * width_ratio) - height = tf.math.ceil(height * height_ratio) - - if self._gaussian_rad == -1: - radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) - radius = tf.math.maximum(tf.cast(0.0, radius.dtype), tf.math.floor(radius)) - else: - radius = self._gaussian_rad - - tl_heatmaps = preprocessing_ops.draw_gaussian1(tl_heatmaps, [[obj_class, xtl, ytl, radius]]) - br_heatmaps = preprocessing_ops.draw_gaussian1(br_heatmaps, [[obj_class, xbr, ybr, radius]]) - ct_heatmaps = preprocessing_ops.draw_gaussian1(ct_heatmaps, [[obj_class, xct, yct, radius]], scaling_factor=5) - - else: - tl_heatmaps = tf.tensor_scatter_nd_update(tl_heatmaps, - [[tf.cast(obj_class, tf.int32), tf.cast(ytl, tf.int32), tf.cast(xtl, tf.int32)]], [1]) - br_heatmaps = tf.tensor_scatter_nd_update(br_heatmaps, - [[tf.cast(obj_class, tf.int32), tf.cast(ybr, tf.int32), tf.cast(xbr, tf.int32)]], [1]) - ct_heatmaps = tf.tensor_scatter_nd_update(ct_heatmaps, - [[tf.cast(obj_class, tf.int32), tf.cast(yct, tf.int32), tf.cast(xct, tf.int32)]], [1]) - - # Add box offset and size to the ground truth - tl_offset = tf.tensor_scatter_nd_update(tl_offset, [[tag_ind, 0], [tag_ind, 1]], [fxtl - xtl, fytl - ytl]) - br_offset = tf.tensor_scatter_nd_update(br_offset, [[tag_ind, 0], [tag_ind, 1]], [fxbr - xbr, fybr - ybr]) - ct_offset = tf.tensor_scatter_nd_update(ct_offset, [[tag_ind, 0], [tag_ind, 1]], [fxct - xct, fyct - yct]) - size = tf.tensor_scatter_nd_update(size, [[tag_ind, 0], [tag_ind, 1]], [width, height]) - - # Initialy the mask is zeros, but each valid box needs to be unmasked - box_mask = tf.tensor_scatter_nd_update(box_mask, [[tag_ind]], [1]) - - # Contains the y and x coordinate of the box center in the heatmap - box_indices = tf.tensor_scatter_nd_update(box_indices, [[tag_ind, 0], [tag_ind, 1]], [yct, xct]) - - # Make heatmaps of shape [height, width, num_classes] - tl_heatmaps = tf.transpose(tl_heatmaps, perm=[1, 2, 0]) - br_heatmaps = tf.transpose(br_heatmaps, perm=[1, 2, 0]) - ct_heatmaps = tf.transpose(ct_heatmaps, perm=[1, 2, 0]) - - labels = { - 'tl_heatmaps': tl_heatmaps, - 'br_heatmaps': br_heatmaps, - 'ct_heatmaps': ct_heatmaps, - 'tl_offset': tl_offset, - 'br_offset': br_offset, - 'ct_offset': ct_offset, - 'size': size, - 'box_mask': box_mask, - 'box_indices': box_indices - } - return labels def _build_heatmap_and_regressed_features(self, labels, diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 63205d234..0c1da6ae7 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -123,67 +123,6 @@ def write_all(ta, index, values): ta = ta.write(index + i, values[i, ...]) return ta, index + i -# scaling_factor doesn't do anything right now -@tf.function -def draw_gaussian1(heatmap, blobs, scaling_factor=1, dtype=tf.float32): - """ - Draws a gaussian heatmap around a center point given a radius. - Params: - heatmap (tf.Tensor): heatmap placeholder to fill - blobs (tf.Tensor): a tensor whose last dimension is 4 integers for - the category of the object, center (x, y), and for radius of the - gaussian - scaling_factor (int): scaling factor for gaussian - """ - blobs = tf.cast(blobs, tf.int32) - category = blobs[..., 0] - x = blobs[..., 1] - y = blobs[..., 2] - radius = blobs[..., 3] - num_boxes = tf.shape(radius)[0] - - diameter = 2 * radius + 1 - - heatmap_shape = tf.shape(heatmap) - height, width = heatmap_shape[-2], heatmap_shape[-1] - - left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) - top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) - - # print('heatmap ',heatmap) - # print(len(heatmap)) - # print('category ',category) - - # TODO: make sure this replicates original functionality - # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] - # masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] - # np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - update_count = tf.reduce_sum((bottom + top) * (right + left)) - masked_gaussian_ta = tf.TensorArray(dtype, size=update_count) - heatmap_mask_ta = tf.TensorArray(tf.int32, element_shape=tf.TensorShape((3,)), size=update_count) - i = 0 - for j in range(num_boxes): - cat = category[j] - X = x[j] - Y = y[j] - R = radius[j] - l = left[j] - r = right[j] - t = top[j] - b = bottom[j] - - gaussian = _gaussian_penalty(R, dtype=dtype) - masked_gaussian_instance = tf.reshape(gaussian[R - t:R + b, R - l:R + r], (-1,)) - heatmap_mask_instance = cartesian_product([cat], tf.range(Y - t, Y + b), tf.range(X - l, X + r)) - masked_gaussian_ta, _ = write_all(masked_gaussian_ta, i, masked_gaussian_instance) - heatmap_mask_ta, i = write_all(heatmap_mask_ta, i, heatmap_mask_instance) - masked_gaussian = masked_gaussian_ta.stack() - heatmap_mask = heatmap_mask_ta.stack() - heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) - heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, tf.cast(masked_gaussian * scaling_factor, heatmap.dtype)) - # print('after ',heatmap) - return heatmap - @tf.function def draw_gaussian(hm_shape, blob, dtype, scaling_factor=1): """ Draws an instance of a 2D gaussian on a heatmap. diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 2deb6baa2..3e9202fd6 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -282,10 +282,10 @@ def validation_step(self, inputs, model, metrics=None): logs.update({self.coco_metric.name: (label, coco_model_outputs)}) - # if metrics: - # for m in metrics: - # m.update_state(loss_metrics[m.name]) - # logs.update({m.name: m.result()}) + if metrics: + for m in metrics: + m.update_state(loss_metrics[m.name]) + logs.update({m.name: m.result()}) return logs diff --git a/orbit/standard_runner.py b/orbit/standard_runner.py index 19d753c18..8290c5ffb 100755 --- a/orbit/standard_runner.py +++ b/orbit/standard_runner.py @@ -38,13 +38,12 @@ from typing import Any, Optional import tensorflow as tf +from absl import logging from orbit import runner from orbit.utils import loop_fns -from absl import logging - def _log(message: str): """Logs `message` to the `info` log, and also prints to stdout.""" logging.info(message) From 28923c9d7ac17f6c373b371893022659b616517b Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 20 Apr 2021 21:56:29 +0000 Subject: [PATCH 112/132] killed? --- centernet/configs/centernet.py | 10 +++++----- centernet/configs/experiments/centernet-eval-tpu.yaml | 4 +--- centernet/tasks/centernet.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 3f8f359b1..ad134ecc4 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -286,7 +286,7 @@ def centernet_custom() -> cfg.ExperimentConfig: summary_interval=8000, checkpoint_interval=10000, train_steps=num_batches, - validation_steps=1000, + validation_steps=625, validation_interval=10, optimizer_config=optimization.OptimizationConfig({ 'optimizer': { @@ -328,7 +328,7 @@ def centernet_custom() -> cfg.ExperimentConfig: def centernet_tpu() -> cfg.ExperimentConfig: """COCO object detection with CenterNet.""" train_batch_size = 1 - eval_batch_size = 1 + eval_batch_size = 8 base_default = 1200000 num_batches = 1200000 * 64 / train_batch_size @@ -347,14 +347,14 @@ def centernet_tpu() -> cfg.ExperimentConfig: # 'val*'), is_training=False, global_batch_size=eval_batch_size, - shuffle_buffer_size=10000)), + shuffle_buffer_size=100)), trainer=cfg.TrainerConfig( steps_per_loop=2000, summary_interval=8000, checkpoint_interval=10000, train_steps=num_batches, - validation_steps=1000, - validation_interval=10, + validation_steps=625, + validation_interval=1, optimizer_config=optimization.OptimizationConfig({ 'optimizer': { 'type': 'sgd', diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index 3ee54e38a..e151a1adf 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -24,7 +24,7 @@ task: validation_data: input_path: 'gs://tensorflow2/coco_records/val/2017*' global_batch_size: 8 - tfds_download: true + tfds_download: false dtype: bfloat16 is_training: false drop_remainder: false @@ -37,7 +37,6 @@ task: gaussian_rad: -1 gaussian_iou: 0.7 output_dims: 128 - dtype: bfloat16 shuffle_buffer_size: 100 subtasks: detection: @@ -51,7 +50,6 @@ task: corner_pull_weight: 0.1 corner_push_weight: 0.1 offset_weight: 1.0 - segmentation: per_category_metrics: false weight_decay: 0.0005 init_checkpoint: '' diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index b54256029..3ca0c16f6 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -48,6 +48,7 @@ def build_inputs(self, params, input_context=None): parser_fn=parser.parse_fn(params.is_training), postprocess_fn=parser.postprocess_fn(params.is_training)) dataset = reader.read(input_context=input_context) + print(dataset) return dataset def build_model(self): @@ -253,7 +254,6 @@ def validation_step(self, inputs, model, metrics=None): num_replicas = scale_replicas y_pred = model(image, training=False) - print(y_pred) y_pred = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), y_pred) loss, loss_metrics = self.build_losses( y_pred['raw_output'], From ef8769458898c9f25c65b99498875b041c41b56b Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 20 Apr 2021 23:15:13 +0000 Subject: [PATCH 113/132] got rid of logging statements --- centernet/tasks/centernet.py | 3 --- official/core/base_trainer.py | 12 ------------ official/core/train_lib.py | 2 -- orbit/controller.py | 2 -- orbit/standard_runner.py | 20 -------------------- orbit/utils/loop_fns.py | 1 - 6 files changed, 40 deletions(-) diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 3ca0c16f6..6acfa88f0 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -244,7 +244,6 @@ def train_step(self, inputs, model, optimizer, metrics=None): def validation_step(self, inputs, model, metrics=None): # get the data point - print("(centernet_task) - validation step") image, label = inputs scale_replicas = tf.distribute.get_strategy().num_replicas_in_sync @@ -288,11 +287,9 @@ def validation_step(self, inputs, model, metrics=None): for m in metrics: m.update_state(loss_metrics[m.name]) logs.update({m.name: m.result()}) - print("(centernet_task) - validation_step, logs: ", logs) return logs def aggregate_logs(self, state=None, step_outputs=None): - print("(centernet_task) - aggregate_logs") if not state: self.coco_metric.reset_states() state = self.coco_metric diff --git a/official/core/base_trainer.py b/official/core/base_trainer.py index 8eaacc2b9..a626a8e37 100755 --- a/official/core/base_trainer.py +++ b/official/core/base_trainer.py @@ -299,38 +299,28 @@ def step_fn(inputs): def eval_begin(self): """Sets up metrics.""" - logging.info("(base_trainer.py - eval_begin) In eval_begin of base trainer") for metric in self.validation_metrics + [self.validation_loss]: metric.reset_states() # Swaps weights to test on weights moving average. if self.optimizer and isinstance( self.optimizer, optimization.ExponentialMovingAverage): self.optimizer.swap_weights() - logging.info("(base_trainer.py - eval_begin) Exitingeval_begin of base trainer") - def eval_step(self, iterator): """See base class.""" - logging.info("(base_trainer.py - eval_step) In eval_step of base trainer") def step_fn(inputs): - logging.info("(base_trainer.py - eval_step - step_fn) Running task.validation_step()") logs = self.task.validation_step( inputs, model=self.model, metrics=self.validation_metrics) if self.task.loss in logs: self._validation_loss.update_state(logs[self.task.loss]) - logging.info("(base_trainer.py - eval_step - step_fn) Returning logs from task.validation_step()") return logs distributed_outputs = self.strategy.run(step_fn, args=(next(iterator),)) - logging.info(f"(base trainer.py - eval_step) distributed_outputs are: {distributed_outputs}") - logging.info("(base_trainer.py - eval_step) Got logs from step_fn") - logging.info("(base_trainer.py - eval_step) Exiting eval_step") return tf.nest.map_structure(self.strategy.experimental_local_results, distributed_outputs) def eval_end(self, aggregated_logs=None): """Processes evaluation results.""" - logging.info("(base_trainer.py - eval_end) In eval_end of base trainer") logs = {} for metric in self.validation_metrics: logs[metric.name] = metric.result() @@ -357,9 +347,7 @@ def eval_end(self, aggregated_logs=None): if self.optimizer and isinstance( self.optimizer, optimization.ExponentialMovingAverage): self.optimizer.swap_weights() - logging.info("(base_trainer.py - eval_end) Exiting eval_end of base trainer") return logs def eval_reduce(self, state=None, step_outputs=None): - logging.info("(base_trainer.py - eval_reduce) Starting and Exiting eval_reduce of base trainer") return self.task.aggregate_logs(state, step_outputs) diff --git a/official/core/train_lib.py b/official/core/train_lib.py index f7a07832e..f6db7875f 100755 --- a/official/core/train_lib.py +++ b/official/core/train_lib.py @@ -101,9 +101,7 @@ def run_experiment(distribution_strategy: tf.distribute.Strategy, eval_steps=params.trainer.validation_steps, eval_interval=params.trainer.validation_interval) elif mode == 'eval': - logging.info('About to run evaluate from controller') controller.evaluate(steps=params.trainer.validation_steps) - logging.info('Done running evaluate from controller') elif mode == 'continuous_eval': def timeout_fn(): diff --git a/orbit/controller.py b/orbit/controller.py index e1f787b26..bed2af34d 100755 --- a/orbit/controller.py +++ b/orbit/controller.py @@ -250,9 +250,7 @@ def evaluate(self, steps: int = -1) -> Optional[runner.Output]: start = time.time() with self.eval_summary_manager.summary_writer().as_default(): steps_tensor = tf.convert_to_tensor(steps, dtype=tf.int32) - _log(f"(controller.py - evaluate) Running self.evaluator.evaluate()") eval_output = self.evaluator.evaluate(steps_tensor) - _log(f"(controller.py - evaluate) Done running self.evaluator.evaluate()") eval_output = tf.nest.map_structure(utils.get_value, eval_output or {}) elapsed = time.time() - start diff --git a/orbit/standard_runner.py b/orbit/standard_runner.py index 8290c5ffb..b26dff4bd 100755 --- a/orbit/standard_runner.py +++ b/orbit/standard_runner.py @@ -38,18 +38,11 @@ from typing import Any, Optional import tensorflow as tf -from absl import logging from orbit import runner from orbit.utils import loop_fns -def _log(message: str): - """Logs `message` to the `info` log, and also prints to stdout.""" - logging.info(message) - print(message) - - @dataclasses.dataclass(frozen=True) class StandardTrainerOptions: """Advanced options for `orbit.StandardTrainer`. @@ -299,38 +292,26 @@ def evaluate(self, num_steps: tf.Tensor) -> Optional[runner.Output]: ValueError: If `options.use_tf_while_loop` is `True` and `num_steps` is unspecified. """ - _log("(standard_runner.py - evaluate) In standard evaluator 'evaluate' function") if self._eval_options.use_tf_while_loop and num_steps == -1: - _log("Value error") raise ValueError("Looping until exhausted is not supported if " "`options.use_tf_while_loop` is `True`") - _log("(standard_runner.py - evaluate) About to run eval_begin()") outputs = self.eval_begin() # pylint: disable=assignment-from-no-return - _log("(standard_runner.py - evaluate) Done running eval_begin()") has_state = outputs is not None if self._eval_loop_fn is None: - _log("(standard_runner.py - evaluate) self._eval_loop_fn is None, about to run create_eval_loop_fn") self._eval_loop_fn = _create_eval_loop_fn( self.eval_step, has_state=has_state, options=self._eval_options) - _log("(standard_runner.py - evaluate) Done running create_eval_loop_fn") - _log("(standard_runner.py - evaluate) Convering eval_dataset into eval_iterator") eval_iter = tf.nest.map_structure(iter, self.eval_dataset) - _log("(standard_runner.py - evaluate) Done convering eval_dataset into eval_iterator") if self._eval_options.use_tf_while_loop and not has_state: self._eval_loop_fn(eval_iter, num_steps) else: - _log("(standard_runner.py - evaluate) Getting outputs from eval_loop_fn") outputs = self._eval_loop_fn( eval_iter, num_steps, state=outputs, reduce_fn=self.eval_reduce) - _log("(standard_runner.py - evaluate) Done getting outputs") if outputs is None: - _log("(standard_runner.py - evaluate) outputs is None") return self.eval_end() else: - _log("(standard_runner.py - evaluate) outputs is not None") return self.eval_end(outputs) def eval_begin(self) -> Any: @@ -389,7 +370,6 @@ def eval_end(self, *args) -> Optional[runner.Output]: written to logs and as TensorBoard summaries. It can also be a nested dictionary, yielding a hierarchy of summary directories. """ - _log("eval_end()") pass def eval_reduce(self, diff --git a/orbit/utils/loop_fns.py b/orbit/utils/loop_fns.py index 80ccda7b4..559485f37 100755 --- a/orbit/utils/loop_fns.py +++ b/orbit/utils/loop_fns.py @@ -71,7 +71,6 @@ def loop_fn(iterator, num_steps, state=None, reduce_fn=None): # async remote eager, we need to wrap the loop body in `async_scope`. with tf.experimental.async_scope(): while num_steps == -1 or step < num_steps: - print("(loop_fns.py) step: ", step) outputs = step_fn(iterator) if reduce_fn is not None: state = reduce_fn(state, outputs) From 10dc0fdfb07a68a682714b89a4e722cd89f4977d Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 20 Apr 2021 20:46:02 -0400 Subject: [PATCH 114/132] try getting rid of corner labels --- centernet/dataloaders/centernet_input.py | 64 ++-- centernet/tasks/_centernet_input.py | 326 ------------------ .../tasks/centernet_object_detection_test.py | 195 ----------- centernet/tasks/show_heatmaps.py | 109 ------ 4 files changed, 32 insertions(+), 662 deletions(-) delete mode 100644 centernet/tasks/_centernet_input.py delete mode 100644 centernet/tasks/centernet_object_detection_test.py delete mode 100644 centernet/tasks/show_heatmaps.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 5a47caf95..ce85ad65f 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -164,8 +164,8 @@ def _build_heatmap_and_regressed_features(self, # Offset computations to make up for discretization error # used for offset maps - tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) - br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) + # tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) + # br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) # Get the scaled box dimensions for computing the gaussian radius @@ -179,13 +179,13 @@ def _build_heatmap_and_regressed_features(self, box_widths_heights = tf.stack([box_widths, box_heights], axis=-1) # Center/corner heatmaps - tl_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) - br_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) + # tl_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) + # br_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) ct_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) # Maps for offset and size features for each instance of a box - tl_offset = tf.zeros((self._max_num_instances, 2), self._dtype) - br_offset = tf.zeros((self._max_num_instances, 2), self._dtype) + # tl_offset = tf.zeros((self._max_num_instances, 2), self._dtype) + # br_offset = tf.zeros((self._max_num_instances, 2), self._dtype) ct_offset = tf.zeros((self._max_num_instances, 2), self._dtype) size = tf.zeros((self._max_num_instances, 2), self._dtype) @@ -205,39 +205,39 @@ def _build_heatmap_and_regressed_features(self, else: radius = tf.constant([self._gaussian_rad] * num_objects, self._dtype) # These blobs contain information needed to draw the gaussian - tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) - br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) + # tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) + # br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) ct_blobs = tf.stack([classes, xct, yct, radius], axis=-1) # Get individual gaussian contributions from each bounding box - tl_gaussians = tf.map_fn( - fn=lambda x: preprocessing_ops.draw_gaussian( - tf.shape(tl_heatmap), x, self._dtype), elems=tl_blobs) - br_gaussians = tf.map_fn( - fn=lambda x: preprocessing_ops.draw_gaussian( - tf.shape(br_heatmap), x, self._dtype), elems=br_blobs) + # tl_gaussians = tf.map_fn( + # fn=lambda x: preprocessing_ops.draw_gaussian( + # tf.shape(tl_heatmap), x, self._dtype), elems=tl_blobs) + # br_gaussians = tf.map_fn( + # fn=lambda x: preprocessing_ops.draw_gaussian( + # tf.shape(br_heatmap), x, self._dtype), elems=br_blobs) ct_gaussians = tf.map_fn( fn=lambda x: preprocessing_ops.draw_gaussian( tf.shape(ct_heatmap), x, self._dtype), elems=ct_blobs) # Combine contributions into single heatmaps - tl_heatmap = tf.math.reduce_max(tl_gaussians, axis=0) - br_heatmap = tf.math.reduce_max(br_gaussians, axis=0) + # tl_heatmap = tf.math.reduce_max(tl_gaussians, axis=0) + # br_heatmap = tf.math.reduce_max(br_gaussians, axis=0) ct_heatmap = tf.math.reduce_max(ct_gaussians, axis=0) else: # Instead of a gaussian, insert 1s in the center and corner heatmaps - tl_hm_update_indices = tf.cast( - tf.stack([ytl, xtl, classes], axis=-1), tf.int32) - br_hm_update_indices = tf.cast( - tf.stack([ybr, xbr, classes], axis=-1), tf.int32) + # tl_hm_update_indices = tf.cast( + # tf.stack([ytl, xtl, classes], axis=-1), tf.int32) + # br_hm_update_indices = tf.cast( + # tf.stack([ybr, xbr, classes], axis=-1), tf.int32) ct_hm_update_indices = tf.cast( tf.stack([yct, xct, classes], axis=-1), tf.int32) - tl_heatmap = tf.tensor_scatter_nd_update(tl_heatmap, - tl_hm_update_indices, [1] * num_objects) - br_heatmap = tf.tensor_scatter_nd_update(br_heatmap, - br_hm_update_indices, [1] * num_objects) + # tl_heatmap = tf.tensor_scatter_nd_update(tl_heatmap, + # tl_hm_update_indices, [1] * num_objects) + # br_heatmap = tf.tensor_scatter_nd_update(br_heatmap, + # br_hm_update_indices, [1] * num_objects) ct_heatmap = tf.tensor_scatter_nd_update(ct_heatmap, ct_hm_update_indices, [1] * num_objects) @@ -247,10 +247,10 @@ def _build_heatmap_and_regressed_features(self, update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) # Write the offsets of each box instance - tl_offset = tf.tensor_scatter_nd_update( - tl_offset, update_indices, tl_offset_values) - br_offset = tf.tensor_scatter_nd_update( - br_offset, update_indices, br_offset_values) + # tl_offset = tf.tensor_scatter_nd_update( + # tl_offset, update_indices, tl_offset_values) + # br_offset = tf.tensor_scatter_nd_update( + # br_offset, update_indices, br_offset_values) ct_offset = tf.tensor_scatter_nd_update( ct_offset, update_indices, ct_offset_values) @@ -268,11 +268,11 @@ def _build_heatmap_and_regressed_features(self, box_indices, update_indices, box_index_values) labels = { - 'tl_heatmaps': tl_heatmap, - 'br_heatmaps': br_heatmap, + # 'tl_heatmaps': tl_heatmap, + # 'br_heatmaps': br_heatmap, 'ct_heatmaps': ct_heatmap, - 'tl_offset': tl_offset, - 'br_offset': br_offset, + # 'tl_offset': tl_offset, + # 'br_offset': br_offset, 'ct_offset': ct_offset, 'size': size, 'box_mask': box_mask, diff --git a/centernet/tasks/_centernet_input.py b/centernet/tasks/_centernet_input.py deleted file mode 100644 index c07630cdd..000000000 --- a/centernet/tasks/_centernet_input.py +++ /dev/null @@ -1,326 +0,0 @@ -import tensorflow as tf -from absl import logging -from tensorflow.keras.mixed_precision import experimental as mixed_precision - -from centernet.dataloaders import centernet_input -from official.core import base_task, input_reader, task_factory -from official.vision.beta.evaluation import coco_evaluator -from yolo.configs import yolo as exp_cfg -from yolo.dataloaders import yolo_input -from yolo.dataloaders.decoders import tfds_coco_decoder -from yolo.ops.box_ops import xcycwh_to_yxyx -from yolo.ops.kmeans_anchors import BoxGenInputReader - - -@task_factory.register_task_cls(exp_cfg.YoloTask) -class YoloTask(base_task.Task): - """A single-replica view of training procedure. - RetinaNet task provides artifacts for training/evalution procedures, including - loading/iterating over Datasets, initializing the model, calculating the loss, - post-processing, and customized metrics with reduction. - """ - - def __init__(self, params, logging_dir: str = None): - super().__init__(params, logging_dir) - self._loss_dict = None - self._num_boxes = None - self._anchors_built = False - - self._masks = None - self._path_scales = None - self._x_y_scales = None - self.coco_metric = None - return - - def build_model(self): - """get an instance of Yolo v3 or v4""" - from yolo.modeling.Yolo import build_yolo - params = self.task_config.train_data - model_base_cfg = self.task_config.model - l2_weight_decay = self.task_config.weight_decay / 2.0 - - masks, path_scales, xy_scales = self._get_masks() - self._get_boxes(gen_boxes=params.is_training) - - input_specs = tf.keras.layers.InputSpec(shape=[None] + - model_base_cfg.input_size) - l2_regularizer = ( - tf.keras.regularizers.l2(l2_weight_decay) if l2_weight_decay else None) - - model, losses = build_yolo(input_specs, model_base_cfg, l2_regularizer, - masks, xy_scales, path_scales) - self._loss_dict = losses - return model - - def build_losses(self, outputs, labels, aux_losses=None): - loss = 0.0 - loss_box = 0.0 - loss_conf = 0.0 - loss_class = 0.0 - metric_dict = dict() - - grid = labels['grid_form'] - for key in outputs.keys(): - # _loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](labels, outputs[key]) - _loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](grid[key], outputs[key]) - #_loss, _loss_box, _loss_conf, _loss_class, _avg_iou, _recall50 = self._loss_dict[key](labels[key], outputs[key]) - loss += _loss - loss_box += _loss_box - loss_conf += _loss_conf - loss_class += _loss_class - metric_dict[f"recall50_{key}"] = tf.stop_gradient(_recall50) - metric_dict[f"avg_iou_{key}"] = tf.stop_gradient(_avg_iou) - - metric_dict['box_loss'] = loss_box - metric_dict['conf_loss'] = loss_conf - metric_dict['class_loss'] = loss_class - - return loss, metric_dict - - def build_metrics(self, training=True): - #return super().build_metrics(training=training) - if not training: - self.coco_metric = coco_evaluator.COCOEvaluator( - annotation_file=self.task_config.annotation_file, - include_mask=False, - need_rescale_bboxes=False, - per_category_metrics=self._task_config.per_category_metrics) - return [] - - def train_step(self, inputs, model, optimizer, metrics=None): - #get the data point - image, label = inputs - num_replicas = tf.distribute.get_strategy().num_replicas_in_sync - with tf.GradientTape() as tape: - # compute a prediction - # cast to float32 - y_pred = model(image, training=True) - loss, metrics = self.build_losses(y_pred['raw_output'], label) - scaled_loss = loss / num_replicas - - # scale the loss for numerical stability - if isinstance(optimizer, mixed_precision.LossScaleOptimizer): - scaled_loss = optimizer.get_scaled_loss(scaled_loss) - # compute the gradient - train_vars = model.trainable_variables - gradients = tape.gradient(scaled_loss, train_vars) - # get unscaled loss if the scaled_loss was used - if isinstance(optimizer, mixed_precision.LossScaleOptimizer): - gradients = optimizer.get_unscaled_gradients(gradients) - if self.task_config.gradient_clip_norm > 0.0: - gradients, _ = tf.clip_by_global_norm(gradients, - self.task_config.gradient_clip_norm) - optimizer.apply_gradients(zip(gradients, train_vars)) - - #custom metrics - logs = {'loss': loss} - logs.update(metrics) - - #tf.print("loss: ", logs["loss"], end = "\n") - tf.print(logs, end='\n') - - ret = '\033[F' * (len(logs.keys()) + 1) - tf.print(ret, end='\n') - - return logs - - def validation_step(self, inputs, model, metrics=None): - #get the data point - image, label = inputs - - # computer detivative and apply gradients - y_pred = model(image, training=False) - loss, metrics = self.build_losses(y_pred['raw_output'], label) - - # #custom metrics - loss_metrics = {'loss': loss} - loss_metrics.update(metrics) - label['boxes'] = xcycwh_to_yxyx(label['bbox']) - del label['bbox'] - - coco_model_outputs = { - 'detection_boxes': y_pred['bbox'], - 'detection_scores': y_pred['confidence'], - 'detection_classes': y_pred['classes'], - 'num_detections': tf.shape(y_pred['bbox'])[:-1], - 'source_id': label['source_id'], - } - - loss_metrics.update({self.coco_metric.name: (label, coco_model_outputs)}) - return loss_metrics - - def aggregate_logs(self, state=None, step_outputs=None): - #return super().aggregate_logs(state=state, step_outputs=step_outputs) - if not state: - self.coco_metric.reset_states() - state = self.coco_metric - self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], - step_outputs[self.coco_metric.name][1]) - return state - - def reduce_aggregated_logs(self, aggregated_logs): - #return super().reduce_aggregated_logsI(aggregated_logs) - return self.coco_metric.result() - - @property - def anchors(self): - return self.task_config.model.boxes - - def _get_boxes(self, gen_boxes=True): - # gen_boxes = params.is_training - if gen_boxes and self.task_config.model.boxes == None and not self._anchors_built: - # must save the boxes! - model_base_cfg = self.task_config.model - self._num_boxes = (model_base_cfg.max_level - model_base_cfg.min_level + - 1) * model_base_cfg.boxes_per_scale - decoder = tfds_coco_decoder.MSCOCODecoder() - reader = BoxGenInputReader( - params, - dataset_fn=tf.data.TFRecordDataset, - decoder_fn=decoder.decode, - parser_fn=None) - anchors = reader.read( - k=9, image_width=params.parser.image_w, input_context=input_context) - self.task_config.model.set_boxes(anchors) - self._anchors_built = True - del reader - - return self.task_config.model.boxes - - def _get_masks(self, - xy_exponential=False, - exp_base=2, - xy_scale_base='default_value'): - start = 0 - boxes = {} - path_scales = {} - scale_x_y = {} - - if xy_scale_base == 'default_base': - xy_scale_base = 0.05 - xy_scale_base = xy_scale_base / ( - self._boxes_per_level * (self._max_level - self._min_level + 1) - 1) - elif xy_scale_base == 'default_value': - xy_scale_base = 0.00625 - - params = self.task_config.model - - if self._masks == None or self._path_scales == None or self._x_y_scales == None: - for i in range(params.min_level, params.max_level + 1): - boxes[str(i)] = list(range(start, params.boxes_per_scale + start)) - path_scales[str(i)] = 2**i - if xy_exponential: - scale_x_y[str(i)] = 1.0 + xy_scale_base * (exp_base**i) - else: - scale_x_y[str(i)] = 1.0 - start += params.boxes_per_scale - - self._masks = boxes - self._path_scales = path_scales - self._x_y_scales = scale_x_y - - return self._masks, self._path_scales, self._x_y_scales - - def initialize(self, model: tf.keras.Model): - if self.task_config.load_darknet_weights: - from yolo.utils import DarkNetConverter - from yolo.utils._darknet2tf.load_weights import split_converter - from yolo.utils._darknet2tf.load_weights2 import load_weights_backbone - from yolo.utils._darknet2tf.load_weights2 import load_weights_neck - from yolo.utils._darknet2tf.load_weights2 import load_head - from yolo.utils._darknet2tf.load_weights2 import load_weights_prediction_layers - from yolo.utils.downloads.file_manager import download - - weights_file = self.task_config.model.darknet_weights_file - config_file = self.task_config.model.darknet_weights_cfg - - if ('cache' not in weights_file and 'cache' not in config_file): - list_encdec = DarkNetConverter.read(config_file, weights_file) - else: - import os - path = os.path.abspath('cache') - if (not os.path.isdir(path)): - os.mkdir(path) - - cfg = f"{path}/cfg/{config_file.split('/')[-1]}" - if not os.path.isfile(cfg): - download(config_file.split('/')[-1]) - - wgt = f"{path}/weights/{weights_file.split('/')[-1]}" - if not os.path.isfile(wgt): - download(weights_file.split('/')[-1]) - - list_encdec = DarkNetConverter.read(cfg, wgt) - - splits = model.backbone._splits - if 'neck_split' in splits.keys(): - encoder, neck, decoder = split_converter(list_encdec, - splits['backbone_split'], - splits['neck_split']) - else: - encoder, decoder = split_converter(list_encdec, - splits['backbone_split']) - neck = None - - load_weights_backbone(model.backbone, encoder) - model.backbone.trainable = False - - if self.task_config.darknet_load_decoder: - if neck != None: - load_weights_neck(model.decoder.neck, neck) - model.decoder.neck.trainable = False - cfgheads = load_head(model.decoder.head, decoder) - model.decoder.head.trainable = False - load_weights_prediction_layers(cfgheads, model.head) - model.head.trainable = False - else: - """Loading pretrained checkpoint.""" - if not self.task_config.init_checkpoint: - return - - ckpt_dir_or_file = self.task_config.init_checkpoint - if tf.io.gfile.isdir(ckpt_dir_or_file): - ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) - - # Restoring checkpoint. - if self.task_config.init_checkpoint_modules == 'all': - ckpt = tf.train.Checkpoint(**model.checkpoint_items) - status = ckpt.restore(ckpt_dir_or_file) - status.assert_consumed() - elif self.task_config.init_checkpoint_modules == 'backbone': - ckpt = tf.train.Checkpoint(backbone=model.backbone) - status = ckpt.restore(ckpt_dir_or_file) - status.expect_partial().assert_existing_objects_matched() - else: - assert "Only 'all' or 'backbone' can be used to initialize the model." - - logging.info('Finished loading pretrained checkpoint from %s', - ckpt_dir_or_file) - - -if __name__ == '__main__': - import matplotlib.pyplot as plt - from yolo.utils.run_utils import prep_gpu - prep_gpu() - - config = exp_cfg.YoloTask(model=exp_cfg.Yolo(base='v3')) - task = YoloTask(config) - model = task.build_model() - model.summary() - task.initialize(model) - - train_data = task.build_inputs(config.train_data) - # test_data = task.build_inputs(config.task.validation_data) - - for l, (i, j) in enumerate(train_data): - preds = model(i, training=False) - boxes = xcycwh_to_yxyx(j['bbox']) - - i = tf.image.draw_bounding_boxes(i, boxes, [[1.0, 0.0, 0.0]]) - - i = tf.image.draw_bounding_boxes(i, preds['bbox'], [[0.0, 1.0, 0.0]]) - plt.imshow(i[0].numpy()) - plt.show() - - if l > 2: - break diff --git a/centernet/tasks/centernet_object_detection_test.py b/centernet/tasks/centernet_object_detection_test.py deleted file mode 100644 index c94df866e..000000000 --- a/centernet/tasks/centernet_object_detection_test.py +++ /dev/null @@ -1,195 +0,0 @@ -import numpy as np -import tensorflow as tf - -import centernet.ops.loss_ops as utils -import centernet.tasks as tasks -from centernet.losses import (l1_localization_loss, - penalty_reduced_logistic_focal_loss) -from centernet.tasks.centernet import CenterNetTask - - -def gaussian2D(shape, sigma=1): - m, n = [(ss - 1.) / 2. for ss in shape] - y, x = np.ogrid[-m:m+1,-n:n+1] - - h = np.exp(-(x * x + y * y) / (2 * sigma * sigma)) - h[h < np.finfo(h.dtype).eps * h.max()] = 0 - return h - -def draw_gaussian(heatmap, center, radius, k=1, delte=6): - diameter = 2 * radius + 1 - gaussian = gaussian2D((diameter, diameter), sigma=diameter / delte) - - x, y = center - - height, width = heatmap.shape[0:2] - - left, right = min(x, radius), min(width - x, radius + 1) - top, bottom = min(y, radius), min(height - y, radius + 1) - - masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] - masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] - np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - -def gaussian_radius(det_size, min_overlap): - height, width = det_size - - a1 = 1 - b1 = (height + width) - c1 = width * height * (1 - min_overlap) / (1 + min_overlap) - sq1 = np.sqrt(b1 ** 2 - 4 * a1 * c1) - r1 = (b1 + sq1) / 2 - - a2 = 4 - b2 = 2 * (height + width) - c2 = (1 - min_overlap) * width * height - sq2 = np.sqrt(b2 ** 2 - 4 * a2 * c2) - r2 = (b2 + sq2) / 2 - - a3 = 4 * min_overlap - b3 = -2 * min_overlap * (height + width) - c3 = (min_overlap - 1) * width * height - sq3 = np.sqrt(b3 ** 2 - 4 * a3 * c3) - r3 = (b3 + sq3) / 2 - return min(r1, r2, r3) - -def generate_heatmaps(batch_size, categories, output_size, detections, gaussian_iou=0.7): - - max_tag_len = 1 - - tl_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) - br_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) - ct_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) - - tl_regrs = np.zeros((max_tag_len, 2), dtype=np.float32) - br_regrs = np.zeros((max_tag_len, 2), dtype=np.float32) - ct_regrs = np.zeros((max_tag_len, 2), dtype=np.float32) - tl_tags = np.zeros((max_tag_len), dtype=np.int64) - br_tags = np.zeros((max_tag_len), dtype=np.int64) - ct_tags = np.zeros((max_tag_len), dtype=np.int64) - tag_lens = np.zeros((batch_size, ), dtype=np.int32) - - width_ratio = 1 - height_ratio = 1 - - for b_ind, detection_batches in enumerate(detections): - for ind, detection in enumerate(detection_batches): - category = int(detection[-1]) - #category = 0 - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] - xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. - - fxtl = (xtl * width_ratio) - fytl = (ytl * height_ratio) - fxbr = (xbr * width_ratio) - fybr = (ybr * height_ratio) - fxct = (xct * width_ratio) - fyct = (yct * height_ratio) - - xtl = int(xtl) - ytl = int(ytl) - xbr = int(xbr) - ybr = int(ybr) - xct = int(xct) - yct = int(yct) - - width = detection[2] - detection[0] - height = detection[3] - detection[1] - - radius = gaussian_radius((height, width), gaussian_iou) - radius = max(0, int(radius)) - - draw_gaussian(tl_heatmaps[b_ind, category], [xtl, ytl], radius) - draw_gaussian(br_heatmaps[b_ind, category], [xbr, ybr], radius) - draw_gaussian(ct_heatmaps[b_ind, category], [xct, yct], radius, delte = 5) - - tl_regrs[ind, :] = [fxtl - xtl, fytl - ytl] - br_regrs[ind, :] = [fxbr - xbr, fybr - ybr] - ct_regrs[ind, :] = [fxct - xct, fyct - yct] - tl_tags[ind] = ytl * output_size[1] + xtl - br_tags[ind] = ybr * output_size[1] + xbr - ct_tags[ind] = yct * output_size[1] + xct - - ct_regrs = tf.convert_to_tensor(ct_regrs, dtype=np.float32) - ct_tags = tf.convert_to_tensor(ct_tags, dtype=np.int64) - ct_heatmaps = tf.convert_to_tensor(ct_heatmaps, dtype=np.float32) - - return tl_heatmaps, br_heatmaps, ct_heatmaps, tl_regrs, br_regrs, ct_regrs, tl_tags, br_tags, ct_tags - -class ObjectDetectionTest(tf.test.TestCase): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.actual = tf.constant([[ - (10, 25, 17, 18, 0) - ]], dtype = tf.float32) - - self.predicted = tf.constant([[ - (10, 30, 15, 17, 0) - ]], dtype = tf.float32) - - def test_generate_heatmaps(self): - labels = dict() - outputs = dict() - - tl_heatmaps, br_heatmaps, ct_heatmaps, tl_regrs, br_regrs, ct_regrs, tl_tags, br_tags, ct_tags = generate_heatmaps(1, 2, (416, 416), self.predicted) - ct_heatmaps = tf.reshape(ct_heatmaps, [1, 416, 416, 2]) - - tl_labels_heatmaps, br_labels_heatmaps, ct_labels_heatmaps, tl_regrs_labels, br_regrs_labels, ct_regrs_labels, tl_tags_labels, br_tags_labels, ct_tags_labels = generate_heatmaps(1, 2, (416, 416), self.actual) - ct_labels_heatmaps = tf.reshape(ct_labels_heatmaps, [1, 416, 416, 2]) - - tag_masks = [[[True]]] - - labels = { - 'tl_size': tl_tags_labels, - 'br_size': br_tags_labels, - 'ct_size': ct_tags_labels, - 'tl_heatmaps': tl_labels_heatmaps, - 'br_heatmaps': br_labels_heatmaps, - 'ct_heatmaps': ct_labels_heatmaps, - 'tag_masks': tag_masks, - 'tl_offset': tl_regrs_labels, - 'br_offset': br_regrs_labels, - 'ct_offset': ct_regrs_labels, - } - - outputs = { - 'tl_size': tl_tags, - 'br_size': br_tags, - 'ct_size': ct_tags, - 'tl_heatmaps': tl_heatmaps, - 'br_heatmaps': br_heatmaps, - 'ct_heatmaps': ct_heatmaps, - 'tag_masks': tag_masks, - 'tl_offset': tl_regrs, - 'br_offset': br_regrs, - 'ct_offset': ct_regrs, - } - - task = CenterNetTask(None) - loss, metric = task.build_losses(outputs, labels) - - pass - -if __name__ == '__main__': - ''' - # This code is for visualization - import matplotlib.pyplot as plt - detections = [[ - (10, 300, 15, 370, 0), - (100, 300, 150, 370, 0), - (200, 100, 15, 170, 0), - ], - # more images can go here if you like - ] - tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) - # ct_heatmaps[batch_id, class_id, ...] - - plt.imshow(ct_heatmaps[0, 0, ...]) - plt.show() - # This is to run the test - #plt.imshow(ct_heatmaps[0, 0, ...]) - #plt.imshow(labels_heatmaps[0, 0, ...]) - #plt.show() - ''' - tf.test.main() diff --git a/centernet/tasks/show_heatmaps.py b/centernet/tasks/show_heatmaps.py deleted file mode 100644 index 6a138d686..000000000 --- a/centernet/tasks/show_heatmaps.py +++ /dev/null @@ -1,109 +0,0 @@ -import numpy as np -import tensorflow as tf - -import centernet.tasks as tasks -import centernet.utils as utils - - -def gaussian2D(shape, sigma=1): - m, n = [(ss - 1.) / 2. for ss in shape] - y, x = np.ogrid[-m:m+1,-n:n+1] - - h = np.exp(-(x * x + y * y) / (2 * sigma * sigma)) - h[h < np.finfo(h.dtype).eps * h.max()] = 0 - return h - -def draw_gaussian(heatmap, center, radius, k=1, delte=6): - diameter = 2 * radius + 1 - gaussian = gaussian2D((diameter, diameter), sigma=diameter / delte) - - x, y = center - - height, width = heatmap.shape[0:2] - - left, right = min(x, radius), min(width - x, radius + 1) - top, bottom = min(y, radius), min(height - y, radius + 1) - - masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] - masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] - np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) - -def gaussian_radius(det_size, min_overlap): - height, width = det_size - - a1 = 1 - b1 = (height + width) - c1 = width * height * (1 - min_overlap) / (1 + min_overlap) - sq1 = np.sqrt(b1 ** 2 - 4 * a1 * c1) - r1 = (b1 + sq1) / 2 - - a2 = 4 - b2 = 2 * (height + width) - c2 = (1 - min_overlap) * width * height - sq2 = np.sqrt(b2 ** 2 - 4 * a2 * c2) - r2 = (b2 + sq2) / 2 - - a3 = 4 * min_overlap - b3 = -2 * min_overlap * (height + width) - c3 = (min_overlap - 1) * width * height - sq3 = np.sqrt(b3 ** 2 - 4 * a3 * c3) - r3 = (b3 + sq3) / 2 - return min(r1, r2, r3) - -def generate_heatmaps(batch_size, categories, output_size, detections, gaussian_iou=0.7): - tl_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) - br_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) - ct_heatmaps = np.zeros((batch_size, categories, output_size[0], output_size[1]), dtype=np.float32) - - for b_ind, detection_batches in enumerate(detections): - for ind, detection in enumerate(detection_batches): - category = int(detection[-1]) - #category = 0 - - xtl, ytl = detection[0], detection[1] - xbr, ybr = detection[2], detection[3] - xct, yct = (detection[2] + detection[0])/2., (detection[3]+detection[1])/2. - - xtl = int(xtl) - ytl = int(ytl) - xbr = int(xbr) - ybr = int(ybr) - xct = int(xct) - yct = int(yct) - - width = detection[2] - detection[0] - height = detection[3] - detection[1] - - radius = gaussian_radius((height, width), gaussian_iou) - radius = max(0, int(radius)) - - draw_gaussian(tl_heatmaps[b_ind, category], [xtl, ytl], radius) - draw_gaussian(br_heatmaps[b_ind, category], [xbr, ybr], radius) - draw_gaussian(ct_heatmaps[b_ind, category], [xct, yct], radius, delte = 5) - - return tl_heatmaps, br_heatmaps, ct_heatmaps - -class ObjectDetectionTest(tf.test.TestCase): - def generate_heatmaps(self, dectections): - detections = [[ - (10, 30, 15, 17, 0) - ]] - tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) - pass - -if __name__ == '__main__': - # This code is for visualization - import matplotlib.pyplot as plt - detections = [[ - (10, 300, 15, 370, 0), - (100, 300, 150, 370, 0), - (200, 100, 15, 170, 0), - ], - # more images can go here if you like - ] - tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) - # ct_heatmaps[batch_id, class_id, ...] - plt.imshow(ct_heatmaps[0, 0, ...]) - plt.show() - # This is to run the test - # tf.test.main() From aa1d19a7eb2f0301fe2e32a1db81b66475c3bccb Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 20 Apr 2021 21:09:28 -0400 Subject: [PATCH 115/132] trying moving gt builder to loss --- centernet/dataloaders/centernet_input.py | 10 +- centernet/losses/gt_builder.py | 211 +++++++++++++++++++++++ centernet/tasks/centernet.py | 15 +- 3 files changed, 224 insertions(+), 12 deletions(-) create mode 100644 centernet/losses/gt_builder.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index ce85ad65f..d7810d57f 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -380,11 +380,11 @@ def _build_label(self, image, boxes, classes, width, height, info, data, 'num_detections': num_detections } - heatmap_feature_labels = self._build_heatmap_and_regressed_features( - labels, output_size=[self._output_dims, self._output_dims], - input_size=[self._image_h, self._image_w] - ) - labels.update(heatmap_feature_labels) + # heatmap_feature_labels = self._build_heatmap_and_regressed_features( + # labels, output_size=[self._output_dims, self._output_dims], + # input_size=[self._image_h, self._image_w] + # ) + # labels.update(heatmap_feature_labels) return image, labels def postprocess_fn(self, is_training): diff --git a/centernet/losses/gt_builder.py b/centernet/losses/gt_builder.py new file mode 100644 index 000000000..6a6570bad --- /dev/null +++ b/centernet/losses/gt_builder.py @@ -0,0 +1,211 @@ +import tensorflow as tf + + +def _build_heatmap_and_regressed_features(labels, + output_size=[128, 128], + input_size=[512, 512], + dtype=tf.float32, + class_offset=1): + """ Generates the ground truth labels for centernet. + + Ground truth labels are generated by splatting gaussians on heatmaps for + corners and centers. Regressed features (offsets and sizes) are also + generated. + + Args: + labels: A dictionary of COCO ground truth labels with at minimum the following fields: + bbox: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension + corresponds to the top left x, top left y, bottom right x, and + bottom left y coordinates of the bounding box + classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each + box, given in the same order as the boxes + num_detections: A `Tensor` or int that gives the number of objects in the image + output_size: A `list` of length 2 containing the desired output height + and width of the heatmaps + input_size: A `list` of length 2 the expected input height and width of + the image + Returns: + Dictionary of labels with the following fields: + 'tl_heatmaps': A `Tensor` of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the top left location and class of the object + 'br_heatmaps': `Tensor` of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the bottom right location and class of the object + 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], + heatmap with splatted gaussians centered at the positions and channels + corresponding to the center location and class of the object + 'tl_offset': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the top-left + corner of an object. All other entires are 0 + 'br_offset': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the + bottom-right corner of an object. All other entires are 0 + 'ct_offset': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the x-offset and y-offset of the center of + an object. All other entires are 0 + 'size': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the width and height of an object. All + other entires are 0 + 'box_mask': `Tensor` of shape [max_num_instances], where the first + num_boxes entries are 1. All other entires are 0 + 'box_indices': `Tensor` of shape [max_num_instances, 2], where the first + num_boxes entries contain the y-center and x-center of a valid box. + These are used to extract the regressed box features from the + prediction when computing the loss + """ + + # Get relevant bounding box and class information from labels + # only keep the first num_objects boxes and classes + num_objects = labels['num_detections'] + boxes = labels['bbox'][:num_objects] + classes = labels['classes'][:num_objects] - class_offset + + # Compute scaling factors for center/corner positions on heatmap + input_size = tf.cast(input_size, dtype) + output_size = tf.cast(output_size, dtype) + input_h, input_w = input_size[0], input_size[1] + output_h, output_w = output_size[0], output_size[1] + + width_ratio = output_w / input_w + height_ratio = output_h / input_h + + # Original box coordinates + ytl, ybr = boxes[..., 0], boxes[..., 2] + xtl, xbr = boxes[..., 1], boxes[..., 3] + yct = (ytl + ybr) / 2 + xct = (xtl + xbr) / 2 + + # Scaled box coordinates (could be floating point) + fxtl = xtl * width_ratio + fytl = ytl * height_ratio + fxbr = xbr * width_ratio + fybr = ybr * height_ratio + fxct = xct * width_ratio + fyct = yct * height_ratio + + # Floor the scaled box coordinates to be placed on heatmaps + xtl = tf.math.floor(fxtl) + ytl = tf.math.floor(fytl) + xbr = tf.math.floor(fxbr) + ybr = tf.math.floor(fybr) + xct = tf.math.floor(fxct) + yct = tf.math.floor(fyct) + + # Offset computations to make up for discretization error + # used for offset maps + # tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) + # br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) + ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) + + # Get the scaled box dimensions for computing the gaussian radius + box_widths = boxes[..., 3] - boxes[..., 1] + box_heights = boxes[..., 2] - boxes[..., 0] + + box_widths = box_widths * width_ratio + box_heights = box_heights * height_ratio + + # Used for size map + box_widths_heights = tf.stack([box_widths, box_heights], axis=-1) + + # Center/corner heatmaps + # tl_heatmap = tf.zeros((output_h, output_w, num_classes), dtype) + # br_heatmap = tf.zeros((output_h, output_w, num_classes), dtype) + ct_heatmap = tf.zeros((output_h, output_w, num_classes), dtype) + + # Maps for offset and size features for each instance of a box + # tl_offset = tf.zeros((max_num_instances, 2), dtype) + # br_offset = tf.zeros((max_num_instances, 2), dtype) + ct_offset = tf.zeros((max_num_instances, 2), dtype) + size = tf.zeros((max_num_instances, 2), dtype) + + # Mask for valid box instances and their center indices in the heatmap + box_mask = tf.zeros((max_num_instances), tf.int32) + box_indices = tf.zeros((max_num_instances, 2), tf.int32) + + if use_gaussian_bump: + # Need to gaussians around the centers and corners of the objects + + # First compute the desired gaussian radius + if gaussian_rad == -1: + radius = tf.map_fn(fn=lambda x: preprocessing_ops.gaussian_radius(x), + elems=tf.math.ceil(box_widths_heights)) + radius = tf.math.maximum(tf.math.floor(radius), + tf.cast(1.0, radius.dtype)) + else: + radius = tf.constant([gaussian_rad] * num_objects, dtype) + # These blobs contain information needed to draw the gaussian + # tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) + # br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) + ct_blobs = tf.stack([classes, xct, yct, radius], axis=-1) + + # Get individual gaussian contributions from each bounding box + # tl_gaussians = tf.map_fn( + # fn=lambda x: preprocessing_ops.draw_gaussian( + # tf.shape(tl_heatmap), x, dtype), elems=tl_blobs) + # br_gaussians = tf.map_fn( + # fn=lambda x: preprocessing_ops.draw_gaussian( + # tf.shape(br_heatmap), x, dtype), elems=br_blobs) + ct_gaussians = tf.map_fn( + fn=lambda x: preprocessing_ops.draw_gaussian( + tf.shape(ct_heatmap), x, dtype), elems=ct_blobs) + + # Combine contributions into single heatmaps + # tl_heatmap = tf.math.reduce_max(tl_gaussians, axis=0) + # br_heatmap = tf.math.reduce_max(br_gaussians, axis=0) + ct_heatmap = tf.math.reduce_max(ct_gaussians, axis=0) + + else: + # Instead of a gaussian, insert 1s in the center and corner heatmaps + # tl_hm_update_indices = tf.cast( + # tf.stack([ytl, xtl, classes], axis=-1), tf.int32) + # br_hm_update_indices = tf.cast( + # tf.stack([ybr, xbr, classes], axis=-1), tf.int32) + ct_hm_update_indices = tf.cast( + tf.stack([yct, xct, classes], axis=-1), tf.int32) + + # tl_heatmap = tf.tensor_scatter_nd_update(tl_heatmap, + # tl_hm_update_indices, [1] * num_objects) + # br_heatmap = tf.tensor_scatter_nd_update(br_heatmap, + # br_hm_update_indices, [1] * num_objects) + ct_heatmap = tf.tensor_scatter_nd_update(ct_heatmap, + ct_hm_update_indices, [1] * num_objects) + + # Indices used to update offsets and sizes for valid box instances + update_indices = preprocessing_ops.cartesian_product( + tf.range(num_objects), tf.range(2)) + update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) + + # Write the offsets of each box instance + # tl_offset = tf.tensor_scatter_nd_update( + # tl_offset, update_indices, tl_offset_values) + # br_offset = tf.tensor_scatter_nd_update( + # br_offset, update_indices, br_offset_values) + ct_offset = tf.tensor_scatter_nd_update( + ct_offset, update_indices, ct_offset_values) + + # Write the size of each bounding box + size = tf.tensor_scatter_nd_update(size, update_indices, box_widths_heights) + + # Initially the mask is zeros, so now we unmask each valid box instance + mask_indices = tf.expand_dims(tf.range(num_objects), -1) + mask_values = tf.repeat(1, num_objects) + box_mask = tf.tensor_scatter_nd_update(box_mask, mask_indices, mask_values) + + # Write the y and x coordinate of each box center in the heatmap + box_index_values = tf.cast(tf.stack([yct, xct], axis=-1), dtype=tf.int32) + box_indices = tf.tensor_scatter_nd_update( + box_indices, update_indices, box_index_values) + + labels = { + # 'tl_heatmaps': tl_heatmap, + # 'br_heatmaps': br_heatmap, + 'ct_heatmaps': ct_heatmap, + # 'tl_offset': tl_offset, + # 'br_offset': br_offset, + 'ct_offset': ct_offset, + 'size': size, + 'box_mask': box_mask, + 'box_indices': box_indices + } + return labels diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 6acfa88f0..89e608f1e 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -4,7 +4,7 @@ from centernet.configs import centernet as exp_cfg from centernet.dataloaders import centernet_input -from centernet.losses import (l1_localization_loss, +from centernet.losses import (gt_builder, l1_localization_loss, penalty_reduced_logistic_focal_loss) from centernet.ops import loss_ops from official.core import base_task, input_reader, task_factory @@ -119,21 +119,22 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo loss = 0.0 metric_dict = dict() + gt_label = gt_builder._build_heatmap_and_regressed_features(labels) # Create loss functions object_center_loss_fn = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) # Set up box indices so that they have a batch element as well - box_indices = loss_ops.add_batch_to_indices(labels['box_indices']) + box_indices = loss_ops.add_batch_to_indices(gt_label['box_indices']) - box_mask = tf.cast(labels['box_mask'], dtype=tf.float32) - num_boxes = loss_ops._to_float32(loss_ops.get_num_instances_from_weights(labels['box_mask'])) + box_mask = tf.cast(gt_label['box_mask'], dtype=tf.float32) + num_boxes = loss_ops._to_float32(loss_ops.get_num_instances_from_weights(gt_label['box_mask'])) # Calculate center heatmap loss pred_ct_heatmap_list = outputs['ct_heatmaps'] true_flattened_ct_heatmap = loss_ops._flatten_spatial_dimensions( - labels['ct_heatmaps']) + gt_label['ct_heatmaps']) true_flattened_ct_heatmap = tf.cast(true_flattened_ct_heatmap, tf.float32) total_center_loss = 0.0 @@ -149,7 +150,7 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo # Calculate scale loss pred_scale_list = outputs['ct_size'] - true_scale = labels['size'] + true_scale = gt_label['size'] true_scale = tf.cast(true_scale, tf.float32) total_scale_loss = 0.0 @@ -164,7 +165,7 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo # Calculate offset loss pred_offset_list = outputs['ct_offset'] - true_offset = labels['ct_offset'] + true_offset = gt_label['ct_offset'] true_offset = tf.cast(true_offset, tf.float32) total_offset_loss = 0.0 From 2a3439e17fbc2518ed584f429e5da2ac167d16e0 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 21 Apr 2021 16:39:45 +0000 Subject: [PATCH 116/132] using gt builder in loss --- centernet/losses/gt_builder.py | 14 +++++++++++--- centernet/tasks/centernet.py | 6 +++++- orbit/utils/loop_fns.py | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/centernet/losses/gt_builder.py b/centernet/losses/gt_builder.py index 6a6570bad..eceaa4244 100644 --- a/centernet/losses/gt_builder.py +++ b/centernet/losses/gt_builder.py @@ -1,10 +1,16 @@ import tensorflow as tf +from centernet.ops import preprocessing_ops def _build_heatmap_and_regressed_features(labels, output_size=[128, 128], input_size=[512, 512], dtype=tf.float32, + num_classes=90, + max_num_instances=128, + use_gaussian_bump=True, + gaussian_rad=-1, + gaussian_iou=0.7, class_offset=1): """ Generates the ground truth labels for centernet. @@ -58,8 +64,8 @@ def _build_heatmap_and_regressed_features(labels, # Get relevant bounding box and class information from labels # only keep the first num_objects boxes and classes num_objects = labels['num_detections'] - boxes = labels['bbox'][:num_objects] - classes = labels['classes'][:num_objects] - class_offset + boxes = labels['bbox'] + classes = labels['classes'] - class_offset # Compute scaling factors for center/corner positions on heatmap input_size = tf.cast(input_size, dtype) @@ -128,12 +134,14 @@ def _build_heatmap_and_regressed_features(labels, # First compute the desired gaussian radius if gaussian_rad == -1: + print(box_widths_heights) radius = tf.map_fn(fn=lambda x: preprocessing_ops.gaussian_radius(x), elems=tf.math.ceil(box_widths_heights)) + print(radius) radius = tf.math.maximum(tf.math.floor(radius), tf.cast(1.0, radius.dtype)) else: - radius = tf.constant([gaussian_rad] * num_objects, dtype) + radius = tf.constant([gaussian_rad] * max_num_instances, dtype) # These blobs contain information needed to draw the gaussian # tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) # br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 89e608f1e..971508630 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -119,7 +119,9 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo loss = 0.0 metric_dict = dict() - gt_label = gt_builder._build_heatmap_and_regressed_features(labels) + gt_label = tf.vectorized_map( + fn=gt_builder._build_heatmap_and_regressed_features, + elems=labels) # Create loss functions object_center_loss_fn = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) @@ -294,6 +296,8 @@ def aggregate_logs(self, state=None, step_outputs=None): if not state: self.coco_metric.reset_states() state = self.coco_metric + + print(step_outputs) self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], step_outputs[self.coco_metric.name][1]) return state diff --git a/orbit/utils/loop_fns.py b/orbit/utils/loop_fns.py index 559485f37..c9df0af5d 100755 --- a/orbit/utils/loop_fns.py +++ b/orbit/utils/loop_fns.py @@ -74,6 +74,7 @@ def loop_fn(iterator, num_steps, state=None, reduce_fn=None): outputs = step_fn(iterator) if reduce_fn is not None: state = reduce_fn(state, outputs) + print(state) step += 1 return state except (StopIteration, tf.errors.OutOfRangeError): From b461e36dd2d965bdb7c2a8a4c4f2f8f1283b05f5 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 21 Apr 2021 14:28:49 -0400 Subject: [PATCH 117/132] config update --- centernet/configs/centernet.py | 10 ++++++++- .../experiments/centernet-eval-tpu.yaml | 9 ++++++++ centernet/losses/gt_builder.py | 16 +++++++++++--- centernet/tasks/centernet.py | 22 ++++++++++++++++--- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index ad134ecc4..949f179dd 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -142,7 +142,15 @@ class SegmentationLoss(Loss): class Losses(hyperparams.Config): detection: DetectionLoss = DetectionLoss() segmentation: SegmentationLoss = SegmentationLoss() - + image_h: int = 512 + image_w: int = 512 + output_dims: int = 128 + max_num_instances: int = 128 + use_gaussian_bump: bool = True + gaussian_rad: int = -1 + gaussian_iou: float = 0.7 + class_offset: int = 1 + dtype: str = 'float32' @dataclasses.dataclass class CenterNetDecoder(hyperparams.Config): diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index e151a1adf..917198844 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -50,6 +50,15 @@ task: corner_pull_weight: 0.1 corner_push_weight: 0.1 offset_weight: 1.0 + image_h: 512 + image_w: 512 + output_dims: 128 + max_num_instances: 128 + use_gaussian_bump: true + gaussian_rad: -1 + gaussian_iou: 0.7 + class_offset: 1 + dtype: float32 per_category_metrics: false weight_decay: 0.0005 init_checkpoint: '' diff --git a/centernet/losses/gt_builder.py b/centernet/losses/gt_builder.py index eceaa4244..40e6a23d7 100644 --- a/centernet/losses/gt_builder.py +++ b/centernet/losses/gt_builder.py @@ -2,16 +2,17 @@ from centernet.ops import preprocessing_ops + def _build_heatmap_and_regressed_features(labels, output_size=[128, 128], input_size=[512, 512], - dtype=tf.float32, num_classes=90, max_num_instances=128, use_gaussian_bump=True, gaussian_rad=-1, gaussian_iou=0.7, - class_offset=1): + class_offset=1, + dtype='float32'): """ Generates the ground truth labels for centernet. Ground truth labels are generated by splatting gaussians on heatmaps for @@ -60,7 +61,16 @@ def _build_heatmap_and_regressed_features(labels, These are used to extract the regressed box features from the prediction when computing the loss """ - + if dtype == 'float16': + dtype = tf.float16 + elif dtype == 'bfloat16': + dtype = tf.bfloat16 + elif dtype == 'float32': + dtype = tf.float32 + else: + raise Exception( + 'Unsupported datatype used in ground truth builder only {float16, bfloat16, or float32}' + ) # Get relevant bounding box and class information from labels # only keep the first num_objects boxes and classes num_objects = labels['num_detections'] diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 971508630..45254cee6 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -117,11 +117,27 @@ def get_decoder(self, params): def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_losses=None): total_loss = 0.0 loss = 0.0 + loss_params = self.task_config.losses metric_dict = dict() - gt_label = tf.vectorized_map( - fn=gt_builder._build_heatmap_and_regressed_features, - elems=labels) + # gt_label = tf.vectorized_map( + # fn=gt_builder._build_heatmap_and_regressed_features, + # elems=labels) + + gt_label = tf.map_fn( + fn=lambda x: gt_builder._build_heatmap_and_regressed_features( + labels=x, + output_size=[loss_params.image_h, loss_params.image_w], + input_size=[loss_params.output_dims, loss_params.output_dims], + num_classes=self.task_config.model.num_classes, + max_num_instances=loss_params.max_num_instances, + use_gaussian_bump=loss_params.use_gaussian_bump, + gaussian_rad=loss_params.gaussian_rad, + gaussian_iou=loss_params.gaussian_iou, + class_offset=loss_params.class_offset, + dtype=loss_params.dtype), + elems=labels + ) # Create loss functions object_center_loss_fn = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) From 70215ac66fd22add25092e62e70f00cf1885ca5c Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 21 Apr 2021 19:59:58 +0000 Subject: [PATCH 118/132] eval runs now, debugging detgen --- .centernet-eval.log.swp | Bin 0 -> 16384 bytes .../experiments/centernet-eval-tpu.yaml | 2 +- centernet/tasks/centernet.py | 20 ++++++++++++------ orbit/utils/loop_fns.py | 1 - 4 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 .centernet-eval.log.swp diff --git a/.centernet-eval.log.swp b/.centernet-eval.log.swp new file mode 100644 index 0000000000000000000000000000000000000000..20fb4afe3a63ca48d03a9ae3cd989d14dab46112 GIT binary patch literal 16384 zcmeHONslB)6;7{XgAIsPVv!0XR5Qk1U0FS~jQ|Tg@`!`60gpi%SwU20R8>r8W~3rA zdzq#^Hx4XfyR(Sp2iQg!?tl@?6|soz&LRURFz~&|$m*<`83~6F7&%+|+*R>ndGW;; z@m{!NX8p(B_fTij3j_STDhPgh;p#^(cJ2zEzdHybo(>9;8Kvim+NTQF|4}i|<@HT# zrBAGPxGna0!U|yunJu;-wF5nffgVz^NX&L5vO?%g6kAlUm#D>BXYH4kftG<6$iUsf z&LHelqqp7j*3N5QeenghZ>zTqv<$Qiv<$Qiv<$Qiv<$Qiv<&<|Gf?EGf~%Oo&2AF) z@0;BB`u9KG&podEmJRzqHtcWTus`qY_q*~78}`3D`;jaE_=f#&&VE>zJI8AOYJV;} z`;`s5D{gdH(OELw_aH&-$%P}c3<*XgG8g1}aoaU<-9;xMNV?^l^P zMvbgYS(I>NghA=Fk0K_syeycJSE%BpVK}(2WB;@BIIMG-;5CBnD&+?(7KMlksWNIF z_Njd?_%0J#E2OKDGbHs0J200rM9^{D;D5+jOuwto&S)#n( z3sEXn*(^Fs70G+1=5r%x-c&1glkDQT1}}C7Lm?m{@>YGGj3> zI7*34A|U4l4cuLrbKBb|&p!s+k%|=hijdQj-<6B{^3685TEgOwpEegSE}c7bC7Q znq8Sgv5}&gu({wxsmWv8?Yd~f5v`}K>(pLr%E&awylWG!#Ci!SNQ;HYgyw}{MwL1e zOs>4Kip#kMV;CE1OZh^ueSEgN%zY>6Ro}0(MQP_zvCK@tv&in-{B=ZF?jgdTq zG}0X1IKixnwWV!~R(n;3jfI;1-WY|dEJ**cv>USOcby$ST7U2Y4D1FFM zQD_-8=Uw$e=P*CS4EsF~Z+doW6^SeZ zRov)_;5wr~Nnvc=@j~SL2(Z@s`ZG)DnUfTvcH6;SfW%Tu&#{rGPCI=nz}7pz3iQ{8(Y>o ztQUtTc>c=}g(*=@xh8?vHtQUt&EdQtFUfgV6-fcs8a%oQkm=-Ra6Kz!6NN%nE@)D; zhag?NMJ`ooSTzHg3S>6*TVC38qW%WwU-9tH_+SoRu~!!bdv6X#KiaPIAEI%?Al>wR%w( zSZccJ;HaTH8!T=UQu2$7jm`FQE%zv+tSZvrt`beA(L;niMr%b*)1VkTL#JC48lKWd z)U~eT>%P*sjj^RNMWaIsOHfKI4$#;picn;AeG-f(bf~$Zsk@vRml;w1Dpax8c(N1Ns9x?*H}Q&wqvc|963} z0-peuz{9{6@LJ%xR|LV&fNud`1U?2tz zfHwg*1HZX52)++|9rzS*8TbIO4ZI%s%N;@R3*g(pmw_h$0XzWQ3)}(x@l+7}7quv%oiiPXoKa2Z28D2H>x^1;HqTq!Z2j=z#rOYdT95vl44kI0$mW^p<%efFbbz1h zbO!xVaJsSKalW1DY<2izftxy=srXklCb&VjF)W7M( zzvg{%V@|t7+n|ris6rCjM%g_^WB+ zucnE=nkN2gn)s_}?60P=znaGWY8v~iY3#42vA>$e{%RWgt7+`7rm?@8M*eCV`KxK< yucnc|nnwO=8u_bf 500500 * 64/16 - validation_steps: 625 # 5063 #625 + validation_steps: -1 # 5063 #625 steps_per_loop: 10000 validation_interval: 1 summary_interval: 10000 diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 45254cee6..122e06cfa 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -120,9 +120,11 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo loss_params = self.task_config.losses metric_dict = dict() - # gt_label = tf.vectorized_map( - # fn=gt_builder._build_heatmap_and_regressed_features, - # elems=labels) + + # TODO: actually compute loss + # returning 0 for now, just trying to run eval + metric_dict['total_loss'] = total_loss + return total_loss, metric_dict gt_label = tf.map_fn( fn=lambda x: gt_builder._build_heatmap_and_regressed_features( @@ -136,9 +138,16 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo gaussian_iou=loss_params.gaussian_iou, class_offset=loss_params.class_offset, dtype=loss_params.dtype), - elems=labels + elems=labels, + dtype={ + 'ct_heatmaps': tf.float32, + 'ct_offset': tf.float32, + 'size': tf.float32, + 'box_mask': tf.int32, + 'box_indices': tf.int32 + } ) - + # Create loss functions object_center_loss_fn = penalty_reduced_logistic_focal_loss.PenaltyReducedLogisticFocalLoss(reduction=tf.keras.losses.Reduction.NONE) localization_loss_fn = l1_localization_loss.L1LocalizationLoss(reduction=tf.keras.losses.Reduction.NONE) @@ -313,7 +322,6 @@ def aggregate_logs(self, state=None, step_outputs=None): self.coco_metric.reset_states() state = self.coco_metric - print(step_outputs) self.coco_metric.update_state(step_outputs[self.coco_metric.name][0], step_outputs[self.coco_metric.name][1]) return state diff --git a/orbit/utils/loop_fns.py b/orbit/utils/loop_fns.py index c9df0af5d..559485f37 100755 --- a/orbit/utils/loop_fns.py +++ b/orbit/utils/loop_fns.py @@ -74,7 +74,6 @@ def loop_fn(iterator, num_steps, state=None, reduce_fn=None): outputs = step_fn(iterator) if reduce_fn is not None: state = reduce_fn(state, outputs) - print(state) step += 1 return state except (StopIteration, tf.errors.OutOfRangeError): From 21dc2c121040f8de1bf0b8a81ea1c48fca4e793a Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 26 Apr 2021 00:13:34 -0400 Subject: [PATCH 119/132] issue likely not in det generator --- centernet/configs/centernet.py | 2 +- .../modeling/layers/detection_generator.py | 26 +++++++++++++------ centernet/modeling/layers/nn_blocks.py | 2 +- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 949f179dd..4f89583cd 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -161,7 +161,7 @@ class CenterNetLayer(hyperparams.Config): max_detections: int = 100 peak_error: float = 1e-6 peak_extract_kernel_size: int = 3 - use_nms: bool = False + use_nms: bool = True center_thresh: float = 0.1 iou_thresh: float = 0.4 class_offset: int = 1 diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 8faa6ccc6..ccaaaca32 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -257,17 +257,20 @@ class prediction for each box. # TF Lite does not support tf.gather with batch_dims > 0, so we need to use # tf_gather_nd instead and here we prepare the indices for that. + # shapes of heatmap output + shape = tf.shape(height_width_predictions) + height, width = shape[1], shape[2] + # combined indices dtype=int32 combined_indices = tf.stack([ self.multi_range(batch_size, value_repetitions=num_boxes), tf.reshape(y_indices, [-1]), tf.reshape(x_indices, [-1]) ], axis=1) - new_height_width = tf.gather_nd(height_width_predictions, combined_indices) - new_height_width = tf.reshape(new_height_width, [batch_size, num_boxes, -1]) - height_width = tf.maximum(new_height_width, 0) + new_height_width = tf.reshape(new_height_width, [batch_size, num_boxes, 2]) + height_width = tf.maximum(new_height_width, 0.0) # height and widths dtype=float32 heights = height_width[..., 0] @@ -275,7 +278,7 @@ class prediction for each box. # Get the offsets of center points new_offsets = tf.gather_nd(offset_predictions, combined_indices) - offsets = tf.reshape(new_offsets, [batch_size, num_boxes, -1]) + offsets = tf.reshape(new_offsets, [batch_size, num_boxes, 2]) # offsets are dtype=float32 y_offsets = offsets[..., 0] @@ -287,11 +290,17 @@ class prediction for each box. detection_classes = channel_indices + self._class_offset num_detections = tf.reduce_sum(tf.cast(detection_scores > 0, dtype=tf.int32), axis=1) + + ymin = y_indices + y_offsets - heights / 2.0 + xmin = x_indices + x_offsets - widths / 2.0 + ymax = y_indices + y_offsets + heights / 2.0 + xmax = x_indices + x_offsets + widths / 2.0 - boxes = tf.stack([y_indices + y_offsets - heights / 2.0, - x_indices + x_offsets - widths / 2.0, - y_indices + y_offsets + heights / 2.0, - x_indices + x_offsets + widths / 2.0], axis=2) + ymin = tf.clip_by_value(ymin, 0., tf.cast(height, ymin.dtype)) + xmin = tf.clip_by_value(xmin, 0., tf.cast(width, xmin.dtype)) + ymax = tf.clip_by_value(ymax, 0., tf.cast(height, ymax.dtype)) + xmax = tf.clip_by_value(xmax, 0., tf.cast(width, xmax.dtype)) + boxes = tf.stack([ymin, xmin, ymax, xmax], axis=2) return boxes, detection_classes, detection_scores, num_detections @@ -329,6 +338,7 @@ def call(self, inputs): # Normalize bounding boxes boxes = boxes / tf.cast(height, boxes.dtype) + boxes = tf.clip_by_value(boxes, 0.0, 1.0) # Apply nms if self._use_nms: diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index c0495064a..5d60e6f1a 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -351,7 +351,7 @@ def build(self, input_shape): # Initialize bias to the last Conv2D Layer self.conv2 = tf.keras.layers.Conv2D(filters=self._output_filters, - kernel_size=(1, 1), + kernel_size=(1, 1), padding='valid', bias_initializer=tf.constant_initializer(self._bias_init)) def call(self, x): From 8f69cd2d686a73e62764450f6d1128a992469e08 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 26 Apr 2021 12:28:17 -0400 Subject: [PATCH 120/132] tested odapi conv block padding --- centernet/modeling/backbones/hourglass.py | 2 +- .../modeling/layers/detection_generator.py | 23 +- centernet/modeling/layers/nn_blocks.py | 293 +++++++++++++----- 3 files changed, 226 insertions(+), 92 deletions(-) diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index ceaa3fc72..dd178d7ee 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -52,7 +52,7 @@ def __init__( filters=input_channel_dims, kernel_size=prelayer_kernel_size, strides=prelayer_strides, - padding='same', + padding='valid', activation='relu', )(x_inter) x_inter = official_nn_blocks.ResidualBlock( diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index ccaaaca32..437704574 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -223,7 +223,6 @@ def get_boxes(self, channel_indices, height_width_predictions, offset_predictions, - batch_size, num_boxes): """ Organizes prediction information into the final bounding boxes. @@ -242,7 +241,6 @@ def get_boxes(self, containing the object size predictions. offset_predictions: A Tensor with shape [batch_size, height, width, 2] containing the object local offset predictions. - batch_size: An integer for the batch size of the input. num_boxes: The number of boxes. Returns: @@ -259,7 +257,7 @@ class prediction for each box. # shapes of heatmap output shape = tf.shape(height_width_predictions) - height, width = shape[1], shape[2] + batch_size, height, width = shape[0], shape[1], shape[2] # combined indices dtype=int32 combined_indices = tf.stack([ @@ -303,6 +301,15 @@ class prediction for each box. boxes = tf.stack([ymin, xmin, ymax, xmax], axis=2) return boxes, detection_classes, detection_scores, num_detections + + def convert_strided_predictions_to_normalized_boxes(self, + boxes, + stride=4.0, + true_image_dims=512): + boxes = boxes * tf.cast(stride, boxes.dtype) + boxes = boxes / tf.cast(true_image_dims, boxes.dtype) + boxes = tf.clip_by_value(boxes, 0.0, 1.0) + return boxes def call(self, inputs): # Get heatmaps from decoded outputs via final hourglass stack output @@ -310,35 +317,29 @@ def call(self, inputs): all_ct_sizes = inputs['ct_size'] all_ct_offsets = inputs['ct_offset'] - # Heatmaps below are all dtype=float32 ct_heatmaps = all_ct_heatmaps[-1] ct_sizes = all_ct_sizes[-1] ct_offsets = all_ct_offsets[-1] shape = tf.shape(ct_heatmaps) - # Values below from shape array are all dtype=int32 batch_size, height, width, num_channels = shape[0], shape[1], shape[2], shape[3] - # Resulting peaks from process_heatmap are dtype=float32 # Process heatmaps using 3x3 max pool and applying sigmoid peaks = self.process_heatmap(ct_heatmaps, kernel_size=self._peak_extract_kernel_size, center_thresh=self._center_thresh) - # scores are dtype=float32, y, x, and channel indices are dtype=int32 # Get top scores along with their x, y, and class scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, batch_size, width, num_channels, k=self._max_detections) - # boxes and scores are dtype=float32, classes and num_dets are dtype=int32 # Parse the score and indices into bounding boxes boxes, classes, scores, num_det = self.get_boxes(scores, - y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, batch_size, self._max_detections) + y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, self._max_detections) # Normalize bounding boxes - boxes = boxes / tf.cast(height, boxes.dtype) - boxes = tf.clip_by_value(boxes, 0.0, 1.0) + boxes = self.convert_strided_predictions_to_normalized_boxes(boxes) # Apply nms if self._use_nms: diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 5d60e6f1a..db81c3a00 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -17,48 +17,195 @@ def call(self, input): return input +# @tf.keras.utils.register_keras_serializable(package='centernet') +# class ConvBN(tf.keras.layers.Layer): +# """ +# Modified Convolution layer to match that of the DarkNet Library. +# The Layer is a standards combination of Conv BatchNorm Activation, +# however, the use of bias in the conv is determined by the use of batch +# normalization. +# Cross Stage Partial networks (CSPNets) were proposed in: +# [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, +# Ping-Yang Chen, Jun-Wei Hsieh +# CSPNet: A New Backbone that can Enhance Learning Capability of CNN. +# arXiv:1911.11929 +# Args: +# filters: integer for output depth, or the number of features to learn +# kernel_size: integer or tuple for the shape of the weight matrix or kernel +# to learn +# strides: integer of tuple how much to move the kernel after each kernel +# use +# padding: string 'valid' or 'same', if same, then pad the image, else do +# not +# dialtion_rate: tuple to indicate how much to modulate kernel weights and +# how many pixels in a feature map to skip +# kernel_initializer: string to indicate which function to use to initialize +# weights +# bias_initializer: string to indicate which function to use to initialize +# bias +# kernel_regularizer: string to indicate which function to use to +# regularizer weights +# bias_regularizer: string to indicate which function to use to regularizer +# bias +# use_bn: boolean for whether to use batch normalization +# use_sync_bn: boolean for whether sync batch normalization statistics +# of all batch norm layers to the models global statistics +# (across all input batches) +# norm_momentum: float for moment to use for batch normalization +# norm_epsilon: float for batch normalization epsilon +# activation: string or None for activation function to use in layer, +# if None activation is replaced by linear +# leaky_alpha: float to use as alpha if activation function is leaky +# **kwargs: Keyword Arguments +# """ + +# def __init__(self, +# filters=1, +# kernel_size=(1, 1), +# strides=(1, 1), +# padding='same', +# dilation_rate=(1, 1), +# kernel_initializer='glorot_uniform', +# bias_initializer='zeros', +# subdivisions=1, +# bias_regularizer=None, +# kernel_regularizer=None, +# use_bn=True, +# use_sync_bn=False, +# norm_momentum=0.99, +# norm_epsilon=0.001, +# activation='leaky', +# leaky_alpha=0.1, +# **kwargs): + +# # convolution params +# self._filters = filters +# self._kernel_size = kernel_size +# self._strides = strides +# self._padding = padding +# self._dilation_rate = dilation_rate +# self._kernel_initializer = kernel_initializer +# self._bias_initializer = bias_initializer +# self._kernel_regularizer = kernel_regularizer + +# self._bias_regularizer = bias_regularizer +# self._subdivisions = subdivisions + +# # batch normalization params +# self._use_bn = use_bn +# self._use_sync_bn = use_sync_bn +# self._norm_momentum = norm_momentum +# self._norm_epsilon = norm_epsilon + +# if tf.keras.backend.image_data_format() == 'channels_last': +# # format: (batch_size, height, width, channels) +# self._bn_axis = -1 +# else: +# # format: (batch_size, channels, width, height) +# self._bn_axis = 1 + +# # activation params +# self._activation = activation +# self._leaky_alpha = leaky_alpha + +# super().__init__(**kwargs) + +# def build(self, input_shape): +# use_bias = not self._use_bn + +# if not TPU_BASE: +# kernel_size = self._kernel_size if isinstance( +# self._kernel_size, int) else self._kernel_size[0] +# dilation_rate = self._dilation_rate if isinstance( +# self._dilation_rate, int) else self._dilation_rate[0] +# if self._padding == 'same' and kernel_size != 1: +# padding = (dilation_rate * (kernel_size - 1)) +# left_shift = padding // 2 +# self._paddings = tf.constant([[0, 0], [left_shift, left_shift], +# [left_shift, left_shift], [0, 0]]) +# # self._zeropad = tf.keras.layers.ZeroPadding2D() +# else: +# self._paddings = tf.constant([[0, 0], [0, 0], [0, 0], [0, 0]]) + +# self.conv = tf.keras.layers.Conv2D( +# filters=self._filters, +# kernel_size=self._kernel_size, +# strides=self._strides, +# padding='valid', +# dilation_rate=self._dilation_rate, +# use_bias=use_bias, +# kernel_initializer=self._kernel_initializer, +# bias_initializer=self._bias_initializer, +# kernel_regularizer=self._kernel_regularizer, +# bias_regularizer=self._bias_regularizer) +# else: +# self.conv = tf.keras.layers.Conv2D( +# filters=self._filters, +# kernel_size=self._kernel_size, +# strides=self._strides, +# padding=self._padding, # 'valid', +# dilation_rate=self._dilation_rate, +# use_bias=use_bias, +# kernel_initializer=self._kernel_initializer, +# bias_initializer=self._bias_initializer, +# kernel_regularizer=self._kernel_regularizer, +# bias_regularizer=self._bias_regularizer) + +# if self._use_bn: +# if self._use_sync_bn: +# self.bn = subnormalization.SubDivSyncBatchNormalization( +# subdivisions=self._subdivisions, +# momentum=self._norm_momentum, +# epsilon=self._norm_epsilon, +# axis=self._bn_axis) +# else: +# self.bn = subnormalization.SubDivBatchNormalization( +# subdivisions=self._subdivisions, +# momentum=self._norm_momentum, +# epsilon=self._norm_epsilon, +# axis=self._bn_axis) + +# if self._activation == 'leaky': +# self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) +# elif self._activation == 'mish': +# self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) +# else: +# self._activation_fn = tf_utils.get_activation( +# self._activation) # tf.keras.layers.Activation(self._activation) + +# def call(self, x): +# if not TPU_BASE: +# x = tf.pad(x, self._paddings, mode='CONSTANT', constant_values=0) +# x = self.conv(x) +# if self._use_bn: +# x = self.bn(x) +# x = self._activation_fn(x) +# return x + +# def get_config(self): +# # used to store/share parameters to reconstruct the model +# layer_config = { +# 'filters': self._filters, +# 'kernel_size': self._kernel_size, +# 'strides': self._strides, +# 'padding': self._padding, +# 'dilation_rate': self._dilation_rate, +# 'kernel_initializer': self._kernel_initializer, +# 'bias_initializer': self._bias_initializer, +# 'bias_regularizer': self._bias_regularizer, +# 'kernel_regularizer': self._kernel_regularizer, +# 'use_bn': self._use_bn, +# 'use_sync_bn': self._use_sync_bn, +# 'norm_momentum': self._norm_momentum, +# 'norm_epsilon': self._norm_epsilon, +# 'activation': self._activation, +# 'leaky_alpha': self._leaky_alpha +# } +# layer_config.update(super().get_config()) +# return layer_config + @tf.keras.utils.register_keras_serializable(package='centernet') class ConvBN(tf.keras.layers.Layer): - """ - Modified Convolution layer to match that of the DarkNet Library. - The Layer is a standards combination of Conv BatchNorm Activation, - however, the use of bias in the conv is determined by the use of batch - normalization. - Cross Stage Partial networks (CSPNets) were proposed in: - [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, - Ping-Yang Chen, Jun-Wei Hsieh - CSPNet: A New Backbone that can Enhance Learning Capability of CNN. - arXiv:1911.11929 - Args: - filters: integer for output depth, or the number of features to learn - kernel_size: integer or tuple for the shape of the weight matrix or kernel - to learn - strides: integer of tuple how much to move the kernel after each kernel - use - padding: string 'valid' or 'same', if same, then pad the image, else do - not - dialtion_rate: tuple to indicate how much to modulate kernel weights and - how many pixels in a feature map to skip - kernel_initializer: string to indicate which function to use to initialize - weights - bias_initializer: string to indicate which function to use to initialize - bias - kernel_regularizer: string to indicate which function to use to - regularizer weights - bias_regularizer: string to indicate which function to use to regularizer - bias - use_bn: boolean for whether to use batch normalization - use_sync_bn: boolean for whether sync batch normalization statistics - of all batch norm layers to the models global statistics - (across all input batches) - norm_momentum: float for moment to use for batch normalization - norm_epsilon: float for batch normalization epsilon - activation: string or None for activation function to use in layer, - if None activation is replaced by linear - leaky_alpha: float to use as alpha if activation function is leaky - **kwargs: Keyword Arguments - """ - def __init__(self, filters=1, kernel_size=(1, 1), @@ -112,44 +259,33 @@ def __init__(self, def build(self, input_shape): use_bias = not self._use_bn + padding = self._padding - if not TPU_BASE: - kernel_size = self._kernel_size if isinstance( - self._kernel_size, int) else self._kernel_size[0] - dilation_rate = self._dilation_rate if isinstance( - self._dilation_rate, int) else self._dilation_rate[0] - if self._padding == 'same' and kernel_size != 1: - padding = (dilation_rate * (kernel_size - 1)) - left_shift = padding // 2 - self._paddings = tf.constant([[0, 0], [left_shift, left_shift], - [left_shift, left_shift], [0, 0]]) - # self._zeropad = tf.keras.layers.ZeroPadding2D() - else: - self._paddings = tf.constant([[0, 0], [0, 0], [0, 0], [0, 0]]) - - self.conv = tf.keras.layers.Conv2D( - filters=self._filters, - kernel_size=self._kernel_size, - strides=self._strides, - padding='valid', - dilation_rate=self._dilation_rate, - use_bias=use_bias, - kernel_initializer=self._kernel_initializer, - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) + kernel_size = self._kernel_size if isinstance( + self._kernel_size, int) else self._kernel_size[0] + + if kernel_size > 1: + padding = 'valid' + if kernel_size == 7: + padding_size = (3, 3) + if kernel_size == 3: + padding_size = (1, 1) + + self.pad = tf.keras.layers.ZeroPadding2D(padding_size) else: - self.conv = tf.keras.layers.Conv2D( - filters=self._filters, - kernel_size=self._kernel_size, - strides=self._strides, - padding=self._padding, # 'valid', - dilation_rate=self._dilation_rate, - use_bias=use_bias, - kernel_initializer=self._kernel_initializer, - bias_initializer=self._bias_initializer, - kernel_regularizer=self._kernel_regularizer, - bias_regularizer=self._bias_regularizer) + self.pad = Identity() + + self.conv = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=self._kernel_size, + strides=self._strides, + padding=padding, + dilation_rate=self._dilation_rate, + use_bias=use_bias, + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) if self._use_bn: if self._use_sync_bn: @@ -174,8 +310,7 @@ def build(self, input_shape): self._activation) # tf.keras.layers.Activation(self._activation) def call(self, x): - if not TPU_BASE: - x = tf.pad(x, self._paddings, mode='CONSTANT', constant_values=0) + x = self.pad(x) x = self.conv(x) if self._use_bn: x = self.bn(x) @@ -204,8 +339,6 @@ def get_config(self): layer_config.update(super().get_config()) return layer_config - - @tf.keras.utils.register_keras_serializable(package='centernet') class HourglassBlock(tf.keras.layers.Layer): """ From 0cea19cbc86a920a9855bd1095d9147b15a58d42 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 26 Apr 2021 15:51:02 -0400 Subject: [PATCH 121/132] moved box normalization params to config --- centernet/configs/centernet.py | 4 ++-- centernet/modeling/CenterNet.py | 7 +++---- centernet/modeling/backbones/hourglass.py | 1 - centernet/modeling/layers/detection_generator.py | 15 ++++++++++----- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 4f89583cd..defdc6a04 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -165,8 +165,8 @@ class CenterNetLayer(hyperparams.Config): center_thresh: float = 0.1 iou_thresh: float = 0.4 class_offset: int = 1 - dtype: str = 'float32' - use_reduction_sum: bool = True + net_down_scale: int = 4 + input_image_dims: int = 512 @dataclasses.dataclass class CenterNetDetection(hyperparams.Config): diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 90a561a84..288fba536 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -7,7 +7,6 @@ from official.core import registry from official.vision.beta.modeling.backbones import factory -# TODO: import prediction and filtering layers when made class CenterNet(ks.Model): @@ -74,7 +73,9 @@ def build_centernet_filter(model_config): use_nms=model_config.filter.use_nms, center_thresh=model_config.filter.center_thresh, iou_thresh=model_config.filter.iou_thresh, - class_offset=model_config.filter.class_offset) + class_offset=model_config.filter.class_offset, + net_down_scale=model_config.filter.net_down_scale, + input_image_dims=model_config.filter.input_image_dims) def build_centernet_head(model_config): return None @@ -96,7 +97,5 @@ def build_centernet(input_specs, task_config, l2_regularization): model.build(input_specs.shape) model.summary() - # TODO: uncommend when filter is implemented - # losses = filter.losses losses = None return model, losses diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index dd178d7ee..fe3ab316a 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -37,7 +37,6 @@ def __init__( x_inter = input # Create some intermediate and postlayers to generate the heatmaps - # (document and make cleaner later) inp_filters = channel_dims_per_stage[0] # Create prelayers if downsampling input diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 437704574..d4b41176e 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -24,6 +24,8 @@ def __init__(self, center_thresh=0.1, iou_thresh=0.4, class_offset=1, + net_down_scale=4, + input_image_dims=512, **kwargs): """ Args: @@ -42,6 +44,7 @@ def __init__(self, iou_thresh: A float for the threshold IOU when filtering bounding boxes. class_offset: An integer indicating to add an offset to the class prediction if the dataset labels have been shifted. + net_down_scale: An integer that specifies call Returns: Dictionary with keys 'bbox', 'classes', 'confidence', and 'num_dets' @@ -62,6 +65,10 @@ def __init__(self, self._iou_thresh = iou_thresh self._class_offset = class_offset + + # Box normalization parameters + self._net_down_scale = net_down_scale + self._input_image_dims = input_image_dims def process_heatmap(self, feature_map, @@ -303,11 +310,9 @@ class prediction for each box. return boxes, detection_classes, detection_scores, num_detections def convert_strided_predictions_to_normalized_boxes(self, - boxes, - stride=4.0, - true_image_dims=512): - boxes = boxes * tf.cast(stride, boxes.dtype) - boxes = boxes / tf.cast(true_image_dims, boxes.dtype) + boxes): + boxes = boxes * tf.cast(self._net_down_scale, boxes.dtype) + boxes = boxes / tf.cast(self._input_image_dims, boxes.dtype) boxes = tf.clip_by_value(boxes, 0.0, 1.0) return boxes From 58d736f93a955fe9ff1773cc0b1041e02445d4c7 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 28 Apr 2021 02:35:25 -0400 Subject: [PATCH 122/132] new residual block for centernet, ready to eval --- centernet/modeling/backbones/hourglass.py | 70 ++++--- .../modeling/layers/detection_generator.py | 4 +- centernet/modeling/layers/nn_blocks.py | 198 +++++++++++++++++- .../utils/weight_utils/config_classes.py | 78 +++---- centernet/utils/weight_utils/config_data.py | 87 ++++---- centernet/utils/weight_utils/load_weights.py | 10 +- 6 files changed, 317 insertions(+), 130 deletions(-) diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index fe3ab316a..a80a68a15 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -4,11 +4,12 @@ from centernet.configs import backbones as cfg from centernet.modeling.layers import nn_blocks -# from official.vision.beta.modeling.backbones import factory -from official.vision.beta.modeling.layers import \ - nn_blocks as official_nn_blocks +# from official.vision.beta.modeling.layers import \ +# nn_blocks as official_nn_blocks from utils import register +BATCH_NORM_MOMENTUM = 0.1 +BATCH_NORM_EPSILON = 1e-5 @tf.keras.utils.register_keras_serializable(package='centernet') class Hourglass(tf.keras.Model): @@ -34,12 +35,10 @@ def __init__( """ # yapf: disable input = tf.keras.layers.Input(shape=input_specs.shape[1:], name='input') - x_inter = input - # Create some intermediate and postlayers to generate the heatmaps inp_filters = channel_dims_per_stage[0] - # Create prelayers if downsampling input + # Create downsampling layers if initial_downsample: prelayer_kernel_size = 7 prelayer_strides = 2 @@ -47,17 +46,24 @@ def __init__( prelayer_kernel_size = 3 prelayer_strides = 1 - x_inter = nn_blocks.ConvBN( - filters=input_channel_dims, - kernel_size=prelayer_kernel_size, - strides=prelayer_strides, - padding='valid', - activation='relu', - )(x_inter) - x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=True, strides=prelayer_strides, - )(x_inter) - + x_downsampled = nn_blocks.ConvBN(filters=input_channel_dims, + kernel_size=prelayer_kernel_size, + strides=prelayer_strides, + padding='valid', + activation='relu', + use_sync_bn=True, + norm_momentum=BATCH_NORM_MOMENTUM, + norm_epsilon=BATCH_NORM_EPSILON)(input) + + x_downsampled = nn_blocks.CenterNetResidualBlock( + filters=inp_filters, + use_projection=True, + strides=prelayer_strides, + use_sync_bn=True, + norm_momentum=BATCH_NORM_MOMENTUM, + norm_epsilon=BATCH_NORM_EPSILON)(x_downsampled) + + # Used for storing each hourglass heatmap output all_heatmaps = [] for i in range(num_hourglasses): @@ -65,15 +71,18 @@ def __init__( x_hg = nn_blocks.HourglassBlock( channel_dims_per_stage=channel_dims_per_stage, blocks_per_stage=blocks_per_stage, - )(x_inter) + )(x_downsampled) # cnvs x_hg = nn_blocks.ConvBN( filters=inp_filters, kernel_size=(3, 3), strides=(1, 1), - padding='same', + padding='valid', activation='relu', + use_sync_bn=True, + norm_momentum=BATCH_NORM_MOMENTUM, + norm_epsilon=BATCH_NORM_EPSILON )(x_hg) all_heatmaps.append(x_hg) @@ -86,8 +95,11 @@ def __init__( kernel_size=(1, 1), strides=(1, 1), padding='same', - activation='linear' - )(x_inter) + activation='identity', + use_sync_bn=True, + norm_momentum=BATCH_NORM_MOMENTUM, + norm_epsilon=BATCH_NORM_EPSILON + )(x_downsampled) # inters_ inter_hg_conv2 = nn_blocks.ConvBN( @@ -95,16 +107,20 @@ def __init__( kernel_size=(1, 1), strides=(1, 1), padding='same', - activation='linear' + activation='identity', + use_sync_bn=True, + norm_momentum=BATCH_NORM_MOMENTUM, + norm_epsilon=BATCH_NORM_EPSILON )(x_hg) - x_inter = tf.keras.layers.Add()([inter_hg_conv1, inter_hg_conv2]) - x_inter = tf.keras.layers.ReLU()(x_inter) + x_downsampled = tf.keras.layers.Add()([inter_hg_conv1, inter_hg_conv2]) + x_downsampled = tf.keras.layers.ReLU()(x_downsampled) # inters - x_inter = official_nn_blocks.ResidualBlock( - filters=inp_filters, use_projection=False, strides=1 - )(x_inter) + x_downsampled = nn_blocks.CenterNetResidualBlock( + filters=inp_filters, use_projection=False, strides=1, + norm_momentum=BATCH_NORM_MOMENTUM, norm_epsilon=BATCH_NORM_EPSILON + )(x_downsampled) # yapf: enable super().__init__(inputs=input, outputs=all_heatmaps, **kwargs) diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index d4b41176e..8ec651628 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -356,11 +356,11 @@ def call(self, inputs): confidence=scores, k=self._max_detections, limit_pre_thresh=True, pre_nms_thresh=self._center_thresh, nms_thresh=self._iou_thresh) - num_dets = tf.reduce_sum(tf.cast(scores > 0, dtype=tf.int32), axis=1) + num_det = tf.reduce_sum(tf.cast(scores > 0, dtype=tf.int32), axis=1) return { 'bbox': boxes, 'classes': classes, 'confidence': scores, - 'num_dets': num_dets + 'num_dets': num_det } diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index db81c3a00..92cbc3d11 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -5,7 +5,9 @@ from official.vision.beta.modeling.layers import \ nn_blocks as official_nn_blocks -TPU_BASE = True +TPU_BASE = False +BATCH_NORM_MOMENTUM = 0.1 +BATCH_NORM_EPSILON = 1e-5 @tf.keras.utils.register_keras_serializable(package='centernet') class Identity(tf.keras.layers.Layer): @@ -204,6 +206,182 @@ def call(self, input): # layer_config.update(super().get_config()) # return layer_config +@tf.keras.utils.register_keras_serializable(package='centernet') +class CenterNetResidualBlock(tf.keras.layers.Layer): + """A residual block.""" + + def __init__(self, + filters, + strides, + use_projection=False, + se_ratio=None, + resnetd_shortcut=False, + stochastic_depth_drop_rate=None, + kernel_initializer='VarianceScaling', + kernel_regularizer=None, + bias_regularizer=None, + activation='relu', + use_sync_bn=False, + norm_momentum=0.99, + norm_epsilon=0.001, + **kwargs): + """A residual block with BN after convolutions. + + Args: + filters: `int` number of filters for the first two convolutions. Note that + the third and final convolution will use 4 times as many filters. + strides: `int` block stride. If greater than 1, this block will ultimately + downsample the input. + use_projection: `bool` for whether this block should use a projection + shortcut (versus the default identity shortcut). This is usually `True` + for the first block of a block group, which may change the number of + filters and the resolution. + se_ratio: `float` or None. Ratio of the Squeeze-and-Excitation layer. + resnetd_shortcut: `bool` if True, apply the resnetd style modification to + the shortcut connection. Not implemented in residual blocks. + stochastic_depth_drop_rate: `float` or None. if not None, drop rate for + the stochastic depth layer. + kernel_initializer: kernel_initializer for convolutional layers. + kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D. + Default to None. + bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d. + Default to None. + activation: `str` name of the activation function. + use_sync_bn: if True, use synchronized batch normalization. + norm_momentum: `float` normalization omentum for the moving average. + norm_epsilon: `float` small float added to variance to avoid dividing by + zero. + **kwargs: keyword arguments to be passed. + """ + super(CenterNetResidualBlock, self).__init__(**kwargs) + + self._filters = filters + self._strides = strides + self._use_projection = use_projection + self._se_ratio = se_ratio + self._resnetd_shortcut = resnetd_shortcut + self._use_sync_bn = use_sync_bn + self._activation = activation + self._stochastic_depth_drop_rate = stochastic_depth_drop_rate + self._kernel_initializer = kernel_initializer + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon + self._kernel_regularizer = kernel_regularizer + self._bias_regularizer = bias_regularizer + + if use_sync_bn: + self._norm = tf.keras.layers.experimental.SyncBatchNormalization + else: + self._norm = tf.keras.layers.BatchNormalization + if tf.keras.backend.image_data_format() == 'channels_last': + self._bn_axis = -1 + else: + self._bn_axis = 1 + self._activation_fn = tf_utils.get_activation(activation) + + def build(self, input_shape): + if self._use_projection: + self._shortcut = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=1, + strides=self._strides, + use_bias=False, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + self._norm0 = self._norm( + axis=self._bn_axis, + momentum=self._norm_momentum, + epsilon=self._norm_epsilon) + self.pad = tf.keras.layers.ZeroPadding2D(padding=(1, 1)) + self._conv1 = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=3, + strides=self._strides, + padding='valid', + use_bias=False, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + self._norm1 = self._norm( + axis=self._bn_axis, + momentum=self._norm_momentum, + epsilon=self._norm_epsilon) + + self._conv2 = tf.keras.layers.Conv2D( + filters=self._filters, + kernel_size=3, + strides=1, + padding='same', + use_bias=False, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + self._norm2 = self._norm( + axis=self._bn_axis, + momentum=self._norm_momentum, + epsilon=self._norm_epsilon) + + if self._se_ratio and self._se_ratio > 0 and self._se_ratio <= 1: + self._squeeze_excitation = nn_layers.SqueezeExcitation( + in_filters=self._filters, + out_filters=self._filters, + se_ratio=self._se_ratio, + kernel_initializer=self._kernel_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer) + else: + self._squeeze_excitation = None + + if self._stochastic_depth_drop_rate: + self._stochastic_depth = nn_layers.StochasticDepth( + self._stochastic_depth_drop_rate) + else: + self._stochastic_depth = None + + super(CenterNetResidualBlock, self).build(input_shape) + + def get_config(self): + config = { + 'filters': self._filters, + 'strides': self._strides, + 'use_projection': self._use_projection, + 'se_ratio': self._se_ratio, + 'resnetd_shortcut': self._resnetd_shortcut, + 'stochastic_depth_drop_rate': self._stochastic_depth_drop_rate, + 'kernel_initializer': self._kernel_initializer, + 'kernel_regularizer': self._kernel_regularizer, + 'bias_regularizer': self._bias_regularizer, + 'activation': self._activation, + 'use_sync_bn': self._use_sync_bn, + 'norm_momentum': self._norm_momentum, + 'norm_epsilon': self._norm_epsilon + } + base_config = super(CenterNetResidualBlock, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def call(self, inputs, training=None): + shortcut = inputs + if self._use_projection: + shortcut = self._shortcut(shortcut) + shortcut = self._norm0(shortcut) + + x = self.pad(inputs) + x = self._conv1(x) + x = self._norm1(x) + x = self._activation_fn(x) + + x = self._conv2(x) + x = self._norm2(x) + + if self._squeeze_excitation: + x = self._squeeze_excitation(x) + + if self._stochastic_depth: + x = self._stochastic_depth(x, training=training) + + return self._activation_fn(x + shortcut) + @tf.keras.utils.register_keras_serializable(package='centernet') class ConvBN(tf.keras.layers.Layer): def __init__(self, @@ -289,14 +467,12 @@ def build(self, input_shape): if self._use_bn: if self._use_sync_bn: - self.bn = subnormalization.SubDivSyncBatchNormalization( - subdivisions=self._subdivisions, + self.bn = tf.keras.layers.experimental.SyncBatchNormalization( momentum=self._norm_momentum, epsilon=self._norm_epsilon, axis=self._bn_axis) else: - self.bn = subnormalization.SubDivBatchNormalization( - subdivisions=self._subdivisions, + self.bn = tf.keras.layers.BatchNormalization( momentum=self._norm_momentum, epsilon=self._norm_epsilon, axis=self._bn_axis) @@ -383,9 +559,10 @@ def make_repeated_residual_blocks(self, reps, out_channels, stride = initial_stride if i == 0 else 1 skip_conv = stride > 1 - blocks.append(official_nn_blocks.ResidualBlock( + blocks.append(CenterNetResidualBlock( filters=residual_channels, strides=stride, - use_projection=skip_conv)) + use_projection=skip_conv, use_sync_bn=True, + norm_momentum=BATCH_NORM_MOMENTUM, norm_epsilon=BATCH_NORM_EPSILON)) if reps == 1: stride = initial_stride @@ -394,9 +571,10 @@ def make_repeated_residual_blocks(self, reps, out_channels, stride = 1 skip_conv = residual_channels != out_channels - blocks.append(official_nn_blocks.ResidualBlock( + blocks.append(CenterNetResidualBlock( filters=out_channels, strides=stride, - use_projection=skip_conv)) + use_projection=skip_conv, use_sync_bn=True, + norm_momentum=BATCH_NORM_MOMENTUM, norm_epsilon=BATCH_NORM_EPSILON)) return tf.keras.Sequential(blocks) @@ -414,8 +592,6 @@ def build(self, input_shape): self.encoder_block2 = self.make_repeated_residual_blocks(reps=self._reps, out_channels=self._filters_downsampled, initial_stride=2, initial_skip=self._filters != self._filters_downsampled) - # self.pool = tf.keras.layers.MaxPool2D(pool_size=2) - # recursively define inner hourglasses self.inner_hg = type(self)( channel_dims_per_stage=self._channel_dims_per_stage[1:], diff --git a/centernet/utils/weight_utils/config_classes.py b/centernet/utils/weight_utils/config_classes.py index 1ffc5a565..f9ee7c286 100644 --- a/centernet/utils/weight_utils/config_classes.py +++ b/centernet/utils/weight_utils/config_classes.py @@ -6,18 +6,17 @@ class Config(ABC): - def get_weights(self): - return None - - def load_weights(self, layer): - weights = self.get_weights() - layer.set_weights(weights) - n_weights = 0 - - for w in weights: - n_weights += w.size - return n_weights + def get_weights(self): + return None + + def load_weights(self, layer): + weights = self.get_weights() + layer.set_weights(weights) + n_weights = 0 + for w in weights: + n_weights += w.size + return n_weights @dataclass class convBnCFG(Config): @@ -96,28 +95,28 @@ def __post_init__(self): def get_weights(self): weights = [ - self.skip_weights, - self.skip_gamma, - self.skip_beta, - - self.conv_block_weights, - self.conv_block_gamma, - self.conv_block_beta, - - self.conv_weights, - self.norm_gamma, - self.norm_beta, - - self.skip_moving_mean, - self.skip_moving_variance, - self.conv_block_moving_mean, - self.conv_block_moving_variance, - self.norm_moving_mean, - self.norm_moving_variance, - ] + self.skip_weights, + self.skip_gamma, + self.skip_beta, + + self.conv_block_weights, + self.conv_block_gamma, + self.conv_block_beta, + + self.conv_weights, + self.norm_gamma, + self.norm_beta, + + self.skip_moving_mean, + self.skip_moving_variance, + self.conv_block_moving_mean, + self.conv_block_moving_variance, + self.norm_moving_mean, + self.norm_moving_variance, + ] weights = [x for x in weights if x is not None] - return weights + return weights @dataclass class hourglassCFG(Config): @@ -151,13 +150,16 @@ def load_weights(self, layer): n_weights = 0 if not self.is_last_stage: - enc_dec_layers = [layer.submodules[0], - layer.submodules[1], - layer.submodules[3]] - enc_dec_weight_dicts = [self.weights_dict['encoder_block1'], - self.weights_dict['encoder_block2'], - self.weights_dict['decoder_block']] - + enc_dec_layers = [ + layer.submodules[0], + layer.submodules[1], + layer.submodules[3] + ] + enc_dec_weight_dicts = [ + self.weights_dict['encoder_block1'], + self.weights_dict['encoder_block2'], + self.weights_dict['decoder_block'] + ] for l, weights_dict in zip(enc_dec_layers, enc_dec_weight_dicts): n_weights += self.load_block_weights(l, weights_dict) diff --git a/centernet/utils/weight_utils/config_data.py b/centernet/utils/weight_utils/config_data.py index de3f82c42..bf12de17c 100644 --- a/centernet/utils/weight_utils/config_data.py +++ b/centernet/utils/weight_utils/config_data.py @@ -11,59 +11,52 @@ class BackboneConfigData(): weights_dict: Dict = field(repr=False, default=None) - def __post_init__(self): - self.hourglass104_512 = [ - # Downsampling Layers - convBnCFG(weights_dict=self.weights_dict['downsample_input']['conv_block']), - residualBlockCFG(weights_dict=self.weights_dict['downsample_input']['residual_block']), - # Hourglass - hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['0']), - convBnCFG(weights_dict=self.weights_dict['output_conv']['0']), - # Intermediate - convBnCFG(weights_dict=self.weights_dict['intermediate_conv1']['0']), - convBnCFG(weights_dict=self.weights_dict['intermediate_conv2']['0']), - residualBlockCFG(weights_dict=self.weights_dict['intermediate_residual']['0']), - # Hourglass - hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['1']), - convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), - ] - - self.extremenet = [ - # Downsampling Layers - convBnCFG(weights_dict=self.weights_dict['downsample_input']['conv_block']), - residualBlockCFG(weights_dict=self.weights_dict['downsample_input']['residual_block']), - # Hourglass - hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['0']), - convBnCFG(weights_dict=self.weights_dict['output_conv']['0']), - # Intermediate - convBnCFG(weights_dict=self.weights_dict['intermediate_conv1']['0']), - convBnCFG(weights_dict=self.weights_dict['intermediate_conv2']['0']), - residualBlockCFG(weights_dict=self.weights_dict['intermediate_residual']['0']), - # Hourglass - hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['1']), - convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), - ] - def get_cfg_list(self, name): if name == 'hourglass104_512': - return self.hourglass104_512 + return [ + # Downsampling Layers + convBnCFG(weights_dict=self.weights_dict['downsample_input']['conv_block']), + residualBlockCFG(weights_dict=self.weights_dict['downsample_input']['residual_block']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['0']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['0']), + # Intermediate + convBnCFG(weights_dict=self.weights_dict['intermediate_conv1']['0']), + convBnCFG(weights_dict=self.weights_dict['intermediate_conv2']['0']), + residualBlockCFG(weights_dict=self.weights_dict['intermediate_residual']['0']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['1']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), + ] + elif name == 'extremenet': - return self.extremenet + return [ + # Downsampling Layers + convBnCFG(weights_dict=self.weights_dict['downsample_input']['conv_block']), + residualBlockCFG(weights_dict=self.weights_dict['downsample_input']['residual_block']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['0']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['0']), + # Intermediate + convBnCFG(weights_dict=self.weights_dict['intermediate_conv1']['0']), + convBnCFG(weights_dict=self.weights_dict['intermediate_conv2']['0']), + residualBlockCFG(weights_dict=self.weights_dict['intermediate_residual']['0']), + # Hourglass + hourglassCFG(weights_dict=self.weights_dict['hourglass_network']['1']), + convBnCFG(weights_dict=self.weights_dict['output_conv']['1']), + ] @dataclass class DecoderConfigData(): - weights_dict: Dict = field(repr=False, default=None) - - def __post_init__(self): - self.detection_2d = [ - decoderConvCFG(weights_dict=self.weights_dict['object_center']['0']), - decoderConvCFG(weights_dict=self.weights_dict['object_center']['1']), - decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['0']), - decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['1']), - decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['0']), - decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['1']) - ] + weights_dict: Dict = field(repr=False, default=None) def get_cfg_list(self, name): if name == 'detection_2d': - return self.detection_2d + return [ + decoderConvCFG(weights_dict=self.weights_dict['object_center']['0']), + decoderConvCFG(weights_dict=self.weights_dict['object_center']['1']), + decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['0']), + decoderConvCFG(weights_dict=self.weights_dict['box.Soffset']['1']), + decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['0']), + decoderConvCFG(weights_dict=self.weights_dict['box.Sscale']['1']) + ] diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index 916380fa4..e119342ba 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -3,8 +3,9 @@ from centernet.configs.centernet import CenterNetTask from centernet.modeling.CenterNet import build_centernet -from centernet.modeling.layers.nn_blocks import (CenterNetDecoderConv, ConvBN, - HourglassBlock) +from centernet.modeling.layers.nn_blocks import (CenterNetDecoderConv, + CenterNetResidualBlock, + ConvBN, HourglassBlock) from centernet.utils.weight_utils.config_classes import (convBnCFG, decoderConvCFG, hourglassCFG, @@ -13,7 +14,6 @@ DecoderConfigData) from centernet.utils.weight_utils.tf_to_dict import ( get_model_weights_as_dict, write_dict_as_tree) -from official.vision.beta.modeling.layers.nn_blocks import ResidualBlock CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' SAVED_MODEL_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\saved_model' @@ -73,11 +73,11 @@ def load_weights_backbone(backbone, weights_dict, backbone_name): backbone_layers = backbone.layers cfgs = get_backbone_layer_cfgs(weights_dict, backbone_name) n_weights_total = 0 - + cfg = cfgs.pop(0) for i in range(len(backbone_layers)): layer = backbone_layers[i] - if isinstance(layer, (ConvBN, HourglassBlock, ResidualBlock)): + if isinstance(layer, (ConvBN, HourglassBlock, CenterNetResidualBlock)): n_weights = cfg.load_weights(layer) print("Loading weights for: {}, weights loaded: {}".format(cfg, n_weights)) n_weights_total += n_weights From 55a039385c643f5df915dba79da5ce63819a00db Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 28 Apr 2021 19:23:39 +0000 Subject: [PATCH 123/132] config update --- .centernet-eval.log.swp | Bin 16384 -> 0 bytes centernet/configs/centernet.py | 1 + .../configs/experiments/centernet-eval-tpu.yaml | 2 +- .../modeling/layers/detection_generator.py | 14 ++------------ 4 files changed, 4 insertions(+), 13 deletions(-) delete mode 100644 .centernet-eval.log.swp diff --git a/.centernet-eval.log.swp b/.centernet-eval.log.swp deleted file mode 100644 index 20fb4afe3a63ca48d03a9ae3cd989d14dab46112..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHONslB)6;7{XgAIsPVv!0XR5Qk1U0FS~jQ|Tg@`!`60gpi%SwU20R8>r8W~3rA zdzq#^Hx4XfyR(Sp2iQg!?tl@?6|soz&LRURFz~&|$m*<`83~6F7&%+|+*R>ndGW;; z@m{!NX8p(B_fTij3j_STDhPgh;p#^(cJ2zEzdHybo(>9;8Kvim+NTQF|4}i|<@HT# zrBAGPxGna0!U|yunJu;-wF5nffgVz^NX&L5vO?%g6kAlUm#D>BXYH4kftG<6$iUsf z&LHelqqp7j*3N5QeenghZ>zTqv<$Qiv<$Qiv<$Qiv<$Qiv<&<|Gf?EGf~%Oo&2AF) z@0;BB`u9KG&podEmJRzqHtcWTus`qY_q*~78}`3D`;jaE_=f#&&VE>zJI8AOYJV;} z`;`s5D{gdH(OELw_aH&-$%P}c3<*XgG8g1}aoaU<-9;xMNV?^l^P zMvbgYS(I>NghA=Fk0K_syeycJSE%BpVK}(2WB;@BIIMG-;5CBnD&+?(7KMlksWNIF z_Njd?_%0J#E2OKDGbHs0J200rM9^{D;D5+jOuwto&S)#n( z3sEXn*(^Fs70G+1=5r%x-c&1glkDQT1}}C7Lm?m{@>YGGj3> zI7*34A|U4l4cuLrbKBb|&p!s+k%|=hijdQj-<6B{^3685TEgOwpEegSE}c7bC7Q znq8Sgv5}&gu({wxsmWv8?Yd~f5v`}K>(pLr%E&awylWG!#Ci!SNQ;HYgyw}{MwL1e zOs>4Kip#kMV;CE1OZh^ueSEgN%zY>6Ro}0(MQP_zvCK@tv&in-{B=ZF?jgdTq zG}0X1IKixnwWV!~R(n;3jfI;1-WY|dEJ**cv>USOcby$ST7U2Y4D1FFM zQD_-8=Uw$e=P*CS4EsF~Z+doW6^SeZ zRov)_;5wr~Nnvc=@j~SL2(Z@s`ZG)DnUfTvcH6;SfW%Tu&#{rGPCI=nz}7pz3iQ{8(Y>o ztQUtTc>c=}g(*=@xh8?vHtQUt&EdQtFUfgV6-fcs8a%oQkm=-Ra6Kz!6NN%nE@)D; zhag?NMJ`ooSTzHg3S>6*TVC38qW%WwU-9tH_+SoRu~!!bdv6X#KiaPIAEI%?Al>wR%w( zSZccJ;HaTH8!T=UQu2$7jm`FQE%zv+tSZvrt`beA(L;niMr%b*)1VkTL#JC48lKWd z)U~eT>%P*sjj^RNMWaIsOHfKI4$#;picn;AeG-f(bf~$Zsk@vRml;w1Dpax8c(N1Ns9x?*H}Q&wqvc|963} z0-peuz{9{6@LJ%xR|LV&fNud`1U?2tz zfHwg*1HZX52)++|9rzS*8TbIO4ZI%s%N;@R3*g(pmw_h$0XzWQ3)}(x@l+7}7quv%oiiPXoKa2Z28D2H>x^1;HqTq!Z2j=z#rOYdT95vl44kI0$mW^p<%efFbbz1h zbO!xVaJsSKalW1DY<2izftxy=srXklCb&VjF)W7M( zzvg{%V@|t7+n|ris6rCjM%g_^WB+ zucnE=nkN2gn)s_}?60P=znaGWY8v~iY3#42vA>$e{%RWgt7+`7rm?@8M*eCV`KxK< yucnc|nnwO=8u_bf 500500 * 64/16 - validation_steps: -1 # 5063 #625 + validation_steps: 625 # 5063 #625 steps_per_loop: 10000 validation_interval: 1 summary_interval: 10000 diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 8ec651628..2ab7e503e 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -3,9 +3,6 @@ from yolo.ops.nms_ops import nms -# from tensorflow import keras as ks - - @tf.keras.utils.register_keras_serializable(package='centernet') class CenterNetLayer(ks.Model): @@ -108,11 +105,6 @@ def process_heatmap(self, feature_map_peaks = ( feature_map * tf.cast(feature_map_peak_mask, feature_map.dtype)) - # Zero out peaks whose scores do not exceed threshold - valid_peaks_mask = feature_map_peaks > center_thresh - - feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, feature_map_peaks.dtype) - return feature_map_peaks def get_row_col_channel_indices_from_flattened_indices(self, @@ -294,8 +286,6 @@ class prediction for each box. detection_classes = channel_indices + self._class_offset - num_detections = tf.reduce_sum(tf.cast(detection_scores > 0, dtype=tf.int32), axis=1) - ymin = y_indices + y_offsets - heights / 2.0 xmin = x_indices + x_offsets - widths / 2.0 ymax = y_indices + y_offsets + heights / 2.0 @@ -307,7 +297,7 @@ class prediction for each box. xmax = tf.clip_by_value(xmax, 0., tf.cast(width, xmax.dtype)) boxes = tf.stack([ymin, xmin, ymax, xmax], axis=2) - return boxes, detection_classes, detection_scores, num_detections + return boxes, detection_classes, detection_scores def convert_strided_predictions_to_normalized_boxes(self, boxes): @@ -340,7 +330,7 @@ def call(self, inputs): batch_size, width, num_channels, k=self._max_detections) # Parse the score and indices into bounding boxes - boxes, classes, scores, num_det = self.get_boxes(scores, + boxes, classes, scores = self.get_boxes(scores, y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, self._max_detections) # Normalize bounding boxes From 1d9a44d73d0b717d1ecc228edb4df565b56dc9d8 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 28 Apr 2021 15:27:43 -0400 Subject: [PATCH 124/132] backbone validation --- centernet/modeling/backbones/hourglass.py | 3 +- .../modeling/layers/detection_generator.py | 15 +- centernet/modeling/layers/nn_blocks.py | 228 ++------- centernet/validation/config_classes.py | 177 +++++++ centernet/validation/load_weights.py | 78 +++ centernet/validation/odapi_backbone.py | 463 ++++++++++++++++++ .../validation/odapi_detection_generator.py | 212 ++++++++ 7 files changed, 977 insertions(+), 199 deletions(-) create mode 100644 centernet/validation/config_classes.py create mode 100644 centernet/validation/load_weights.py create mode 100644 centernet/validation/odapi_backbone.py create mode 100644 centernet/validation/odapi_detection_generator.py diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index a80a68a15..569042e96 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -119,7 +119,8 @@ def __init__( # inters x_downsampled = nn_blocks.CenterNetResidualBlock( filters=inp_filters, use_projection=False, strides=1, - norm_momentum=BATCH_NORM_MOMENTUM, norm_epsilon=BATCH_NORM_EPSILON + use_sync_bn=True, norm_momentum=BATCH_NORM_MOMENTUM, + norm_epsilon=BATCH_NORM_EPSILON )(x_downsampled) # yapf: enable diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 8ec651628..d93b39664 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -3,9 +3,6 @@ from yolo.ops.nms_ops import nms -# from tensorflow import keras as ks - - @tf.keras.utils.register_keras_serializable(package='centernet') class CenterNetLayer(ks.Model): @@ -108,10 +105,10 @@ def process_heatmap(self, feature_map_peaks = ( feature_map * tf.cast(feature_map_peak_mask, feature_map.dtype)) - # Zero out peaks whose scores do not exceed threshold - valid_peaks_mask = feature_map_peaks > center_thresh + # # Zero out peaks whose scores do not exceed threshold + # valid_peaks_mask = feature_map_peaks > center_thresh - feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, feature_map_peaks.dtype) + # feature_map_peaks = feature_map_peaks * tf.cast(valid_peaks_mask, feature_map_peaks.dtype) return feature_map_peaks @@ -293,8 +290,6 @@ class prediction for each box. x_indices = tf.cast(x_indices, dtype=widths.dtype) detection_classes = channel_indices + self._class_offset - - num_detections = tf.reduce_sum(tf.cast(detection_scores > 0, dtype=tf.int32), axis=1) ymin = y_indices + y_offsets - heights / 2.0 xmin = x_indices + x_offsets - widths / 2.0 @@ -307,7 +302,7 @@ class prediction for each box. xmax = tf.clip_by_value(xmax, 0., tf.cast(width, xmax.dtype)) boxes = tf.stack([ymin, xmin, ymax, xmax], axis=2) - return boxes, detection_classes, detection_scores, num_detections + return boxes, detection_classes, detection_scores def convert_strided_predictions_to_normalized_boxes(self, boxes): @@ -340,7 +335,7 @@ def call(self, inputs): batch_size, width, num_channels, k=self._max_detections) # Parse the score and indices into bounding boxes - boxes, classes, scores, num_det = self.get_boxes(scores, + boxes, classes, scores = self.get_boxes(scores, y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, self._max_detections) # Normalize bounding boxes diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 92cbc3d11..73b0ff700 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -18,194 +18,6 @@ def __init__(self, **kwargs): def call(self, input): return input - -# @tf.keras.utils.register_keras_serializable(package='centernet') -# class ConvBN(tf.keras.layers.Layer): -# """ -# Modified Convolution layer to match that of the DarkNet Library. -# The Layer is a standards combination of Conv BatchNorm Activation, -# however, the use of bias in the conv is determined by the use of batch -# normalization. -# Cross Stage Partial networks (CSPNets) were proposed in: -# [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, -# Ping-Yang Chen, Jun-Wei Hsieh -# CSPNet: A New Backbone that can Enhance Learning Capability of CNN. -# arXiv:1911.11929 -# Args: -# filters: integer for output depth, or the number of features to learn -# kernel_size: integer or tuple for the shape of the weight matrix or kernel -# to learn -# strides: integer of tuple how much to move the kernel after each kernel -# use -# padding: string 'valid' or 'same', if same, then pad the image, else do -# not -# dialtion_rate: tuple to indicate how much to modulate kernel weights and -# how many pixels in a feature map to skip -# kernel_initializer: string to indicate which function to use to initialize -# weights -# bias_initializer: string to indicate which function to use to initialize -# bias -# kernel_regularizer: string to indicate which function to use to -# regularizer weights -# bias_regularizer: string to indicate which function to use to regularizer -# bias -# use_bn: boolean for whether to use batch normalization -# use_sync_bn: boolean for whether sync batch normalization statistics -# of all batch norm layers to the models global statistics -# (across all input batches) -# norm_momentum: float for moment to use for batch normalization -# norm_epsilon: float for batch normalization epsilon -# activation: string or None for activation function to use in layer, -# if None activation is replaced by linear -# leaky_alpha: float to use as alpha if activation function is leaky -# **kwargs: Keyword Arguments -# """ - -# def __init__(self, -# filters=1, -# kernel_size=(1, 1), -# strides=(1, 1), -# padding='same', -# dilation_rate=(1, 1), -# kernel_initializer='glorot_uniform', -# bias_initializer='zeros', -# subdivisions=1, -# bias_regularizer=None, -# kernel_regularizer=None, -# use_bn=True, -# use_sync_bn=False, -# norm_momentum=0.99, -# norm_epsilon=0.001, -# activation='leaky', -# leaky_alpha=0.1, -# **kwargs): - -# # convolution params -# self._filters = filters -# self._kernel_size = kernel_size -# self._strides = strides -# self._padding = padding -# self._dilation_rate = dilation_rate -# self._kernel_initializer = kernel_initializer -# self._bias_initializer = bias_initializer -# self._kernel_regularizer = kernel_regularizer - -# self._bias_regularizer = bias_regularizer -# self._subdivisions = subdivisions - -# # batch normalization params -# self._use_bn = use_bn -# self._use_sync_bn = use_sync_bn -# self._norm_momentum = norm_momentum -# self._norm_epsilon = norm_epsilon - -# if tf.keras.backend.image_data_format() == 'channels_last': -# # format: (batch_size, height, width, channels) -# self._bn_axis = -1 -# else: -# # format: (batch_size, channels, width, height) -# self._bn_axis = 1 - -# # activation params -# self._activation = activation -# self._leaky_alpha = leaky_alpha - -# super().__init__(**kwargs) - -# def build(self, input_shape): -# use_bias = not self._use_bn - -# if not TPU_BASE: -# kernel_size = self._kernel_size if isinstance( -# self._kernel_size, int) else self._kernel_size[0] -# dilation_rate = self._dilation_rate if isinstance( -# self._dilation_rate, int) else self._dilation_rate[0] -# if self._padding == 'same' and kernel_size != 1: -# padding = (dilation_rate * (kernel_size - 1)) -# left_shift = padding // 2 -# self._paddings = tf.constant([[0, 0], [left_shift, left_shift], -# [left_shift, left_shift], [0, 0]]) -# # self._zeropad = tf.keras.layers.ZeroPadding2D() -# else: -# self._paddings = tf.constant([[0, 0], [0, 0], [0, 0], [0, 0]]) - -# self.conv = tf.keras.layers.Conv2D( -# filters=self._filters, -# kernel_size=self._kernel_size, -# strides=self._strides, -# padding='valid', -# dilation_rate=self._dilation_rate, -# use_bias=use_bias, -# kernel_initializer=self._kernel_initializer, -# bias_initializer=self._bias_initializer, -# kernel_regularizer=self._kernel_regularizer, -# bias_regularizer=self._bias_regularizer) -# else: -# self.conv = tf.keras.layers.Conv2D( -# filters=self._filters, -# kernel_size=self._kernel_size, -# strides=self._strides, -# padding=self._padding, # 'valid', -# dilation_rate=self._dilation_rate, -# use_bias=use_bias, -# kernel_initializer=self._kernel_initializer, -# bias_initializer=self._bias_initializer, -# kernel_regularizer=self._kernel_regularizer, -# bias_regularizer=self._bias_regularizer) - -# if self._use_bn: -# if self._use_sync_bn: -# self.bn = subnormalization.SubDivSyncBatchNormalization( -# subdivisions=self._subdivisions, -# momentum=self._norm_momentum, -# epsilon=self._norm_epsilon, -# axis=self._bn_axis) -# else: -# self.bn = subnormalization.SubDivBatchNormalization( -# subdivisions=self._subdivisions, -# momentum=self._norm_momentum, -# epsilon=self._norm_epsilon, -# axis=self._bn_axis) - -# if self._activation == 'leaky': -# self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha) -# elif self._activation == 'mish': -# self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x)) -# else: -# self._activation_fn = tf_utils.get_activation( -# self._activation) # tf.keras.layers.Activation(self._activation) - -# def call(self, x): -# if not TPU_BASE: -# x = tf.pad(x, self._paddings, mode='CONSTANT', constant_values=0) -# x = self.conv(x) -# if self._use_bn: -# x = self.bn(x) -# x = self._activation_fn(x) -# return x - -# def get_config(self): -# # used to store/share parameters to reconstruct the model -# layer_config = { -# 'filters': self._filters, -# 'kernel_size': self._kernel_size, -# 'strides': self._strides, -# 'padding': self._padding, -# 'dilation_rate': self._dilation_rate, -# 'kernel_initializer': self._kernel_initializer, -# 'bias_initializer': self._bias_initializer, -# 'bias_regularizer': self._bias_regularizer, -# 'kernel_regularizer': self._kernel_regularizer, -# 'use_bn': self._use_bn, -# 'use_sync_bn': self._use_sync_bn, -# 'norm_momentum': self._norm_momentum, -# 'norm_epsilon': self._norm_epsilon, -# 'activation': self._activation, -# 'leaky_alpha': self._leaky_alpha -# } -# layer_config.update(super().get_config()) -# return layer_config - @tf.keras.utils.register_keras_serializable(package='centernet') class CenterNetResidualBlock(tf.keras.layers.Layer): """A residual block.""" @@ -402,6 +214,46 @@ def __init__(self, activation='leaky', leaky_alpha=0.1, **kwargs): + """ + Modified Convolution layer based on the DarkNet Library, with changes + such that it is compatiable with the CenterNet backbone. + The Layer is a standards combination of Conv BatchNorm Activation, + however, the use of bias in the conv is determined by the use of batch + normalization. + Cross Stage Partial networks (CSPNets) were proposed in: + [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, + Ping-Yang Chen, Jun-Wei Hsieh + CSPNet: A New Backbone that can Enhance Learning Capability of CNN. + arXiv:1911.11929 + Args: + filters: integer for output depth, or the number of features to learn + kernel_size: integer or tuple for the shape of the weight matrix or kernel + to learn + strides: integer of tuple how much to move the kernel after each kernel + use + padding: string 'valid' or 'same', if same, then pad the image, else do + not + dialtion_rate: tuple to indicate how much to modulate kernel weights and + how many pixels in a feature map to skip + kernel_initializer: string to indicate which function to use to initialize + weights + bias_initializer: string to indicate which function to use to initialize + bias + kernel_regularizer: string to indicate which function to use to + regularizer weights + bias_regularizer: string to indicate which function to use to regularizer + bias + use_bn: boolean for whether to use batch normalization + use_sync_bn: boolean for whether sync batch normalization statistics + of all batch norm layers to the models global statistics + (across all input batches) + norm_momentum: float for moment to use for batch normalization + norm_epsilon: float for batch normalization epsilon + activation: string or None for activation function to use in layer, + if None activation is replaced by linear + leaky_alpha: float to use as alpha if activation function is leaky + **kwargs: Keyword Arguments + """ # convolution params self._filters = filters diff --git a/centernet/validation/config_classes.py b/centernet/validation/config_classes.py new file mode 100644 index 000000000..52183f9a8 --- /dev/null +++ b/centernet/validation/config_classes.py @@ -0,0 +1,177 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import ClassVar, Dict, List, Tuple + +import numpy as np + + +class Config(ABC): + def get_weights(self): + return None + + def load_weights(self, layer): + weights = self.get_weights() + layer.set_weights(weights) + n_weights = 0 + + for w in weights: + n_weights += w.size + return n_weights + +@dataclass +class ODAPIconvBnCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + weights: np.array = field(repr=False, default=None) + beta: np.array = field(repr=False, default=None) + gamma: np.array = field(repr=False, default=None) + moving_mean: np.array = field(repr=False, default=None) + moving_variance: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_weights_dict = self.weights_dict['conv'] + norm_weights_dict = self.weights_dict['norm'] + + self.weights = conv_weights_dict['kernel'] + self.beta = norm_weights_dict['beta'] + self.gamma = norm_weights_dict['gamma'] + self.moving_mean = norm_weights_dict['moving_mean'] + self.moving_variance = norm_weights_dict['moving_variance'] + + def get_weights(self): + return [ + self.weights, + self.gamma, + self.beta, + self.moving_mean, + self.moving_variance + ] + +@dataclass +class ODAPIresidualBlockCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + + skip_weights: np.array = field(repr=False, default=None) + skip_beta: np.array = field(repr=False, default=None) + skip_gamma: np.array = field(repr=False, default=None) + skip_moving_mean: np.array = field(repr=False, default=None) + skip_moving_variance: np.array = field(repr=False, default=None) + + conv_weights: np.array = field(repr=False, default=None) + norm_beta: np.array = field(repr=False, default=None) + norm_gamma: np.array = field(repr=False, default=None) + norm_moving_mean: np.array = field(repr=False, default=None) + norm_moving_variance: np.array = field(repr=False, default=None) + + conv_block_weights: np.array = field(repr=False, default=None) + conv_block_beta: np.array = field(repr=False, default=None) + conv_block_gamma: np.array = field(repr=False, default=None) + conv_block_moving_mean: np.array = field(repr=False, default=None) + conv_block_moving_variance: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_weights_dict = self.weights_dict['conv'] + norm_weights_dict = self.weights_dict['norm'] + conv_block_weights_dict = self.weights_dict['conv_block'] + + if 'skip' in self.weights_dict: + skip_weights_dict = self.weights_dict['skip'] + self.skip_weights = skip_weights_dict['conv']['kernel'] + self.skip_beta = skip_weights_dict['norm']['beta'] + self.skip_gamma = skip_weights_dict['norm']['gamma'] + self.skip_moving_mean = skip_weights_dict['norm']['moving_mean'] + self.skip_moving_variance = skip_weights_dict['norm']['moving_variance'] + + self.conv_weights = conv_weights_dict['kernel'] + self.norm_beta = norm_weights_dict['beta'] + self.norm_gamma = norm_weights_dict['gamma'] + self.norm_moving_mean = norm_weights_dict['moving_mean'] + self.norm_moving_variance = norm_weights_dict['moving_variance'] + + self.conv_block_weights = conv_block_weights_dict['conv']['kernel'] + self.conv_block_beta = conv_block_weights_dict['norm']['beta'] + self.conv_block_gamma = conv_block_weights_dict['norm']['gamma'] + self.conv_block_moving_mean = conv_block_weights_dict['norm']['moving_mean'] + self.conv_block_moving_variance = conv_block_weights_dict['norm']['moving_variance'] + + def get_weights(self): + weights = [ + self.conv_block_weights, + self.conv_block_gamma, + self.conv_block_beta, + + self.conv_weights, + self.norm_gamma, + self.norm_beta, + + self.skip_weights, + self.skip_gamma, + self.skip_beta, + + self.conv_block_moving_mean, + self.conv_block_moving_variance, + self.norm_moving_mean, + self.norm_moving_variance, + self.skip_moving_mean, + self.skip_moving_variance, + ] + + weights = [x for x in weights if x is not None] + return weights + + +@dataclass +class ODAPIhourglassCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + is_last_stage: bool = field(repr=False, default=None) + + def __post_init__(self): + self.is_last_stage = False if 'inner_block' in self.weights_dict else True + + def generate_block_weights(self, weights_dict): + reps = len(weights_dict.keys()) + weights = [] + n_weights = 0 + + for i in range(reps): + res_config = ODAPIresidualBlockCFG(weights_dict=weights_dict[str(i)]) + res_weights = res_config.get_weights() + weights += res_weights + + for w in res_weights: + n_weights += w.size + + return weights, n_weights + + def load_block_weights(self, layer, weight_dict): + block_weights, n_weights = self.generate_block_weights(weight_dict) + layer.set_weights(block_weights) + return n_weights + + def load_weights(self, layer): + n_weights = 0 + + enc_dec_layers = [ + layer.submodules[1], + layer.submodules[2], + layer.submodules[4] + ] + enc_dec_weight_dicts = [ + self.weights_dict['encoder_block1'], + self.weights_dict['encoder_block2'], + self.weights_dict['decoder_block'] + ] + + for l, weights_dict in zip(enc_dec_layers, enc_dec_weight_dicts): + n_weights += self.load_block_weights(l, weights_dict) + + if len(self.weights_dict['inner_block']) == 1: # still in an outer hourglass + inner_weights_dict = self.weights_dict['inner_block']['0'] + inner_hg_layer = layer.submodules[3] + inner_hg_cfg = type(self)(weights_dict=inner_weights_dict) + n_weights += inner_hg_cfg.load_weights(inner_hg_layer) + else: + inner_weights_dict = self.weights_dict['inner_block'] # inner residual block chain + inner_layer = layer.submodules[3] + n_weights += self.load_block_weights(inner_layer, inner_weights_dict) + + return n_weights diff --git a/centernet/validation/load_weights.py b/centernet/validation/load_weights.py new file mode 100644 index 000000000..d298d2ee3 --- /dev/null +++ b/centernet/validation/load_weights.py @@ -0,0 +1,78 @@ +import numpy as np +import tensorflow as tf + +from centernet.configs.centernet import CenterNetTask +from centernet.modeling.CenterNet import build_centernet +from centernet.utils.weight_utils.load_weights import ( + get_model_weights_as_dict, load_weights_backbone, load_weights_model) +from centernet.validation.config_classes import (ODAPIconvBnCFG, + ODAPIhourglassCFG, + ODAPIresidualBlockCFG) +from centernet.validation.odapi_backbone import hourglass_104 + +CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' + +def load_weights_odapi_backbone(weights_dict): + odapi_backbone = hourglass_104() + odapi_backbone.build(input_shape=[1, 512, 512, 3]) + + # load weights for downsample block (comprised of a conv and res block) + downsample_conv_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['downsample_input']['conv_block']) + downsample_res_cfg = ODAPIresidualBlockCFG(weights_dict=weights_dict['downsample_input']['residual_block']) + downsample_conv_cfg.load_weights(odapi_backbone.downsample_input.conv_block) + downsample_res_cfg.load_weights(odapi_backbone.downsample_input.residual_block) + + # load weights for encoder/decoder blocks + hourglass_0_cfg = ODAPIhourglassCFG(weights_dict=weights_dict['hourglass_network']['0']) + hourglass_1_cfg = ODAPIhourglassCFG(weights_dict=weights_dict['hourglass_network']['1']) + hourglass_0_cfg.load_weights(odapi_backbone.hourglass_network[0]) + hourglass_1_cfg.load_weights(odapi_backbone.hourglass_network[1]) + + # load weights for hourglass output conv blocks + out_conv_0_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['output_conv']['0']) + out_conv_1_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['output_conv']['1']) + out_conv_0_cfg.load_weights(odapi_backbone.output_conv[0]) + out_conv_1_cfg.load_weights(odapi_backbone.output_conv[1]) + + # load weights for intermediate conv and residual blocks + inter_conv_1_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['intermediate_conv1']['0']) + inter_conv_2_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['intermediate_conv2']['0']) + inter_res_2_cfg = ODAPIresidualBlockCFG(weights_dict=weights_dict['intermediate_residual']['0']) + inter_conv_1_cfg.load_weights(odapi_backbone.intermediate_conv1[0]) + inter_conv_2_cfg.load_weights(odapi_backbone.intermediate_conv2[0]) + inter_res_2_cfg.load_weights(odapi_backbone.intermediate_residual[0]) + + return odapi_backbone + +@tf.function +def compare(model_1, model_2, input): + out_1 = model_1(input) + out_2 = model_2(input) + return out_1, out_2 + +if __name__ == '__main__': + # testing if the output between our backbone and the ODAPI backbone matches + weights_dict, n_weights = get_model_weights_as_dict(CKPT_PATH) + backbone_weights_dict = weights_dict['model']['_feature_extractor']['_network'] + + # load weights into odapi backbone + odapi_backbone = load_weights_odapi_backbone(backbone_weights_dict) + + # load weights into tfmg centernet model + input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) + config = CenterNetTask() + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + + test_input = tf.random.uniform( + shape=[1, 512, 512, 3], minval=0, maxval=1, dtype=tf.dtypes.float32, seed=45 + ) + + tfmg_output, odapi_output = compare(odapi_backbone, model.backbone, test_input) + tfmg_hm1 = tfmg_output[0] + tfmg_hm2 = tfmg_output[1] + odapi_hm1 = odapi_output[0] + odapi_hm2 = odapi_output[1] + print(tf.math.reduce_sum(odapi_hm2-tfmg_hm2)) + print(tf.math.reduce_sum(odapi_hm1-tfmg_hm1)) diff --git a/centernet/validation/odapi_backbone.py b/centernet/validation/odapi_backbone.py new file mode 100644 index 000000000..bec4b9f9a --- /dev/null +++ b/centernet/validation/odapi_backbone.py @@ -0,0 +1,463 @@ +import tensorflow as tf + +BATCH_NORM_EPSILON = 1e-5 +BATCH_NORM_MOMENTUM = 0.1 +BATCH_NORM_FUSED = True + + +class IdentityLayer(tf.keras.layers.Layer): + """A layer which passes through the input as it is.""" + + def call(self, inputs): + return inputs + + +def _get_padding_for_kernel_size(kernel_size): + if kernel_size == 7: + return (3, 3) + elif kernel_size == 3: + return (1, 1) + else: + raise ValueError('Padding for kernel size {} not known.'.format( + kernel_size)) + + +def batchnorm(): + try: + return tf.keras.layers.experimental.SyncBatchNormalization( + name='batchnorm', epsilon=1e-5, momentum=0.1) + except AttributeError: + return tf.keras.layers.BatchNormalization( + name='batchnorm', epsilon=1e-5, momentum=0.1, fused=BATCH_NORM_FUSED) + + +class ConvolutionalBlock(tf.keras.layers.Layer): + """Block that aggregates Convolution + Norm layer + ReLU.""" + + def __init__(self, kernel_size, out_channels, stride=1, relu=True, + padding='same'): + """Initializes the Convolutional block. + Args: + kernel_size: int, convolution kernel size. + out_channels: int, the desired number of output channels. + stride: Integer, stride used in the convolution. + relu: bool, whether to use relu at the end of the layer. + padding: str, the padding scheme to use when kernel_size <= 1 + """ + super(ConvolutionalBlock, self).__init__() + + if kernel_size > 1: + padding = 'valid' + padding_size = _get_padding_for_kernel_size(kernel_size) + + # TODO(vighneshb) Explore if removing and using padding option in conv + # layer works. + self.pad = tf.keras.layers.ZeroPadding2D(padding_size) + else: + self.pad = IdentityLayer() + + self.conv = tf.keras.layers.Conv2D( + filters=out_channels, kernel_size=kernel_size, use_bias=False, + strides=stride, padding=padding) + + self.norm = batchnorm() + + if relu: + self.relu = tf.keras.layers.ReLU() + else: + self.relu = IdentityLayer() + + def call(self, inputs): + net = self.pad(inputs) + net = self.conv(net) + net = self.norm(net) + return self.relu(net) + + +class SkipConvolution(ConvolutionalBlock): + """The skip connection layer for a ResNet.""" + + def __init__(self, out_channels, stride): + """Initializes the skip convolution layer. + Args: + out_channels: int, the desired number of output channels. + stride: int, the stride for the layer. + """ + super(SkipConvolution, self).__init__( + out_channels=out_channels, kernel_size=1, stride=stride, relu=False) + + +class ResidualBlock(tf.keras.layers.Layer): + """A Residual block.""" + + def __init__(self, out_channels, skip_conv=False, kernel_size=3, stride=1, + padding='same'): + """Initializes the Residual block. + Args: + out_channels: int, the desired number of output channels. + skip_conv: bool, whether to use a conv layer for skip connections. + kernel_size: int, convolution kernel size. + stride: Integer, stride used in the convolution. + padding: str, the type of padding to use. + """ + + super(ResidualBlock, self).__init__() + self.conv_block = ConvolutionalBlock( + kernel_size=kernel_size, out_channels=out_channels, stride=stride) + + self.conv = tf.keras.layers.Conv2D( + filters=out_channels, kernel_size=kernel_size, use_bias=False, + strides=1, padding=padding) + self.norm = batchnorm() + + if skip_conv: + self.skip = SkipConvolution(out_channels=out_channels, + stride=stride) + else: + self.skip = IdentityLayer() + + self.relu = tf.keras.layers.ReLU() + + def call(self, inputs): + net = self.conv_block(inputs) + net = self.conv(net) + net = self.norm(net) + net_skip = self.skip(inputs) + return self.relu(net + net_skip) + + +class InputDownsampleBlock(tf.keras.layers.Layer): + """Block for the initial feature downsampling.""" + + def __init__(self, out_channels_initial_conv, out_channels_residual_block): + """Initializes the downsample block. + Args: + out_channels_initial_conv: int, the desired number of output channels + in the initial conv layer. + out_channels_residual_block: int, the desired number of output channels + in the underlying residual block. + """ + + super(InputDownsampleBlock, self).__init__() + self.conv_block = ConvolutionalBlock( + kernel_size=7, out_channels=out_channels_initial_conv, stride=2, + padding='valid') + self.residual_block = ResidualBlock( + out_channels=out_channels_residual_block, stride=2, skip_conv=True) + + def call(self, inputs): + return self.residual_block(self.conv_block(inputs)) + + +class InputConvBlock(tf.keras.layers.Layer): + """Block for the initial feature convolution. + This block is used in the hourglass network when we don't want to downsample + the input. + """ + + def __init__(self, out_channels_initial_conv, out_channels_residual_block): + """Initializes the downsample block. + Args: + out_channels_initial_conv: int, the desired number of output channels + in the initial conv layer. + out_channels_residual_block: int, the desired number of output channels + in the underlying residual block. + """ + + super(InputConvBlock, self).__init__() + + self.conv_block = ConvolutionalBlock( + kernel_size=3, out_channels=out_channels_initial_conv, stride=1, + padding='valid') + self.residual_block = ResidualBlock( + out_channels=out_channels_residual_block, stride=1, skip_conv=True) + + def call(self, inputs): + return self.residual_block(self.conv_block(inputs)) + + +def _make_repeated_residual_blocks(out_channels, num_blocks, + initial_stride=1, residual_channels=None, + initial_skip_conv=False): + """Stack Residual blocks one after the other. + Args: + out_channels: int, the desired number of output channels. + num_blocks: int, the number of residual blocks to be stacked. + initial_stride: int, the stride of the initial residual block. + residual_channels: int, the desired number of output channels in the + intermediate residual blocks. If not specifed, we use out_channels. + initial_skip_conv: bool, if set, the first residual block uses a skip + convolution. This is useful when the number of channels in the input + are not the same as residual_channels. + Returns: + blocks: A list of residual blocks to be applied in sequence. + """ + + blocks = [] + + if residual_channels is None: + residual_channels = out_channels + + for i in range(num_blocks - 1): + # Only use the stride at the first block so we don't repeatedly downsample + # the input + stride = initial_stride if i == 0 else 1 + + # If the stide is more than 1, we cannot use an identity layer for the + # skip connection and are forced to use a conv for the skip connection. + skip_conv = stride > 1 + + if i == 0 and initial_skip_conv: + skip_conv = True + + blocks.append( + ResidualBlock(out_channels=residual_channels, stride=stride, + skip_conv=skip_conv) + ) + + if num_blocks == 1: + # If there is only 1 block, the for loop above is not run, + # therefore we honor the requested stride in the last residual block + stride = initial_stride + # We are forced to use a conv in the skip connection if stride > 1 + skip_conv = stride > 1 + else: + stride = 1 + skip_conv = residual_channels != out_channels + + blocks.append(ResidualBlock(out_channels=out_channels, skip_conv=skip_conv, + stride=stride)) + + # return blocks + return tf.keras.Sequential(blocks) + + +def _apply_blocks(inputs, blocks): + net = inputs + + # for block in blocks: + # net = block(net) + + net = blocks(net) + + return net + + +class EncoderDecoderBlock(tf.keras.layers.Layer): + """An encoder-decoder block which recursively defines the hourglass network.""" + + def __init__(self, num_stages, channel_dims, blocks_per_stage, + stagewise_downsample=True, encoder_decoder_shortcut=True): + """Initializes the encoder-decoder block. + Args: + num_stages: int, Number of stages in the network. At each stage we have 2 + encoder and 1 decoder blocks. The second encoder block downsamples the + input. + channel_dims: int list, the output channels dimensions of stages in + the network. `channel_dims[0]` is used to define the number of + channels in the first encoder block and `channel_dims[1]` is used to + define the number of channels in the second encoder block. The channels + in the recursive inner layers are defined using `channel_dims[1:]` + blocks_per_stage: int list, number of residual blocks to use at each + stage. `blocks_per_stage[0]` defines the number of blocks at the + current stage and `blocks_per_stage[1:]` is used at further stages. + stagewise_downsample: bool, whether or not to downsample before passing + inputs to the next stage. + encoder_decoder_shortcut: bool, whether or not to use shortcut + connections between encoder and decoder. + """ + + super(EncoderDecoderBlock, self).__init__() + + out_channels = channel_dims[0] + out_channels_downsampled = channel_dims[1] + + self.encoder_decoder_shortcut = encoder_decoder_shortcut + + if encoder_decoder_shortcut: + self.merge_features = tf.keras.layers.Add() + self.encoder_block1 = _make_repeated_residual_blocks( + out_channels=out_channels, num_blocks=blocks_per_stage[0], + initial_stride=1) + + initial_stride = 2 if stagewise_downsample else 1 + self.encoder_block2 = _make_repeated_residual_blocks( + out_channels=out_channels_downsampled, + num_blocks=blocks_per_stage[0], initial_stride=initial_stride, + initial_skip_conv=out_channels != out_channels_downsampled) + + if num_stages > 1: + self.inner_block = EncoderDecoderBlock(num_stages - 1, channel_dims[1:], + blocks_per_stage[1:], + stagewise_downsample=stagewise_downsample, + encoder_decoder_shortcut=encoder_decoder_shortcut) + else: + self.inner_block = _make_repeated_residual_blocks( + out_channels=out_channels_downsampled, + num_blocks=blocks_per_stage[1]) + + self.decoder_block = _make_repeated_residual_blocks( + residual_channels=out_channels_downsampled, + out_channels=out_channels, num_blocks=blocks_per_stage[0]) + + self.upsample = tf.keras.layers.UpSampling2D(initial_stride) + + def call(self, inputs): + + if self.encoder_decoder_shortcut: + encoded_outputs = _apply_blocks(inputs, self.encoder_block1) + encoded_downsampled_outputs = _apply_blocks(inputs, self.encoder_block2) + inner_block_outputs = _apply_blocks( + encoded_downsampled_outputs, self.inner_block) + + decoded_outputs = _apply_blocks(inner_block_outputs, self.decoder_block) + upsampled_outputs = self.upsample(decoded_outputs) + + if self.encoder_decoder_shortcut: + return self.merge_features([encoded_outputs, upsampled_outputs]) + else: + return upsampled_outputs + + +class HourglassNetwork(tf.keras.Model): + """The hourglass network.""" + + def __init__(self, num_stages, input_channel_dims, channel_dims_per_stage, + blocks_per_stage, num_hourglasses, initial_downsample=True, + stagewise_downsample=True, encoder_decoder_shortcut=True): + """Intializes the feature extractor. + Args: + num_stages: int, Number of stages in the network. At each stage we have 2 + encoder and 1 decoder blocks. The second encoder block downsamples the + input. + input_channel_dims: int, the number of channels in the input conv blocks. + channel_dims_per_stage: int list, the output channel dimensions of each + stage in the hourglass network. + blocks_per_stage: int list, number of residual blocks to use at each + stage in the hourglass network + num_hourglasses: int, number of hourglas networks to stack + sequentially. + initial_downsample: bool, if set, downsamples the input by a factor of 4 + before applying the rest of the network. Downsampling is done with a 7x7 + convolution kernel, otherwise a 3x3 kernel is used. + stagewise_downsample: bool, whether or not to downsample before passing + inputs to the next stage. + encoder_decoder_shortcut: bool, whether or not to use shortcut + connections between encoder and decoder. + """ + + super(HourglassNetwork, self).__init__() + + self.num_hourglasses = num_hourglasses + self.initial_downsample = initial_downsample + if initial_downsample: + self.downsample_input = InputDownsampleBlock( + out_channels_initial_conv=input_channel_dims, + out_channels_residual_block=channel_dims_per_stage[0] + ) + else: + self.conv_input = InputConvBlock( + out_channels_initial_conv=input_channel_dims, + out_channels_residual_block=channel_dims_per_stage[0] + ) + + self.hourglass_network = [] + self.output_conv = [] + for _ in range(self.num_hourglasses): + self.hourglass_network.append( + EncoderDecoderBlock( + num_stages=num_stages, channel_dims=channel_dims_per_stage, + blocks_per_stage=blocks_per_stage, + stagewise_downsample=stagewise_downsample, + encoder_decoder_shortcut=encoder_decoder_shortcut) + ) + self.output_conv.append( + ConvolutionalBlock(kernel_size=3, + out_channels=channel_dims_per_stage[0]) + ) + + self.intermediate_conv1 = [] + self.intermediate_conv2 = [] + self.intermediate_residual = [] + + for _ in range(self.num_hourglasses - 1): + self.intermediate_conv1.append( + ConvolutionalBlock( + kernel_size=1, out_channels=channel_dims_per_stage[0], relu=False) + ) + self.intermediate_conv2.append( + ConvolutionalBlock( + kernel_size=1, out_channels=channel_dims_per_stage[0], relu=False) + ) + self.intermediate_residual.append( + ResidualBlock(out_channels=channel_dims_per_stage[0]) + ) + + self.intermediate_relu = tf.keras.layers.ReLU() + + def call(self, inputs): + + if self.initial_downsample: + inputs = self.downsample_input(inputs) + else: + inputs = self.conv_input(inputs) + + outputs = [] + + for i in range(self.num_hourglasses): + + hourglass_output = self.hourglass_network[i](inputs) + + output = self.output_conv[i](hourglass_output) + outputs.append(output) + + if i < self.num_hourglasses - 1: + secondary_output = (self.intermediate_conv1[i](inputs) + + self.intermediate_conv2[i](output)) + secondary_output = self.intermediate_relu(secondary_output) + inputs = self.intermediate_residual[i](secondary_output) + + return outputs + + @property + def out_stride(self): + """The stride in the output image of the network.""" + return 4 + + @property + def num_feature_outputs(self): + """Ther number of feature outputs returned by the feature extractor.""" + return self.num_hourglasses + + +def _layer_depth(layer): + """Compute depth of Conv/Residual blocks or lists of them.""" + + if isinstance(layer, list): + return sum([_layer_depth(l) for l in layer]) + + elif isinstance(layer, ConvolutionalBlock): + return 1 + + elif isinstance(layer, ResidualBlock): + return 2 + + else: + raise ValueError('Unknown layer - {}'.format(layer)) + +def hourglass_104(): + """The Hourglass-104 backbone. + The architecture parameters are taken from [1]. + Returns: + network: An HourglassNetwork object implementing the Hourglass-104 + backbone. + [1]: https://arxiv.org/abs/1904.07850 + """ + + return HourglassNetwork( + input_channel_dims=128, + channel_dims_per_stage=[256, 256, 384, 384, 384, 512], + num_hourglasses=2, + num_stages=5, + blocks_per_stage=[2, 2, 2, 2, 2, 4], + ) diff --git a/centernet/validation/odapi_detection_generator.py b/centernet/validation/odapi_detection_generator.py new file mode 100644 index 000000000..b186cb923 --- /dev/null +++ b/centernet/validation/odapi_detection_generator.py @@ -0,0 +1,212 @@ +import tensorflow.compat.v1 as tf + +PEAK_EPSILON = 1e-6 + +class ODAPIDetectionGenerator(tf.keras.Model): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def call(self, inputs): + object_center_prob = tf.nn.sigmoid(inputs['ct_heatmaps'][-1]) + + detection_scores, y_indices, x_indices, channel_indices = ( + top_k_feature_map_locations( + object_center_prob, max_pool_kernel_size=3, + k=100)) + multiclass_scores = tf.gather_nd( + object_center_prob, tf.stack([y_indices, x_indices], -1), batch_dims=1) + + num_detections = tf.reduce_sum(tf.to_int32(detection_scores > 0), axis=1) + boxes_strided = None + + boxes_strided = ( + prediction_tensors_to_boxes(y_indices, x_indices, + inputs['ct_size'][-1], + inputs['ct_offset'][-1])) + boxes = convert_strided_predictions_to_normalized_boxes( + boxes_strided, 4) + return boxes + +def row_col_channel_indices_from_flattened_indices(indices, num_cols, + num_channels): + """Computes row, column and channel indices from flattened indices. + Args: + indices: An integer tensor of any shape holding the indices in the flattened + space. + num_cols: Number of columns in the image (width). + num_channels: Number of channels in the image. + Returns: + row_indices: The row indices corresponding to each of the input indices. + Same shape as indices. + col_indices: The column indices corresponding to each of the input indices. + Same shape as indices. + channel_indices. The channel indices corresponding to each of the input + indices. + """ + # Be careful with this function when running a model in float16 precision + # (e.g. TF.js with WebGL) because the array indices may not be represented + # accurately if they are too large, resulting in incorrect channel indices. + # See: + # https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Precision_limitations_on_integer_values + # + # Avoid using mod operator to make the ops more easy to be compatible with + # different environments, e.g. WASM. + row_indices = (indices // num_channels) // num_cols + col_indices = (indices // num_channels) - row_indices * num_cols + channel_indices_temp = indices // num_channels + channel_indices = indices - channel_indices_temp * num_channels + + return row_indices, col_indices, channel_indices + +def combined_static_and_dynamic_shape(tensor): + """Returns a list containing static and dynamic values for the dimensions. + Returns a list of static and dynamic values for shape dimensions. This is + useful to preserve static shapes when available in reshape operation. + Args: + tensor: A tensor of any type. + Returns: + A list of size tensor.shape.ndims containing integers or a scalar tensor. + """ + static_tensor_shape = tensor.shape.as_list() + dynamic_tensor_shape = tf.shape(tensor) + combined_shape = [] + for index, dim in enumerate(static_tensor_shape): + if dim is not None: + combined_shape.append(dim) + else: + combined_shape.append(dynamic_tensor_shape[index]) + return combined_shape + +def _get_shape(tensor, num_dims): + assert len(tensor.shape.as_list()) == num_dims + return combined_static_and_dynamic_shape(tensor) + +def _to_float32(x): + return tf.cast(x, tf.float32) + +def convert_strided_predictions_to_normalized_boxes(boxes, stride): + """Converts predictions in the output space to normalized boxes. + Boxes falling outside the valid image boundary are clipped to be on the + boundary. + Args: + boxes: A tensor of shape [batch_size, num_boxes, 4] holding the raw + coordinates of boxes in the model's output space. + stride: The stride in the output space. + true_image_shapes: A tensor of shape [batch_size, 3] representing the true + shape of the input not considering padding. + Returns: + boxes: A tensor of shape [batch_size, num_boxes, 4] representing the + coordinates of the normalized boxes. + """ + # Note: We use tf ops instead of functions in box_list_ops to make this + # function compatible with dynamic batch size. + boxes = boxes * stride + boxes = boxes / 512 + boxes = tf.clip_by_value(boxes, 0.0, 1.0) + return boxes + +def _multi_range(limit, + value_repetitions=1, + range_repetitions=1, + dtype=tf.int32): + return tf.reshape( + tf.tile( + tf.expand_dims(tf.range(limit, dtype=dtype), axis=-1), + multiples=[range_repetitions, value_repetitions]), [-1]) + +def top_k_feature_map_locations(feature_map, max_pool_kernel_size=3, k=100, + per_channel=False): + + if not max_pool_kernel_size or max_pool_kernel_size == 1: + feature_map_peaks = feature_map + else: + feature_map_max_pool = tf.nn.max_pool( + feature_map, ksize=max_pool_kernel_size, strides=1, padding='SAME') + + feature_map_peak_mask = tf.math.abs( + feature_map - feature_map_max_pool) < PEAK_EPSILON + + # Zero out everything that is not a peak. + feature_map_peaks = ( + feature_map * _to_float32(feature_map_peak_mask)) + + batch_size, _, width, num_channels = _get_shape(feature_map, 4) + + if per_channel: + if k == 1: + feature_map_flattened = tf.reshape( + feature_map_peaks, [batch_size, -1, num_channels]) + scores = tf.math.reduce_max(feature_map_flattened, axis=1) + peak_flat_indices = tf.math.argmax( + feature_map_flattened, axis=1, output_type=tf.dtypes.int32) + peak_flat_indices = tf.expand_dims(peak_flat_indices, axis=-1) + else: + # Perform top k over batch and channels. + feature_map_peaks_transposed = tf.transpose(feature_map_peaks, + perm=[0, 3, 1, 2]) + feature_map_peaks_transposed = tf.reshape( + feature_map_peaks_transposed, [batch_size, num_channels, -1]) + scores, peak_flat_indices = tf.math.top_k( + feature_map_peaks_transposed, k=k) + # Convert the indices such that they represent the location in the full + # (flattened) feature map of size [batch, height * width * channels]. + channel_idx = tf.range(num_channels)[tf.newaxis, :, tf.newaxis] + peak_flat_indices = num_channels * peak_flat_indices + channel_idx + scores = tf.reshape(scores, [batch_size, -1]) + peak_flat_indices = tf.reshape(peak_flat_indices, [batch_size, -1]) + else: + if k == 1: + feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) + scores = tf.math.reduce_max(feature_map_peaks_flat, axis=1, keepdims=True) + peak_flat_indices = tf.expand_dims(tf.math.argmax( + feature_map_peaks_flat, axis=1, output_type=tf.dtypes.int32), axis=-1) + else: + feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) + scores, peak_flat_indices = tf.math.top_k(feature_map_peaks_flat, k=k) + + # Get x, y and channel indices corresponding to the top indices in the flat + # array. + y_indices, x_indices, channel_indices = ( + row_col_channel_indices_from_flattened_indices( + peak_flat_indices, width, num_channels)) + return scores, y_indices, x_indices, channel_indices + +def prediction_tensors_to_boxes(y_indices, x_indices, height_width_predictions, + offset_predictions): + batch_size, num_boxes = _get_shape(y_indices, 2) + _, height, width, _ = _get_shape(height_width_predictions, 4) + height, width = tf.cast(height, tf.float32), tf.cast(width, tf.float32) + + # TF Lite does not support tf.gather with batch_dims > 0, so we need to use + # tf_gather_nd instead and here we prepare the indices for that. + combined_indices = tf.stack([ + _multi_range(batch_size, value_repetitions=num_boxes), + tf.reshape(y_indices, [-1]), + tf.reshape(x_indices, [-1]) + ], axis=1) + + new_height_width = tf.gather_nd(height_width_predictions, combined_indices) + new_height_width = tf.reshape(new_height_width, [batch_size, num_boxes, 2]) + + new_offsets = tf.gather_nd(offset_predictions, combined_indices) + offsets = tf.reshape(new_offsets, [batch_size, num_boxes, 2]) + + y_indices = _to_float32(y_indices) + x_indices = _to_float32(x_indices) + + height_width = tf.maximum(new_height_width, 0) + heights, widths = tf.unstack(height_width, axis=2) + y_offsets, x_offsets = tf.unstack(offsets, axis=2) + + ymin = y_indices + y_offsets - heights / 2.0 + xmin = x_indices + x_offsets - widths / 2.0 + ymax = y_indices + y_offsets + heights / 2.0 + xmax = x_indices + x_offsets + widths / 2.0 + + ymin = tf.clip_by_value(ymin, 0., height) + xmin = tf.clip_by_value(xmin, 0., width) + ymax = tf.clip_by_value(ymax, 0., height) + xmax = tf.clip_by_value(xmax, 0., width) + boxes = tf.stack([ymin, xmin, ymax, xmax], axis=2) + + return boxes From 904f1645f08bd7229099af66bb0e0d3af9e49ded Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 29 Apr 2021 11:19:43 -0400 Subject: [PATCH 125/132] documentation + config cleanup --- centernet/configs/centernet.py | 8 +- .../experiments/centernet-eval-tpu.yaml | 9 +- .../config_classes.py | 27 +++ .../differential_testing/load_weights.py | 173 ++++++++++++++++++ .../odapi_backbone.py | 0 .../differential_testing/odapi_decoder.py | 109 +++++++++++ .../odapi_detection_generator.py | 8 +- centernet/modeling/CenterNet.py | 8 +- .../modeling/layers/detection_generator.py | 54 ++---- .../utils/weight_utils/config_classes.py | 20 +- centernet/utils/weight_utils/load_weights.py | 5 + centernet/utils/weight_utils/tf_to_dict.py | 5 + centernet/validation/load_weights.py | 78 -------- 13 files changed, 379 insertions(+), 125 deletions(-) rename centernet/{validation => differential_testing}/config_classes.py (87%) create mode 100644 centernet/differential_testing/load_weights.py rename centernet/{validation => differential_testing}/odapi_backbone.py (100%) create mode 100644 centernet/differential_testing/odapi_decoder.py rename centernet/{validation => differential_testing}/odapi_detection_generator.py (98%) delete mode 100644 centernet/validation/load_weights.py diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 5309b4ae1..19d9e042e 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -161,12 +161,12 @@ class CenterNetLayer(hyperparams.Config): max_detections: int = 100 peak_error: float = 1e-6 peak_extract_kernel_size: int = 3 - use_nms: bool = True - center_thresh: float = 0.1 - iou_thresh: float = 0.4 class_offset: int = 1 - net_down_scale: int = 4 + net_down_scale: int = 4 input_image_dims: int = 512 + use_nms: bool = False + nms_pre_thresh: float = 0.1 + nms_thresh: float = 0.4 use_reduction_sum: bool = True @dataclasses.dataclass diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index 917198844..e9df99153 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -17,10 +17,13 @@ task: max_detections: 100 peak_error: 0.000001 peak_extract_kernel_size: 3 - use_nms: false - center_thresh: 0.1 - iou_thresh: 0.4 class_offset: 1 + net_down_scale: 4 + input_image_dims: 512 + use_nms: False + nms_pre_thresh: 0.1 + nms_thresh: 0.4 + use_reduction_sum: True validation_data: input_path: 'gs://tensorflow2/coco_records/val/2017*' global_batch_size: 8 diff --git a/centernet/validation/config_classes.py b/centernet/differential_testing/config_classes.py similarity index 87% rename from centernet/validation/config_classes.py rename to centernet/differential_testing/config_classes.py index 52183f9a8..d5f3e40ec 100644 --- a/centernet/validation/config_classes.py +++ b/centernet/differential_testing/config_classes.py @@ -175,3 +175,30 @@ def load_weights(self, layer): n_weights += self.load_block_weights(inner_layer, inner_weights_dict) return n_weights + +@dataclass +class ODAPIdecoderConvCFG(Config): + weights_dict: Dict = field(repr=False, default=None) + + conv_1_weights: np.array = field(repr=False, default=None) + conv_2_bias: np.array = field(repr=False, default=None) + + conv_2_weights: np.array = field(repr=False, default=None) + conv_1_bias: np.array = field(repr=False, default=None) + + def __post_init__(self): + conv_1_weights_dict = self.weights_dict['layer_with_weights-0'] + conv_2_weights_dict = self.weights_dict['layer_with_weights-1'] + + self.conv_1_weights = conv_1_weights_dict['kernel'] + self.conv_1_bias = conv_1_weights_dict['bias'] + self.conv_2_weights = conv_2_weights_dict['kernel'] + self.conv_2_bias = conv_2_weights_dict['bias'] + + def get_weights(self): + return [ + self.conv_1_weights, + self.conv_1_bias, + self.conv_2_weights, + self.conv_2_bias + ] diff --git a/centernet/differential_testing/load_weights.py b/centernet/differential_testing/load_weights.py new file mode 100644 index 000000000..0eff6aa74 --- /dev/null +++ b/centernet/differential_testing/load_weights.py @@ -0,0 +1,173 @@ +import numpy as np +import tensorflow as tf + +from centernet.configs.centernet import CenterNetTask +from centernet.differential_testing.config_classes import ( + ODAPIconvBnCFG, ODAPIdecoderConvCFG, ODAPIhourglassCFG, + ODAPIresidualBlockCFG) +from centernet.differential_testing.odapi_backbone import hourglass_104 +from centernet.differential_testing.odapi_decoder import \ + _construct_prediction_heads +from centernet.differential_testing.odapi_detection_generator import \ + ODAPIDetectionGenerator +from centernet.modeling.CenterNet import build_centernet +from centernet.utils.weight_utils.load_weights import ( + get_model_weights_as_dict, load_weights_backbone, load_weights_model) + +CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' + +def load_weights_odapi_backbone(weights_dict): + odapi_backbone = hourglass_104() + odapi_backbone.build(input_shape=[1, 512, 512, 3]) + + # load weights for downsample block (comprised of a conv and res block) + downsample_conv_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['downsample_input']['conv_block']) + downsample_res_cfg = ODAPIresidualBlockCFG(weights_dict=weights_dict['downsample_input']['residual_block']) + downsample_conv_cfg.load_weights(odapi_backbone.downsample_input.conv_block) + downsample_res_cfg.load_weights(odapi_backbone.downsample_input.residual_block) + + # load weights for encoder/decoder blocks + hourglass_0_cfg = ODAPIhourglassCFG(weights_dict=weights_dict['hourglass_network']['0']) + hourglass_1_cfg = ODAPIhourglassCFG(weights_dict=weights_dict['hourglass_network']['1']) + hourglass_0_cfg.load_weights(odapi_backbone.hourglass_network[0]) + hourglass_1_cfg.load_weights(odapi_backbone.hourglass_network[1]) + + # load weights for hourglass output conv blocks + out_conv_0_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['output_conv']['0']) + out_conv_1_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['output_conv']['1']) + out_conv_0_cfg.load_weights(odapi_backbone.output_conv[0]) + out_conv_1_cfg.load_weights(odapi_backbone.output_conv[1]) + + # load weights for intermediate conv and residual blocks + inter_conv_1_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['intermediate_conv1']['0']) + inter_conv_2_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['intermediate_conv2']['0']) + inter_res_2_cfg = ODAPIresidualBlockCFG(weights_dict=weights_dict['intermediate_residual']['0']) + inter_conv_1_cfg.load_weights(odapi_backbone.intermediate_conv1[0]) + inter_conv_2_cfg.load_weights(odapi_backbone.intermediate_conv2[0]) + inter_res_2_cfg.load_weights(odapi_backbone.intermediate_residual[0]) + + return odapi_backbone + +def load_weights_odapi_decoder(weights_dict): + odapi_decoder = _construct_prediction_heads(90, 2, -2.19) + for key in odapi_decoder.keys(): + for head in odapi_decoder[key]: + head.build(input_shape=[1, 128, 128, 256]) + + # load weights for downsample block (comprised of a conv and res block) + center_0 = ODAPIdecoderConvCFG(weights_dict=weights_dict['object_center']['0']) + center_1 = ODAPIdecoderConvCFG(weights_dict=weights_dict['object_center']['1']) + offset_0 = ODAPIdecoderConvCFG(weights_dict=weights_dict['box.Soffset']['0']) + offset_1 = ODAPIdecoderConvCFG(weights_dict=weights_dict['box.Soffset']['1']) + scale_0 = ODAPIdecoderConvCFG(weights_dict=weights_dict['box.Sscale']['0']) + scale_1 = ODAPIdecoderConvCFG(weights_dict=weights_dict['box.Sscale']['1']) + + center_0.load_weights(odapi_decoder['ct_heatmaps'][0]) + center_1.load_weights(odapi_decoder['ct_heatmaps'][1]) + offset_0.load_weights(odapi_decoder['ct_offset'][0]) + offset_1.load_weights(odapi_decoder['ct_offset'][1]) + scale_0.load_weights(odapi_decoder['ct_size'][0]) + scale_1.load_weights(odapi_decoder['ct_size'][1]) + + return odapi_decoder + +@tf.function +def compare_det_gen(model_1, model_2, input): + out_1 = model_1(input) + out_2 = model_2(input) + + return out_1, out_2 + +@tf.function +def compare_decoder(model_1, prediction_heads, input): + out_1 = model_1(input) + out_2 = dict() + + for key in prediction_heads.keys(): + out_2[key] = [prediction_heads[key][i](input[i]) for i in range(len(input))] + + return out_1, out_2 + +@tf.function +def compare_backbone(model_1, model_2, input): + out_1 = model_1(input) + out_2 = model_2(input) + return out_1, out_2 + +def differential_test_backbone(tfmg_backbone, odapi_backbone): + print("\n Differential test for backbone \n") + test_input = tf.random.uniform( + shape=[1, 512, 512, 3], minval=0, maxval=1, dtype=tf.dtypes.float32, seed=1439 + ) + + tfmg_output, odapi_output = compare_backbone(odapi_backbone, tfmg_backbone, test_input) + tfmg_hm1 = tfmg_output[0] + tfmg_hm2 = tfmg_output[1] + odapi_hm1 = odapi_output[0] + odapi_hm2 = odapi_output[1] + print("Difference between final output feature maps: ", tf.math.reduce_sum(odapi_hm2-tfmg_hm2)) + print("Difference between intermediate hourglass feature maps: ", tf.math.reduce_sum(odapi_hm1-tfmg_hm1)) + +def differential_test_decoder(tfmg_decoder, odapi_decoder): + print("\n Differential test for decoder \n") + test_input = [ + tf.random.uniform(shape=[1, 128, 128, 256], minval=0, maxval=1, + dtype=tf.dtypes.float32, seed=41965) for _ in range(2)] + + tfmg_output, odapi_output = compare_decoder(tfmg_decoder, odapi_decoder, test_input) + + for key in tfmg_output.keys(): + print("For key: {}, difference between first decoded map: ".format(key)) + print(tf.math.reduce_sum(tfmg_output[key][0] - odapi_output[key][0])) + print("For key: {}, difference between second decoded map: ".format(key)) + print(tf.math.reduce_sum(tfmg_output[key][1] - odapi_output[key][1])) + +def differential_test_det_gen(tfmg_det_gen, odapi_det_gen): + print("\n Differential test for detection generator \n") + ct_heatmaps = [ + tf.random.uniform(shape=[1, 128, 128, 90], minval=0, maxval=1, + dtype=tf.dtypes.float32, seed=475) for _ in range(2)] + offset_heatmaps = [ + tf.random.uniform(shape=[1, 128, 128, 2], minval=0, maxval=1, + dtype=tf.dtypes.float32, seed=425) for _ in range(2)] + size_heatmaps = [ + tf.random.uniform(shape=[1, 128, 128, 2], minval=0, maxval=1, + dtype=tf.dtypes.float32, seed=145) for _ in range(2)] + + test_input = { + 'ct_heatmaps': ct_heatmaps, + 'ct_offset': offset_heatmaps, + 'ct_size': size_heatmaps + } + + tfmg_output, odapi_output = compare_det_gen(model.filter, odapi_det_gen, test_input) + print("all bounding box coodinate differences:", tf.math.reduce_sum(tfmg_output['bbox'] - odapi_output['bbox'])) + print("all class prediction difference: ", tf.math.reduce_sum(tfmg_output['classes'] - odapi_output['classes'])) + print("confidence score prediction difference: ", tf.math.reduce_sum(tfmg_output['confidence'] - odapi_output['confidence'])) + print("number detection difference: ", tf.math.reduce_sum(tfmg_output['num_dets'] - odapi_output['num_dets'])) + +if __name__ == '__main__': + # testing if the output between our backbone and the ODAPI backbone matches + weights_dict, n_weights = get_model_weights_as_dict(CKPT_PATH) + backbone_weights_dict = weights_dict['model']['_feature_extractor']['_network'] + decoder_weights_dict = weights_dict['model']['_prediction_head_dict'] + + # load weights into odapi backbone + odapi_backbone = load_weights_odapi_backbone(backbone_weights_dict) + + # load weights into odapi decoder + odapi_decoder = load_weights_odapi_decoder(decoder_weights_dict) + + # create odapi detection generator + odapi_det_gen = ODAPIDetectionGenerator() + + # load weights into tfmg centernet model + input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) + config = CenterNetTask() + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + + differential_test_backbone(model.backbone, odapi_backbone) + differential_test_decoder(model.decoder, odapi_decoder) + differential_test_det_gen(model.filter, odapi_det_gen) diff --git a/centernet/validation/odapi_backbone.py b/centernet/differential_testing/odapi_backbone.py similarity index 100% rename from centernet/validation/odapi_backbone.py rename to centernet/differential_testing/odapi_backbone.py diff --git a/centernet/differential_testing/odapi_decoder.py b/centernet/differential_testing/odapi_decoder.py new file mode 100644 index 000000000..14acb8a58 --- /dev/null +++ b/centernet/differential_testing/odapi_decoder.py @@ -0,0 +1,109 @@ +import tensorflow as tf + +NUM_SIZE_CHANNELS = 2 +NUM_OFFSET_CHANNELS = 2 + +def _construct_prediction_heads(num_classes, num_feature_outputs, + class_prediction_bias_init): + """Constructs the prediction heads based on the specific parameters. + Args: + num_classes: An integer indicating how many classes in total to predict. + num_feature_outputs: An integer indicating how many feature outputs to use + for calculating the loss. The Objects as Points paper attaches loss + functions to multiple (`num_feature_outputs`) feature maps in the the + backbone. E.g. for the hourglass backbone, `num_feature_outputs` is 2. + class_prediction_bias_init: float, the initial value of bias in the + convolutional kernel of the class prediction head. If set to None, the + bias is initialized with zeros. + Returns: + A dictionary of keras modules generated by calling make_prediction_net + function. It will also create and set a private member of the class when + learning the tracking task. + """ + prediction_heads = {} + prediction_heads['ct_heatmaps'] = _make_prediction_net_list( + num_feature_outputs, + num_classes, + kernel_sizes=(3), + num_filters=(256), + bias_fill=class_prediction_bias_init, + name='center') + + prediction_heads['ct_size'] = _make_prediction_net_list( + num_feature_outputs, NUM_SIZE_CHANNELS, name='box_scale') + prediction_heads['ct_offset'] = _make_prediction_net_list( + num_feature_outputs, NUM_OFFSET_CHANNELS, name='box_offset') + + return prediction_heads + + +def make_prediction_net(num_out_channels, kernel_sizes=(3), num_filters=(256), + bias_fill=None, use_depthwise=False, name=None): + """Creates a network to predict the given number of output channels. + This function is intended to make the prediction heads for the CenterNet + meta architecture. + Args: + num_out_channels: Number of output channels. + kernel_sizes: A list representing the sizes of the conv kernel in the + intermediate layer. Note that the length of the list indicates the number + of intermediate conv layers and it must be the same as the length of the + num_filters. + num_filters: A list representing the number of filters in the intermediate + conv layer. Note that the length of the list indicates the number of + intermediate conv layers. + bias_fill: If not None, is used to initialize the bias in the final conv + layer. + use_depthwise: If true, use SeparableConv2D to construct the Sequential + layers instead of Conv2D. + name: Optional name for the prediction net. + Returns: + net: A keras module which when called on an input tensor of size + [batch_size, height, width, num_in_channels] returns an output + of size [batch_size, height, width, num_out_channels] + """ + if isinstance(kernel_sizes, int) and isinstance(num_filters, int): + kernel_sizes = [kernel_sizes] + num_filters = [num_filters] + assert len(kernel_sizes) == len(num_filters) + if use_depthwise: + conv_fn = tf.keras.layers.SeparableConv2D + else: + conv_fn = tf.keras.layers.Conv2D + + # We name the convolution operations explicitly because Keras, by default, + # uses different names during training and evaluation. By setting the names + # here, we avoid unexpected pipeline breakage in TF1. + out_conv = tf.keras.layers.Conv2D( + num_out_channels, + kernel_size=1) + + if bias_fill is not None: + out_conv.bias_initializer = tf.keras.initializers.constant(bias_fill) + + layers = [] + for idx, (kernel_size, + num_filter) in enumerate(zip(kernel_sizes, num_filters)): + layers.append( + conv_fn( + num_filter, + kernel_size=kernel_size, + padding='same')) + layers.append(tf.keras.layers.ReLU()) + layers.append(out_conv) + net = tf.keras.Sequential(layers, name=name) + return net + +def _make_prediction_net_list(num_feature_outputs, num_out_channels, + kernel_sizes=(3), num_filters=(256), + bias_fill=None, name=None): + prediction_net_list = [] + for i in range(num_feature_outputs): + prediction_net_list.append( + make_prediction_net( + num_out_channels, + kernel_sizes=kernel_sizes, + num_filters=num_filters, + bias_fill=bias_fill, + use_depthwise=False, + name='{}_{}'.format(name, i) if name else name)) + return prediction_net_list diff --git a/centernet/validation/odapi_detection_generator.py b/centernet/differential_testing/odapi_detection_generator.py similarity index 98% rename from centernet/validation/odapi_detection_generator.py rename to centernet/differential_testing/odapi_detection_generator.py index b186cb923..a87ef8709 100644 --- a/centernet/validation/odapi_detection_generator.py +++ b/centernet/differential_testing/odapi_detection_generator.py @@ -25,7 +25,13 @@ def call(self, inputs): inputs['ct_offset'][-1])) boxes = convert_strided_predictions_to_normalized_boxes( boxes_strided, 4) - return boxes + return { + 'bbox': boxes, + 'classes': channel_indices, + 'confidence': detection_scores, + 'num_dets': num_detections + } + def row_col_channel_indices_from_flattened_indices(indices, num_cols, num_channels): diff --git a/centernet/modeling/CenterNet.py b/centernet/modeling/CenterNet.py index 288fba536..4a94e18fb 100644 --- a/centernet/modeling/CenterNet.py +++ b/centernet/modeling/CenterNet.py @@ -70,12 +70,12 @@ def build_centernet_filter(model_config): return CenterNetLayer(max_detections=model_config.filter.max_detections, peak_error=model_config.filter.peak_error, peak_extract_kernel_size=model_config.filter.peak_extract_kernel_size, - use_nms=model_config.filter.use_nms, - center_thresh=model_config.filter.center_thresh, - iou_thresh=model_config.filter.iou_thresh, class_offset=model_config.filter.class_offset, net_down_scale=model_config.filter.net_down_scale, - input_image_dims=model_config.filter.input_image_dims) + input_image_dims=model_config.filter.input_image_dims, + use_nms=model_config.filter.use_nms, + nms_pre_thresh=model_config.filter.nms_pre_thresh, + nms_thresh=model_config.filter.nms_thresh) def build_centernet_head(model_config): return None diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index 976e15d45..d871b5769 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -17,12 +17,12 @@ def __init__(self, max_detections=100, peak_error=1e-6, peak_extract_kernel_size=3, - use_nms=False, - center_thresh=0.1, - iou_thresh=0.4, class_offset=1, net_down_scale=4, input_image_dims=512, + use_nms=False, + nms_pre_thresh=0.1, + nms_thresh=0.4, **kwargs): """ Args: @@ -36,9 +36,6 @@ def __init__(self, locations that have responses greater than its 8-connected neighbors use_nms: Boolean for whether or not to use non-maximum suppression to filter the bounding boxes. - center_thresh: A float that sets the threshold for a heatmap peak to be - considered a valid detection. - iou_thresh: A float for the threshold IOU when filtering bounding boxes. class_offset: An integer indicating to add an offset to the class prediction if the dataset labels have been shifted. net_down_scale: An integer that specifies @@ -56,21 +53,20 @@ def __init__(self, self._peak_error = peak_error self._peak_extract_kernel_size = peak_extract_kernel_size - # Non-maximum suppression parameters - self._use_nms = use_nms - self._center_thresh = center_thresh - self._iou_thresh = iou_thresh - + # Used for adjusting class prediction self._class_offset = class_offset # Box normalization parameters self._net_down_scale = net_down_scale self._input_image_dims = input_image_dims + + self._use_nms = use_nms + self._nms_pre_thresh = nms_pre_thresh + self._nms_thresh = nms_thresh def process_heatmap(self, feature_map, - kernel_size, - center_thresh): + kernel_size): """ Processes the heatmap into peaks for box selection. Given a heatmap, this function first masks out nearby heatmap locations of @@ -82,14 +78,12 @@ def process_heatmap(self, feature_map: A Tensor with shape [batch_size, height, width, num_classes] which is the center heatmap predictions. kernel_size: An integer value for max-pool kernel size. - center_thresh: The threshold value for valid center location scores. Returns: A Tensor with the same shape as the input but with non-valid center prediction locations masked out. """ - # all dtypes in this function are float32 or bool (for masks) feature_map = tf.math.sigmoid(feature_map) if not kernel_size or kernel_size == 1: feature_map_peaks = feature_map @@ -201,7 +195,6 @@ def get_top_k_peaks(self, channel indices corresponding to top_scores. """ # Flatten the entire prediction per batch - # Now the feature_map_peaks are in shape [batch_size, num_classes * width * height] feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1]) # top_scores and top_indices have shape [batch_size, k] @@ -216,7 +209,6 @@ def get_top_k_peaks(self, return top_scores, y_indices, x_indices, channel_indices def get_boxes(self, - detection_scores, y_indices, x_indices, channel_indices, @@ -228,8 +220,6 @@ def get_boxes(self, NOTE: Repurposed from Google OD API. Args: - detection_scores: A Tensor with shape [batch_size, k] containing the - top-k scores. y_indices: A Tensor with shape [batch_size, k] containing the top-k y-indices corresponding to top_scores. x_indices: A Tensor with shape [batch_size, k] containing the top-k @@ -247,8 +237,6 @@ def get_boxes(self, bounding box coordinates in [y_min, x_min, y_max, x_max] format. detection_classes: A Tensor with shape [batch_size, 100] that gives the class prediction for each box. - detection_scores: A Tensor with shape [batch_size, 100] that gives the - confidence score for each box. num_detections: Number of non-zero confidence detections made. """ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use @@ -296,7 +284,7 @@ class prediction for each box. xmax = tf.clip_by_value(xmax, 0., tf.cast(width, xmax.dtype)) boxes = tf.stack([ymin, xmin, ymax, xmax], axis=2) - return boxes, detection_classes, detection_scores + return boxes, detection_classes def convert_strided_predictions_to_normalized_boxes(self, boxes): @@ -321,29 +309,27 @@ def call(self, inputs): # Process heatmaps using 3x3 max pool and applying sigmoid peaks = self.process_heatmap(ct_heatmaps, - kernel_size=self._peak_extract_kernel_size, - center_thresh=self._center_thresh) + kernel_size=self._peak_extract_kernel_size) # Get top scores along with their x, y, and class scores, y_indices, x_indices, channel_indices = self.get_top_k_peaks(peaks, batch_size, width, num_channels, k=self._max_detections) # Parse the score and indices into bounding boxes - boxes, classes, scores = self.get_boxes(scores, - y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, self._max_detections) + boxes, classes = self.get_boxes(y_indices, x_indices, channel_indices, ct_sizes, ct_offsets, self._max_detections) # Normalize bounding boxes boxes = self.convert_strided_predictions_to_normalized_boxes(boxes) # Apply nms - if self._use_nms: - boxes = tf.expand_dims(boxes, axis=-2) - multi_class_scores = tf.gather_nd(peaks, - tf.stack([y_indices, x_indices], -1), batch_dims=1) - - boxes, _, scores = nms(boxes=boxes, classes=multi_class_scores, - confidence=scores, k=self._max_detections, limit_pre_thresh=True, - pre_nms_thresh=self._center_thresh, nms_thresh=self._iou_thresh) + # if self._use_nms: + # boxes = tf.expand_dims(boxes, axis=-2) + # multi_class_scores = tf.gather_nd(peaks, + # tf.stack([y_indices, x_indices], -1), batch_dims=1) + + # boxes, _, scores = nms(boxes=boxes, classes=multi_class_scores, + # confidence=scores, k=self._max_detections, limit_pre_thresh=True, + # pre_nms_thresh=0.1, nms_thresh=0.4) num_det = tf.reduce_sum(tf.cast(scores > 0, dtype=tf.int32), axis=1) diff --git a/centernet/utils/weight_utils/config_classes.py b/centernet/utils/weight_utils/config_classes.py index f9ee7c286..e5a8e9342 100644 --- a/centernet/utils/weight_utils/config_classes.py +++ b/centernet/utils/weight_utils/config_classes.py @@ -1,15 +1,33 @@ +""" +This file contains the layers (Config objects) that are used for parsing the +ODAPI checkpoint weights for CenterNet. + +Currently, the parser is incomplete and has only been tested on +CenterNet Hourglass-104 512x512. +""" + from abc import ABC, abstractmethod from dataclasses import dataclass, field from typing import ClassVar, Dict, List, Tuple import numpy as np +import tensorflow as tf class Config(ABC): def get_weights(self): + """ + This function generates the weights needed to be loaded into the layer. + """ return None - def load_weights(self, layer): + def load_weights(self, layer: tf.keras.layers.Layer) -> int: + """ + Given a layer, this function retrives the weights for that layer in an + appropriate format, and loads them into the layer. Additionally, + the number of weights loaded are returned. If the weights are in an + incorrect format, a ValueError will be raised by set_weights(). + """ weights = self.get_weights() layer.set_weights(weights) n_weights = 0 diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index e119342ba..0dbf04808 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -1,3 +1,8 @@ +""" +This file contains functions used to load the ODAPI CenterNet checkpoint +weights into CenterNet model. +""" + import numpy as np import tensorflow as tf diff --git a/centernet/utils/weight_utils/tf_to_dict.py b/centernet/utils/weight_utils/tf_to_dict.py index 83481d014..2ab60016d 100644 --- a/centernet/utils/weight_utils/tf_to_dict.py +++ b/centernet/utils/weight_utils/tf_to_dict.py @@ -1,3 +1,8 @@ +""" +This file contains functions used to convert a TF checkpoint into a dictionary, +which streamlines the way of loading weights from one model to another. +""" + import tensorflow as tf diff --git a/centernet/validation/load_weights.py b/centernet/validation/load_weights.py deleted file mode 100644 index d298d2ee3..000000000 --- a/centernet/validation/load_weights.py +++ /dev/null @@ -1,78 +0,0 @@ -import numpy as np -import tensorflow as tf - -from centernet.configs.centernet import CenterNetTask -from centernet.modeling.CenterNet import build_centernet -from centernet.utils.weight_utils.load_weights import ( - get_model_weights_as_dict, load_weights_backbone, load_weights_model) -from centernet.validation.config_classes import (ODAPIconvBnCFG, - ODAPIhourglassCFG, - ODAPIresidualBlockCFG) -from centernet.validation.odapi_backbone import hourglass_104 - -CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' - -def load_weights_odapi_backbone(weights_dict): - odapi_backbone = hourglass_104() - odapi_backbone.build(input_shape=[1, 512, 512, 3]) - - # load weights for downsample block (comprised of a conv and res block) - downsample_conv_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['downsample_input']['conv_block']) - downsample_res_cfg = ODAPIresidualBlockCFG(weights_dict=weights_dict['downsample_input']['residual_block']) - downsample_conv_cfg.load_weights(odapi_backbone.downsample_input.conv_block) - downsample_res_cfg.load_weights(odapi_backbone.downsample_input.residual_block) - - # load weights for encoder/decoder blocks - hourglass_0_cfg = ODAPIhourglassCFG(weights_dict=weights_dict['hourglass_network']['0']) - hourglass_1_cfg = ODAPIhourglassCFG(weights_dict=weights_dict['hourglass_network']['1']) - hourglass_0_cfg.load_weights(odapi_backbone.hourglass_network[0]) - hourglass_1_cfg.load_weights(odapi_backbone.hourglass_network[1]) - - # load weights for hourglass output conv blocks - out_conv_0_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['output_conv']['0']) - out_conv_1_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['output_conv']['1']) - out_conv_0_cfg.load_weights(odapi_backbone.output_conv[0]) - out_conv_1_cfg.load_weights(odapi_backbone.output_conv[1]) - - # load weights for intermediate conv and residual blocks - inter_conv_1_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['intermediate_conv1']['0']) - inter_conv_2_cfg = ODAPIconvBnCFG(weights_dict=weights_dict['intermediate_conv2']['0']) - inter_res_2_cfg = ODAPIresidualBlockCFG(weights_dict=weights_dict['intermediate_residual']['0']) - inter_conv_1_cfg.load_weights(odapi_backbone.intermediate_conv1[0]) - inter_conv_2_cfg.load_weights(odapi_backbone.intermediate_conv2[0]) - inter_res_2_cfg.load_weights(odapi_backbone.intermediate_residual[0]) - - return odapi_backbone - -@tf.function -def compare(model_1, model_2, input): - out_1 = model_1(input) - out_2 = model_2(input) - return out_1, out_2 - -if __name__ == '__main__': - # testing if the output between our backbone and the ODAPI backbone matches - weights_dict, n_weights = get_model_weights_as_dict(CKPT_PATH) - backbone_weights_dict = weights_dict['model']['_feature_extractor']['_network'] - - # load weights into odapi backbone - odapi_backbone = load_weights_odapi_backbone(backbone_weights_dict) - - # load weights into tfmg centernet model - input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) - config = CenterNetTask() - model, loss = build_centernet(input_specs=input_specs, - task_config=config, l2_regularization=0) - load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') - - test_input = tf.random.uniform( - shape=[1, 512, 512, 3], minval=0, maxval=1, dtype=tf.dtypes.float32, seed=45 - ) - - tfmg_output, odapi_output = compare(odapi_backbone, model.backbone, test_input) - tfmg_hm1 = tfmg_output[0] - tfmg_hm2 = tfmg_output[1] - odapi_hm1 = odapi_output[0] - odapi_hm2 = odapi_output[1] - print(tf.math.reduce_sum(odapi_hm2-tfmg_hm2)) - print(tf.math.reduce_sum(odapi_hm1-tfmg_hm1)) From 217903c45bd5ce4c81d38becbaeb9bd6d6efae68 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 30 Apr 2021 03:00:30 -0400 Subject: [PATCH 126/132] modeling documentation --- centernet/configs/backbones.py | 3 + centernet/configs/centernet.py | 2 +- centernet/modeling/backbones/hourglass.py | 168 ++-- .../modeling/layers/detection_generator.py | 16 +- centernet/modeling/layers/nn_blocks.py | 35 +- centernet/modeling/layers/subnormalization.py | 901 ------------------ centernet/utils/weight_utils/load_weights.py | 7 +- 7 files changed, 126 insertions(+), 1006 deletions(-) delete mode 100644 centernet/modeling/layers/subnormalization.py diff --git a/centernet/configs/backbones.py b/centernet/configs/backbones.py index a15d42972..6128ac9fa 100644 --- a/centernet/configs/backbones.py +++ b/centernet/configs/backbones.py @@ -32,6 +32,9 @@ class Hourglass(hyperparams.Config): default_factory=lambda: [2, 2, 2, 2, 2, 4]) num_hourglasses: int = 2 initial_downsample: bool = True + norm_momentum: float = 0.1 + norm_episilon: float = 1e-5 + @dataclasses.dataclass class Backbone(backbones.Backbone): diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 19d9e042e..211d50a65 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -164,7 +164,7 @@ class CenterNetLayer(hyperparams.Config): class_offset: int = 1 net_down_scale: int = 4 input_image_dims: int = 512 - use_nms: bool = False + use_nms: bool = True nms_pre_thresh: float = 0.1 nms_thresh: float = 0.4 use_reduction_sum: bool = True diff --git a/centernet/modeling/backbones/hourglass.py b/centernet/modeling/backbones/hourglass.py index 569042e96..e80d8a6b9 100644 --- a/centernet/modeling/backbones/hourglass.py +++ b/centernet/modeling/backbones/hourglass.py @@ -4,12 +4,8 @@ from centernet.configs import backbones as cfg from centernet.modeling.layers import nn_blocks -# from official.vision.beta.modeling.layers import \ -# nn_blocks as official_nn_blocks from utils import register -BATCH_NORM_MOMENTUM = 0.1 -BATCH_NORM_EPSILON = 1e-5 @tf.keras.utils.register_keras_serializable(package='centernet') class Hourglass(tf.keras.Model): @@ -18,27 +14,35 @@ class Hourglass(tf.keras.Model): """ def __init__( - self, - input_channel_dims: int, - channel_dims_per_stage: List[int], - blocks_per_stage: List[int], - num_hourglasses: int, - initial_downsample: bool = True, - input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 3]), - **kwargs): + self, + input_channel_dims: int, + channel_dims_per_stage: List[int], + blocks_per_stage: List[int], + num_hourglasses: int, + initial_downsample: bool = True, + norm_momentum=0.1, + norm_episilon=1e-5, + input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 3]), + **kwargs): """ Args: - channel_dims_per_stage: list of filter sizes for Residual blocks - blocks_per_stage: list of residual block repetitions per down/upsample - num_hourglasses: integer, number of hourglass modules in backbone - pre_layers: tf.keras layer to process input before stacked hourglasses + input_channel_dims: integer, number of filters used to downsample the + input image + channel_dims_per_stage: list, containing of number of filters for the + residual blocks in the hourglass blocks + blocks_per_stage: list of residual block repetitions to use in the + hourglass blocks + num_hourglasses: integer, number of hourglass blocks in backbone + initial_downsample: bool, whether or not to downsample the input + norm_momentum: float, momentum for the batch normalization layers + norm_episilon: float, epsilon for the batch normalization layers """ # yapf: disable input = tf.keras.layers.Input(shape=input_specs.shape[1:], name='input') inp_filters = channel_dims_per_stage[0] - # Create downsampling layers + # Downsample the input if initial_downsample: prelayer_kernel_size = 7 prelayer_strides = 2 @@ -46,81 +50,76 @@ def __init__( prelayer_kernel_size = 3 prelayer_strides = 1 - x_downsampled = nn_blocks.ConvBN(filters=input_channel_dims, - kernel_size=prelayer_kernel_size, - strides=prelayer_strides, - padding='valid', - activation='relu', - use_sync_bn=True, - norm_momentum=BATCH_NORM_MOMENTUM, - norm_epsilon=BATCH_NORM_EPSILON)(input) + x_downsampled = nn_blocks.CenterNetConvBN( + filters=input_channel_dims, + kernel_size=prelayer_kernel_size, + strides=prelayer_strides, + padding='valid', + activation='relu', + use_sync_bn=True, + norm_momentum=norm_momentum, + norm_epsilon=norm_episilon)(input) x_downsampled = nn_blocks.CenterNetResidualBlock( filters=inp_filters, use_projection=True, strides=prelayer_strides, use_sync_bn=True, - norm_momentum=BATCH_NORM_MOMENTUM, - norm_epsilon=BATCH_NORM_EPSILON)(x_downsampled) + norm_momentum=norm_momentum, + norm_epsilon=norm_episilon)(x_downsampled) - # Used for storing each hourglass heatmap output all_heatmaps = [] - for i in range(num_hourglasses): - # Create hourglass stacks + # Create an hourglass stack x_hg = nn_blocks.HourglassBlock( - channel_dims_per_stage=channel_dims_per_stage, - blocks_per_stage=blocks_per_stage, + channel_dims_per_stage=channel_dims_per_stage, + blocks_per_stage=blocks_per_stage, )(x_downsampled) - # cnvs - x_hg = nn_blocks.ConvBN( - filters=inp_filters, - kernel_size=(3, 3), - strides=(1, 1), - padding='valid', - activation='relu', - use_sync_bn=True, - norm_momentum=BATCH_NORM_MOMENTUM, - norm_epsilon=BATCH_NORM_EPSILON + x_hg = nn_blocks.CenterNetConvBN( + filters=inp_filters, + kernel_size=(3, 3), + strides=(1, 1), + padding='valid', + activation='relu', + use_sync_bn=True, + norm_momentum=norm_momentum, + norm_epsilon=norm_episilon )(x_hg) all_heatmaps.append(x_hg) - # between hourglasses, we insert intermediate layers + # Intermediate conv and residual layers between hourglasses if i < num_hourglasses - 1: - # cnvs_ - inter_hg_conv1 = nn_blocks.ConvBN( - filters=inp_filters, - kernel_size=(1, 1), - strides=(1, 1), - padding='same', - activation='identity', - use_sync_bn=True, - norm_momentum=BATCH_NORM_MOMENTUM, - norm_epsilon=BATCH_NORM_EPSILON + inter_hg_conv1 = nn_blocks.CenterNetConvBN( + filters=inp_filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + activation='identity', + use_sync_bn=True, + norm_momentum=norm_momentum, + norm_epsilon=norm_episilon )(x_downsampled) - # inters_ - inter_hg_conv2 = nn_blocks.ConvBN( - filters=inp_filters, - kernel_size=(1, 1), - strides=(1, 1), - padding='same', - activation='identity', - use_sync_bn=True, - norm_momentum=BATCH_NORM_MOMENTUM, - norm_epsilon=BATCH_NORM_EPSILON + inter_hg_conv2 = nn_blocks.CenterNetConvBN( + filters=inp_filters, + kernel_size=(1, 1), + strides=(1, 1), + padding='same', + activation='identity', + use_sync_bn=True, + norm_momentum=norm_momentum, + norm_epsilon=norm_episilon )(x_hg) x_downsampled = tf.keras.layers.Add()([inter_hg_conv1, inter_hg_conv2]) x_downsampled = tf.keras.layers.ReLU()(x_downsampled) - # inters x_downsampled = nn_blocks.CenterNetResidualBlock( - filters=inp_filters, use_projection=False, strides=1, - use_sync_bn=True, norm_momentum=BATCH_NORM_MOMENTUM, - norm_epsilon=BATCH_NORM_EPSILON + filters=inp_filters, use_projection=False, strides=1, + use_sync_bn=True, norm_momentum=norm_momentum, + norm_epsilon=norm_episilon )(x_downsampled) # yapf: enable @@ -131,15 +130,19 @@ def __init__( self._blocks_per_stage = blocks_per_stage self._num_hourglasses = num_hourglasses self._initial_downsample = initial_downsample + self._norm_momentum = norm_momentum + self._norm_episilon = norm_episilon self._output_specs = [hm.get_shape() for hm in all_heatmaps] def get_config(self): layer_config = { - 'input_channel_dims': self._input_channel_dims, - 'channel_dims_per_stage': self._channel_dims_per_stage, - 'blocks_per_stage': self._blocks_per_stage, - 'num_hourglasses': self._num_hourglasses, - 'initial_downsample': self._initial_downsample + 'input_channel_dims': self._input_channel_dims, + 'channel_dims_per_stage': self._channel_dims_per_stage, + 'blocks_per_stage': self._blocks_per_stage, + 'num_hourglasses': self._num_hourglasses, + 'initial_downsample': self._initial_downsample, + 'norm_momentum': self._norm_momentum, + 'norm_episilon': self._norm_episilon } layer_config.update(super().get_config()) return layer_config @@ -150,10 +153,9 @@ def output_specs(self): # @factory.register_backbone_builder('hourglass') @register.backbone('hourglass', cfg.Hourglass) -def build_hourglass( - input_specs: tf.keras.layers.InputSpec, - model_config, - l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: +def build_hourglass(input_specs: tf.keras.layers.InputSpec, + model_config, + l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model: """Builds Hourglass backbone from a config.""" backbone_type = model_config.backbone.type backbone_cfg = model_config.backbone.get() @@ -161,9 +163,11 @@ def build_hourglass( f'{backbone_type}') return Hourglass( - input_channel_dims=backbone_cfg.input_channel_dims, - channel_dims_per_stage=backbone_cfg.channel_dims_per_stage, - blocks_per_stage=backbone_cfg.blocks_per_stage, - num_hourglasses=backbone_cfg.num_hourglasses, - initial_downsample=backbone_cfg.initial_downsample, - input_specs=input_specs) + input_channel_dims=backbone_cfg.input_channel_dims, + channel_dims_per_stage=backbone_cfg.channel_dims_per_stage, + blocks_per_stage=backbone_cfg.blocks_per_stage, + num_hourglasses=backbone_cfg.num_hourglasses, + initial_downsample=backbone_cfg.initial_downsample, + norm_momentum=backbone_cfg.norm_momentum, + norm_episilon=backbone_cfg.norm_episilon, + input_specs=input_specs) diff --git a/centernet/modeling/layers/detection_generator.py b/centernet/modeling/layers/detection_generator.py index d871b5769..b336e767a 100644 --- a/centernet/modeling/layers/detection_generator.py +++ b/centernet/modeling/layers/detection_generator.py @@ -322,14 +322,14 @@ def call(self, inputs): boxes = self.convert_strided_predictions_to_normalized_boxes(boxes) # Apply nms - # if self._use_nms: - # boxes = tf.expand_dims(boxes, axis=-2) - # multi_class_scores = tf.gather_nd(peaks, - # tf.stack([y_indices, x_indices], -1), batch_dims=1) - - # boxes, _, scores = nms(boxes=boxes, classes=multi_class_scores, - # confidence=scores, k=self._max_detections, limit_pre_thresh=True, - # pre_nms_thresh=0.1, nms_thresh=0.4) + if self._use_nms: + boxes = tf.expand_dims(boxes, axis=-2) + multi_class_scores = tf.gather_nd(peaks, + tf.stack([y_indices, x_indices], -1), batch_dims=1) + + boxes, _, scores = nms(boxes=boxes, classes=multi_class_scores, + confidence=scores, k=self._max_detections, limit_pre_thresh=True, + pre_nms_thresh=0.1, nms_thresh=0.4) num_det = tf.reduce_sum(tf.cast(scores > 0, dtype=tf.int32), axis=1) diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index 73b0ff700..baa8255d4 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -1,13 +1,9 @@ import tensorflow as tf -from centernet.modeling.layers import subnormalization from official.modeling import tf_utils from official.vision.beta.modeling.layers import \ nn_blocks as official_nn_blocks -TPU_BASE = False -BATCH_NORM_MOMENTUM = 0.1 -BATCH_NORM_EPSILON = 1e-5 @tf.keras.utils.register_keras_serializable(package='centernet') class Identity(tf.keras.layers.Layer): @@ -37,7 +33,8 @@ def __init__(self, norm_momentum=0.99, norm_epsilon=0.001, **kwargs): - """A residual block with BN after convolutions. + """A residual block with BN after convolutions. Modified with padding for + the CenterNet model. Args: filters: `int` number of filters for the first two convolutions. Note that @@ -195,7 +192,7 @@ def call(self, inputs, training=None): return self._activation_fn(x + shortcut) @tf.keras.utils.register_keras_serializable(package='centernet') -class ConvBN(tf.keras.layers.Layer): +class CenterNetConvBN(tf.keras.layers.Layer): def __init__(self, filters=1, kernel_size=(1, 1), @@ -219,7 +216,7 @@ def __init__(self, such that it is compatiable with the CenterNet backbone. The Layer is a standards combination of Conv BatchNorm Activation, however, the use of bias in the conv is determined by the use of batch - normalization. + normalization. Modified with padding for the CenterNet model. Cross Stage Partial networks (CSPNets) were proposed in: [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang Chen, Jun-Wei Hsieh @@ -377,17 +374,23 @@ def __init__(self, channel_dims_per_stage, blocks_per_stage, strides=1, + norm_momentum=0.1, + norm_epsilon=1e-5, **kwargs): """ Args: channel_dims_per_stage: list of filter sizes for Residual blocks blocks_per_stage: list of residual block repetitions per down/upsample strides: integer, stride parameter to the Residual block + norm_momentum: float, momentum for the batch normalization layers + norm_episilon: float, epsilon for the batch normalization layers """ self._order = len(channel_dims_per_stage) - 1 self._channel_dims_per_stage = channel_dims_per_stage self._blocks_per_stage = blocks_per_stage self._strides = strides + self._norm_momentum = norm_momentum + self._norm_epsilon = norm_epsilon assert len(channel_dims_per_stage) == len(blocks_per_stage), 'filter ' \ 'size and residual block repetition lists must have the same length' @@ -401,7 +404,17 @@ def __init__(self, def make_repeated_residual_blocks(self, reps, out_channels, residual_channels=None, - initial_stride=1, initial_skip=False): + initial_stride=1): + """ + Args: + reps: int for desired number of residual blocks + out_channels: int, filter depth of the final residual block + residual_channels: int, filter depth for the first reps - 1 residual + blocks. If None, defaults to the same value as out_channels. If not + equal to out_channels, then uses a projection shortcut in the final + residual block + initial_stride: int, stride for the first residual block + """ blocks = [] if residual_channels is None: @@ -414,7 +427,7 @@ def make_repeated_residual_blocks(self, reps, out_channels, blocks.append(CenterNetResidualBlock( filters=residual_channels, strides=stride, use_projection=skip_conv, use_sync_bn=True, - norm_momentum=BATCH_NORM_MOMENTUM, norm_epsilon=BATCH_NORM_EPSILON)) + norm_momentum=self._norm_momentum, norm_epsilon=self._norm_epsilon)) if reps == 1: stride = initial_stride @@ -426,7 +439,7 @@ def make_repeated_residual_blocks(self, reps, out_channels, blocks.append(CenterNetResidualBlock( filters=out_channels, strides=stride, use_projection=skip_conv, use_sync_bn=True, - norm_momentum=BATCH_NORM_MOMENTUM, norm_epsilon=BATCH_NORM_EPSILON)) + norm_momentum=self._norm_momentum, norm_epsilon=self._norm_epsilon)) return tf.keras.Sequential(blocks) @@ -442,7 +455,7 @@ def build(self, input_shape): out_channels=self._filters) self.encoder_block2 = self.make_repeated_residual_blocks(reps=self._reps, - out_channels=self._filters_downsampled, initial_stride=2, initial_skip=self._filters != self._filters_downsampled) + out_channels=self._filters_downsampled, initial_stride=2) # recursively define inner hourglasses self.inner_hg = type(self)( diff --git a/centernet/modeling/layers/subnormalization.py b/centernet/modeling/layers/subnormalization.py deleted file mode 100644 index dd0aca3ae..000000000 --- a/centernet/modeling/layers/subnormalization.py +++ /dev/null @@ -1,901 +0,0 @@ -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Normalization layers.""" -from __future__ import absolute_import, division, print_function - -import tensorflow as tf -from tensorflow.python.distribute import distribution_strategy_context as ds -from tensorflow.python.distribute import reduce_util -from tensorflow.python.framework import constant_op, dtypes, ops, tensor_shape -from tensorflow.python.keras import backend as K -from tensorflow.python.keras import constraints, initializers, regularizers -from tensorflow.python.keras.engine.base_layer import Layer -from tensorflow.python.keras.engine.input_spec import InputSpec -from tensorflow.python.keras.layers import normalization -from tensorflow.python.keras.utils import control_flow_util -from tensorflow.python.ops import (array_ops, control_flow_ops, init_ops, - math_ops, nn, state_ops) -from tensorflow.python.ops import variables as tf_variables -from tensorflow.python.platform import tf_logging as logging - - -class SubDivBatchNormalization(normalization.BatchNormalizationBase): - - _USE_V2_BEHAVIOR = True - - def __init__(self, - axis=-1, - subdivisions=1, - momentum=0.99, - epsilon=1e-3, - center=True, - scale=True, - beta_initializer='zeros', - gamma_initializer='ones', - moving_mean_initializer='zeros', - moving_variance_initializer='ones', - beta_regularizer=None, - gamma_regularizer=None, - beta_constraint=None, - gamma_constraint=None, - renorm=False, - renorm_clipping=None, - renorm_momentum=0.99, - fused=None, - trainable=True, - adjustment=None, - name=None, - **kwargs): - - super(SubDivBatchNormalization, self).__init__( - axis=axis, - momentum=momentum, - epsilon=epsilon, - center=center, - scale=scale, - beta_initializer=beta_initializer, - gamma_initializer=gamma_initializer, - moving_mean_initializer=moving_mean_initializer, - moving_variance_initializer=moving_variance_initializer, - beta_regularizer=beta_regularizer, - gamma_regularizer=gamma_regularizer, - beta_constraint=beta_constraint, - gamma_constraint=gamma_constraint, - renorm=renorm, - renorm_clipping=renorm_clipping, - renorm_momentum=renorm_momentum, - fused=None - if not renorm else False, #if subdivisions <= 1 else False, #alter this - trainable=trainable, - virtual_batch_size=None, - name=name, - **kwargs) - - self.subdivisions = subdivisions - - def build(self, input_shape): - super().build(input_shape) - input_shape = tensor_shape.TensorShape(input_shape) - if not input_shape.ndims: - raise ValueError('Input has undefined rank:', input_shape) - ndims = len(input_shape) - - # Convert axis to list and resolve negatives - if isinstance(self.axis, int): - self.axis = [self.axis] - - for idx, x in enumerate(self.axis): - if x < 0: - self.axis[idx] = ndims + x - - # Validate axes - for x in self.axis: - if x < 0 or x >= ndims: - raise ValueError('Invalid axis: %d' % x) - if len(self.axis) != len(set(self.axis)): - raise ValueError('Duplicate axis: %s' % self.axis) - - # TODO(yaozhang): if input is not 4D, reshape it to 4D and reshape the - # output back to its original shape accordingly. - # self.fused = None - - if self.fused: - if self._USE_V2_BEHAVIOR: - # TODO(b/173253101): Using fused in the 5D case is currently disabled - # due to a regression on UNet, so it is only currently only supported in - # the 4D case. - if self.fused is None: - self.fused = ndims == 4 - elif self.fused and ndims != 4: - raise ValueError('Batch normalization layers with `fused=True` only ' - 'support 4D or 5D input tensors. ' - 'Received tensor with shape: %s' % - (tuple(input_shape),)) - else: - assert self.fused is not None - self.fused = (ndims == 4 and self._fused_can_be_used()) - # TODO(chrisying): fused batch norm is currently not supported for - # multi-axis batch norm and by extension virtual batches. In some cases, - # it might be possible to use fused batch norm but would require reshaping - # the Tensor to 4D with the axis in 1 or 3 (preferred 1) which is - # particularly tricky. A compromise might be to just support the most - # common use case (turning 5D w/ virtual batch to NCHW) - - if self.axis == [1] and ndims == 4: - self._data_format = 'NCHW' - elif self.axis == [1] and ndims == 5: - self._data_format = 'NCDHW' - elif self.axis == [3] and ndims == 4: - self._data_format = 'NHWC' - elif self.axis == [4] and ndims == 5: - self._data_format = 'NDHWC' - elif ndims == 5: - # 5D tensors that can be passed in but should not use fused batch norm - # due to unsupported axis. - self.fused = False - else: - raise ValueError('Unsupported axis, fused batch norm only supports ' - 'axis == [1] or axis == [3] for 4D input tensors or ' - 'axis == [1] or axis == [4] for 5D input tensors') - - axis_to_dim = {x: input_shape.dims[x].value for x in self.axis} - for x in axis_to_dim: - if axis_to_dim[x] is None: - raise ValueError('Input has undefined `axis` dimension. Input shape: ', - input_shape) - self.input_spec = InputSpec(ndim=ndims, axes=axis_to_dim) - - # get the shape for my weights based on input shape - if len(axis_to_dim) == 1 and self.virtual_batch_size is None: - # Single axis batch norm (most common/default use-case) - param_shape = (list(axis_to_dim.values())[0],) - else: - # Parameter shape is the original shape but with 1 in all non-axis dims - param_shape = [ - axis_to_dim[i] if i in axis_to_dim else 1 for i in range(ndims) - ] - if self.virtual_batch_size is not None: - # When using virtual batches, add an extra dim at index 1 - param_shape.insert(1, 1) - for idx, x in enumerate(self.axis): - self.axis[idx] = x + 1 # Account for added dimension - - try: - # Disable variable partitioning when creating the moving mean and variance - if hasattr(self, '_scope') and self._scope: - partitioner = self._scope.partitioner - self._scope.set_partitioner(None) - else: - partitioner = None - - if self.subdivisions > 1: - self.aggregated_sum_batch = self.add_weight( - name='agg_sum', - shape=param_shape, - dtype=self._param_dtype, - initializer=self.moving_mean_initializer, - synchronization=tf_variables.VariableSynchronization.ON_READ, - trainable=False, - aggregation=tf_variables.VariableAggregation.SUM, - experimental_autocast=False) - - self.aggregated_square_sum_batch = self.add_weight( - name='agg_square_sum', - shape=param_shape, - dtype=self._param_dtype, - initializer=self.moving_variance_initializer, - synchronization=tf_variables.VariableSynchronization.ON_READ, - trainable=False, - aggregation=tf_variables.VariableAggregation.SUM, - experimental_autocast=False) - - self.local_count = self.add_weight( - name='local_sum', - shape=(), - dtype=tf.int32, - initializer=tf.zeros_initializer(), - synchronization=tf_variables.VariableSynchronization.ON_READ, - trainable=False, - aggregation=tf_variables.VariableAggregation.SUM, - experimental_autocast=False) - - self.aggregated_batch_size = self.add_weight( - name='net_batches', - shape=(), - dtype=tf.int32, - initializer=tf.zeros_initializer(), - synchronization=tf_variables.VariableSynchronization.ON_READ, - trainable=False, - aggregation=tf_variables.VariableAggregation.SUM, - experimental_autocast=False) - - finally: - if partitioner: - self._scope.set_partitioner(partitioner) - self.built = True - - def _assign_subdiv_moving_average(self, variable, value, momentum, - subdivsions, count): - with K.name_scope('AssignSubDivMovingAvg') as scope: - with ops.colocate_with(variable): - decay = ops.convert_to_tensor_v2_with_dispatch( - 1.0 - momentum, name='decay') - if decay.dtype != variable.dtype.base_dtype: - decay = math_ops.cast(decay, variable.dtype.base_dtype) - - # get the aggregated update - update_delta = (variable - math_ops.cast(value, variable.dtype)) * decay - - # update at the end of last step - update_delta = array_ops.where((count + 1) % subdivisions == 0, - update_delta, K.zeros_like(update_delta)) - return state_ops.assign_sub(variable, update_delta, name=scope) - - def _assign_subdiv_new_value(self, variable, value, subdivisions, count): - with K.name_scope('AssignNewValue') as scope: - with ops.colocate_with(variable): - update_value = array_ops.where((count + 1) % subdivisions == 0, value, - variable) - return state_ops.assign(variable, update_value, name=scope) - - def _assign_subdiv_rotating_sum(self, variable, value, subdivisions, count, - inputs_size): - with K.name_scope('AssignSubDivRotatedSum') as scope: - with ops.colocate_with(variable): - # reduce it for the current - update_delta = value #/subdivisions - - # if the input size is 0 - if inputs_size is not None: - update_delta = array_ops.where(inputs_size > 0, update_delta, - K.zeros_like(update_delta)) - - # if we are starting a new batch set the variable to 0 by removing it - # from update delta then add the delta to the variable to get - # rid of the value variable - update_delta = array_ops.where(count % subdivisions == 0, - update_delta - variable, update_delta) - return state_ops.assign_add(variable, update_delta, name=scope) - - def _subdiv_calculate_mean_and_var(self, inputs, reduction_axes, keep_dims): - # calculate the - net_sum = math_ops.reduce_sum( - inputs, axis=reduction_axes, keepdims=keep_dims) - squared_mean = math_ops.reduce_sum( - math_ops.square(inputs), axis=reduction_axes, keepdims=keep_dims) - - if self._support_zero_size_input(): - # Keras assumes that batch dimension is the first dimension for Batch - # Normalization. - input_batch_size = array_ops.shape(inputs)[0] - else: - input_batch_size = None - - # get the number of total params you are averaging including batchsize(local) - axes_vals = [ - (array_ops.shape_v2(inputs))[i] for i in range(1, len(reduction_axes)) - ] - multiplier = math_ops.cast(math_ops.reduce_prod(axes_vals), dtypes.float32) - - squared_mean = squared_mean / multiplier - net_sum = net_sum / multiplier - - if input_batch_size is None: - mean, variance = nn.moments(inputs, reduction_axes, keep_dims=keep_dims) - else: - batches_ = math_ops.cast(input_batch_size, self._param_dtype) - mean = net_sum / batches_ - variance = squared_mean / batches_ - math_ops.square( - array_ops.stop_gradient(mean)) - - return mean, net_sum, variance, squared_mean, input_batch_size - - def subdiv_moments(self, inputs, reduction_axes, keep_dims): - # mean and variance only for the current batch - mean, net_sum, variance, squared_mean, input_batch_size = self._subdiv_calculate_mean_and_var( - inputs, reduction_axes, keep_dims) - - if self._support_zero_size_input(): - input_batch_size = 0 if input_batch_size is None else input_batch_size - mean = array_ops.where(input_batch_size > 0, mean, K.zeros_like(mean)) - net_sum = array_ops.where(input_batch_size > 0, net_sum, - K.zeros_like(net_sum)) - variance = array_ops.where(input_batch_size > 0, variance, - K.zeros_like(variance)) - squared_mean = array_ops.where(input_batch_size > 0, squared_mean, - K.zeros_like(squared_mean)) - return mean, net_sum, variance, squared_mean, input_batch_size - - def _subdiv_batch_norm(self, inputs, training=None): - # tf.print('bn', self.local_count) - training = self._get_training_value(training) - - inputs_dtype = inputs.dtype.base_dtype - if inputs_dtype in (dtypes.float16, dtypes.bfloat16): - # Do all math in float32 if given 16-bit inputs for numeric stability. - # In particular, it's very easy for variance to overflow in float16 and - # for safety we also choose to cast bfloat16 to float32. - inputs = math_ops.cast(inputs, dtypes.float32) - - params_dtype = self._param_dtype - - # Compute the axes along which to reduce the mean / variance - input_shape = inputs.shape - ndims = len(input_shape) - reduction_axes = [i for i in range(ndims) if i not in self.axis] - if self.virtual_batch_size is not None: - del reduction_axes[1] # Do not reduce along virtual batch dim - - # Broadcasting only necessary for single-axis batch norm where the axis is - # not the last dimension - broadcast_shape = [1] * ndims - broadcast_shape[self.axis[0]] = input_shape.dims[self.axis[0]].value - - def _broadcast(v): - if (v is not None and len(v.shape) != ndims and - reduction_axes != list(range(ndims - 1))): - return array_ops.reshape(v, broadcast_shape) - return v - - scale, offset = _broadcast(self.gamma), _broadcast(self.beta) - - # what does this do... - def _compose_transforms(scale, offset, then_scale, then_offset): - if then_scale is not None: - scale *= then_scale - offset *= then_scale - if then_offset is not None: - offset += then_offset - return (scale, offset) - - # is training value true false or None - training_value = control_flow_util.constant_value(training) - update_value = (self.local_count + 1) % self.subdivisions == 0 - if training_value == False: # pylint: disable=singleton-comparison,g-explicit-bool-comparison - mean, variance = self.moving_mean, self.moving_variance - else: - # training_value could be True or None -> None means determine at runtime - if self.adjustment: - adj_scale, adj_bias = self.adjustment(array_ops.shape(inputs)) - # Adjust only during training. - adj_scale = control_flow_util.smart_cond( - training, lambda: adj_scale, lambda: array_ops.ones_like(adj_scale)) - adj_bias = control_flow_util.smart_cond( - training, lambda: adj_bias, lambda: array_ops.zeros_like(adj_bias)) - scale, offset = _compose_transforms(adj_scale, adj_bias, scale, offset) - - keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1 - - # normalization stats for the current batch important = mean and squared_mean - mean, net_sum, variance, squared_mean, input_batch_size = self.subdiv_moments( - math_ops.cast(inputs, self._param_dtype), - reduction_axes, - keep_dims=keep_dims) - - # aggregate the things - def _update_aggragate_sum(): - return self._assign_subdiv_rotating_sum(self.aggregated_sum_batch, - net_sum, self.subdivisions, - self.local_count, - input_batch_size) - - def _update_aggragate_squared_sum(): - return self._assign_subdiv_rotating_sum( - self.aggregated_square_sum_batch, squared_mean, self.subdivisions, - self.local_count, input_batch_size) - - def _update_aggragate_batch_size(): - return self._assign_subdiv_rotating_sum(self.aggregated_batch_size, - input_batch_size, - self.subdivisions, - self.local_count, - input_batch_size) - - self.add_update(_update_aggragate_sum) - self.add_update(_update_aggragate_squared_sum) - self.add_update(_update_aggragate_batch_size) - - aggregated_mean = self.aggregated_sum_batch / math_ops.cast( - self.aggregated_batch_size, params_dtype) - aggregated_squared_mean = self.aggregated_square_sum_batch / math_ops.cast( - self.aggregated_batch_size, params_dtype) - aggregated_variance = aggregated_squared_mean - math_ops.square( - aggregated_mean) - - moving_mean = self.moving_mean - moving_variance = self.moving_variance - - # if we are training use the stats for this batch for normalizing this - # value other wise use the moving average - - # should only happen when we update the moving values - mean = control_flow_util.smart_cond( - training, - true_fn=lambda: mean, - false_fn=lambda: ops.convert_to_tensor_v2_with_dispatch(moving_mean)) - variance = control_flow_util.smart_cond( - training, - true_fn=lambda: variance, - false_fn=lambda: ops.convert_to_tensor_v2_with_dispatch( - moving_variance)) - - # circular update of the mean and variance - new_mean = control_flow_util.smart_cond( - update_value, - true_fn=lambda: ops.convert_to_tensor_v2_with_dispatch(aggregated_mean - ), - false_fn=lambda: moving_mean) - - new_variance = control_flow_util.smart_cond( - update_value, - true_fn=lambda: ops.convert_to_tensor_v2_with_dispatch( - aggregated_variance), - false_fn=lambda: moving_variance) - - # # should only be done when the moving mean is updated - # tf.print(new_variance, self.local_count, update_value, self.aggregated_batch_size, self.aggregated_sum_batch) - - if self.renorm: - r, d, new_mean, new_variance = self._renorm_correction_and_moments( - new_mean, new_variance, training, input_batch_size) - # When training, the normalized values (say, x) will be transformed as - # x * gamma + beta without renorm, and (x * r + d) * gamma + beta - # = x * (r * gamma) + (d * gamma + beta) with renorm. - r = _broadcast(array_ops.stop_gradient(r, name='renorm_r')) - d = _broadcast(array_ops.stop_gradient(d, name='renorm_d')) - scale, offset = _compose_transforms(r, d, scale, offset) - - def _do_update(var, value): - """Compute the updates for mean and variance.""" - return self._assign_moving_average(var, value, self.momentum, - self.aggregated_batch_size) - - def mean_update(): - true_branch = lambda: _do_update(self.moving_mean, new_mean) - false_branch = lambda: self.moving_mean - return control_flow_util.smart_cond(training, true_branch, false_branch) - - def variance_update(): - """Update the moving variance.""" - - def true_branch_renorm(): - # We apply epsilon as part of the moving_stddev to mirror the training - # code path. - moving_stddev = _do_update(self.moving_stddev, - math_ops.sqrt(new_variance + self.epsilon)) - return self._assign_new_value( - self.moving_variance, - # Apply relu in case floating point rounding causes it to go - # negative. - K.relu(moving_stddev * moving_stddev - self.epsilon)) - - if self.renorm: - true_branch = true_branch_renorm - else: - true_branch = lambda: _do_update(self.moving_variance, new_variance) - - false_branch = lambda: self.moving_variance - return control_flow_util.smart_cond(training, true_branch, false_branch) - - def update_count(): - with K.name_scope('update_count') as scope: - # update the local count - return state_ops.assign_add( - self.local_count, tf.cast(1, self.local_count.dtype), name=scope) - - self.add_update(mean_update) - self.add_update(variance_update) - self.add_update(update_count) - - mean = math_ops.cast(mean, inputs.dtype) - variance = math_ops.cast(variance, inputs.dtype) - if offset is not None: - offset = math_ops.cast(offset, inputs.dtype) - if scale is not None: - scale = math_ops.cast(scale, inputs.dtype) - outputs = nn.batch_normalization(inputs, _broadcast(mean), - _broadcast(variance), offset, scale, - self.epsilon) - if inputs_dtype in (dtypes.float16, dtypes.bfloat16): - outputs = math_ops.cast(outputs, inputs_dtype) - - # If some components of the shape got lost due to adjustments, fix that. - outputs.set_shape(input_shape) - - if self.virtual_batch_size is not None: - outputs = undo_virtual_batching(outputs) - return outputs - - def call(self, inputs, training=None): - training = self._get_training_value(training) - if self.subdivisions <= 1 or self.subdivisions is None: - return super().call(inputs, training=training) - else: - if self.renorm is False and training is False and self.fused: - # outputs = self._fused_batch_norm(inputs, training=False) - beta = self.beta if self.center else self._beta_const - gamma = self.gamma if self.scale else self._gamma_const - outputs, mean, variance = nn.fused_batch_norm( - inputs, - gamma, - beta, - mean=self.moving_mean, - variance=self.moving_variance, - epsilon=self.epsilon, - is_training=False, - data_format=self._data_format) - return outputs - return self._subdiv_batch_norm(inputs, training=training) - - -class SubDivSyncBatchNormalization(SubDivBatchNormalization): - r"""Normalize and scale inputs or activations synchronously across replicas. - Applies batch normalization to activations of the previous layer at each batch - by synchronizing the global batch statistics across all devices that are - training the model. For specific details about batch normalization please - refer to the `tf.keras.layers.BatchNormalization` layer docs. - If this layer is used when using tf.distribute strategy to train models - across devices/workers, there will be an allreduce call to aggregate batch - statistics across all replicas at every training step. Without tf.distribute - strategy, this layer behaves as a regular `tf.keras.layers.BatchNormalization` - layer. - Example usage: - ``` - strategy = tf.distribute.MirroredStrategy() - with strategy.scope(): - model = tf.keras.Sequential() - model.add(tf.keras.layers.Dense(16)) - model.add(tf.keras.layers.experimental.SyncBatchNormalization()) - ``` - Arguments: - axis: Integer, the axis that should be normalized - (typically the features axis). - For instance, after a `Conv2D` layer with - `data_format="channels_first"`, - set `axis=1` in `BatchNormalization`. - momentum: Momentum for the moving average. - epsilon: Small float added to variance to avoid dividing by zero. - center: If True, add offset of `beta` to normalized tensor. - If False, `beta` is ignored. - scale: If True, multiply by `gamma`. - If False, `gamma` is not used. - When the next layer is linear (also e.g. `nn.relu`), - this can be disabled since the scaling - will be done by the next layer. - beta_initializer: Initializer for the beta weight. - gamma_initializer: Initializer for the gamma weight. - moving_mean_initializer: Initializer for the moving mean. - moving_variance_initializer: Initializer for the moving variance. - beta_regularizer: Optional regularizer for the beta weight. - gamma_regularizer: Optional regularizer for the gamma weight. - beta_constraint: Optional constraint for the beta weight. - gamma_constraint: Optional constraint for the gamma weight. - renorm: Whether to use [Batch Renormalization]( - https://arxiv.org/abs/1702.03275). This adds extra variables during - training. The inference is the same for either value of this parameter. - renorm_clipping: A dictionary that may map keys 'rmax', 'rmin', 'dmax' to - scalar `Tensors` used to clip the renorm correction. The correction - `(r, d)` is used as `corrected_value = normalized_value * r + d`, with - `r` clipped to [rmin, rmax], and `d` to [-dmax, dmax]. Missing rmax, rmin, - dmax are set to inf, 0, inf, respectively. - renorm_momentum: Momentum used to update the moving means and standard - deviations with renorm. Unlike `momentum`, this affects training - and should be neither too small (which would add noise) nor too large - (which would give stale estimates). Note that `momentum` is still applied - to get the means and variances for inference. - trainable: Boolean, if `True` the variables will be marked as trainable. - Call arguments: - inputs: Input tensor (of any rank). - training: Python boolean indicating whether the layer should behave in - training mode or in inference mode. - - `training=True`: The layer will normalize its inputs using the - mean and variance of the current batch of inputs. - - `training=False`: The layer will normalize its inputs using the - mean and variance of its moving statistics, learned during training. - Input shape: - Arbitrary. Use the keyword argument `input_shape` - (tuple of integers, does not include the samples axis) - when using this layer as the first layer in a model. - Output shape: - Same shape as input. - """ - - def __init__(self, - axis=-1, - subdivisions=1, - momentum=0.99, - epsilon=1e-3, - center=True, - scale=True, - beta_initializer='zeros', - gamma_initializer='ones', - moving_mean_initializer='zeros', - moving_variance_initializer='ones', - beta_regularizer=None, - gamma_regularizer=None, - beta_constraint=None, - gamma_constraint=None, - renorm=False, - renorm_clipping=None, - renorm_momentum=0.99, - trainable=True, - adjustment=None, - name=None, - **kwargs): - - # Currently we only support aggregating over the global batch size. - super(SubDivSyncBatchNormalization, self).__init__( - axis=axis, - subdivisions=subdivisions, - momentum=momentum, - epsilon=epsilon, - center=center, - scale=scale, - beta_initializer=beta_initializer, - gamma_initializer=gamma_initializer, - moving_mean_initializer=moving_mean_initializer, - moving_variance_initializer=moving_variance_initializer, - beta_regularizer=beta_regularizer, - gamma_regularizer=gamma_regularizer, - beta_constraint=beta_constraint, - gamma_constraint=gamma_constraint, - renorm=renorm, - renorm_clipping=renorm_clipping, - renorm_momentum=renorm_momentum, - fused=False, - trainable=trainable, - name=name, - **kwargs) - - def _calculate_mean_and_var(self, x, axes, keep_dims): - - with K.name_scope('moments'): - # The dynamic range of fp16 is too limited to support the collection of - # sufficient statistics. As a workaround we simply perform the operations - # on 32-bit floats before converting the mean and variance back to fp16 - y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x - replica_ctx = ds.get_replica_context() - if replica_ctx: - # local to me - local_sum = math_ops.reduce_sum(y, axis=axes, keepdims=True) - local_squared_sum = math_ops.reduce_sum( - math_ops.square(y), axis=axes, keepdims=True) - batch_size = math_ops.cast(array_ops.shape_v2(y)[0], dtypes.float32) - # TODO(b/163099951): batch the all-reduces once we sort out the ordering - # issue for NCCL. We don't have a mechanism to launch NCCL in the same - # order in each replica nowadays, so we limit NCCL to batch all-reduces. - - # get the sum of all replicas (converge all devices) - y_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, local_sum) - # get the sum from all replicas (converge all devices) - y_squared_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, - local_squared_sum) - # get the net batch size from all devices (converge all devices) - global_batch_size = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, - batch_size) - - # get the number of total params you are averaging (local) - axes_vals = [(array_ops.shape_v2(y))[i] for i in range(1, len(axes))] - multiplier = math_ops.cast( - math_ops.reduce_prod(axes_vals), dtypes.float32) - multiplier = multiplier * global_batch_size - - # conver mean var (locally) - mean = y_sum / multiplier - y_squared_mean = y_squared_sum / multiplier - # var = E(x^2) - E(x)^2 - variance = y_squared_mean - math_ops.square(mean) - else: - # if you only have one replica dont worry about it - # Compute true mean while keeping the dims for proper broadcasting. - mean = math_ops.reduce_mean(y, axes, keepdims=True, name='mean') - # sample variance, not unbiased variance - # Note: stop_gradient does not change the gradient that gets - # backpropagated to the mean from the variance calculation, - # because that gradient is zero - variance = math_ops.reduce_mean( - math_ops.squared_difference(y, mean), - axes, - keepdims=True, - name='variance') - if not keep_dims: - mean = array_ops.squeeze(mean, axes) - variance = array_ops.squeeze(variance, axes) - if x.dtype == dtypes.float16: - return (math_ops.cast(mean, dtypes.float16), - math_ops.cast(variance, dtypes.float16)) - else: - return (mean, variance) - - def _subdiv_calculate_mean_and_var(self, x, axes, keep_dims): - - with K.name_scope('moments'): - # The dynamic range of fp16 is too limited to support the collection of - # sufficient statistics. As a workaround we simply perform the operations - # on 32-bit floats before converting the mean and variance back to fp16 - y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x - replica_ctx = ds.get_replica_context() - - if replica_ctx: - # local to me - - local_sum = math_ops.reduce_sum(y, axis=axes, keepdims=True) - local_squared_sum = math_ops.reduce_sum( - math_ops.square(y), axis=axes, keepdims=True) - batch_size = math_ops.cast(array_ops.shape_v2(y)[0], dtypes.float32) - # TODO(b/163099951): batch the all-reduces once we sort out the ordering - # issue for NCCL. We don't have a mechanism to launch NCCL in the same - # order in each replica nowadays, so we limit NCCL to batch all-reduces. - # get the sum of all replicas (converge all devices) - y_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, local_sum) - # get the sum from all replicas (converge all devices) - y_squared_sum = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, - local_squared_sum) - # get the net batch size from all devices (converge all devices) - input_batch_size = replica_ctx.all_reduce(reduce_util.ReduceOp.SUM, - batch_size) - - #tf.print(replica_ctx.replica_id_in_sync_group, replica_ctx.num_replicas_in_sync, batch_size, self.aggregated_square_sum_batch, axes) - # get the number of total params you are averaging (local) - axes_vals = [(array_ops.shape_v2(y))[i] for i in range(1, len(axes))] - multiplier_ = math_ops.cast( - math_ops.reduce_prod(axes_vals), dtypes.float32) - multiplier = multiplier_ * input_batch_size - - # conver mean var (locally) - mean = y_sum / multiplier - y_squared_mean = y_squared_sum / multiplier - # var = E(x^2) - E(x)^2 - variance = y_squared_mean - math_ops.square(mean) - net_sum = y_sum / multiplier_ - squared_mean = y_squared_sum / multiplier_ - - else: - # mean = math_ops.reduce_mean(y, axes, keepdims=True, name='mean') - # # sample variance, not unbiased variance - # # Note: stop_gradient does not change the gradient that gets - # # backpropagated to the mean from the variance calculation, - # # because that gradient is zero - # variance = math_ops.reduce_mean( - # math_ops.squared_difference(y, array_ops.stop_gradient(mean)), - # axes, - # keepdims=True, - # name='variance') - - net_sum = math_ops.reduce_sum(y, axis=axes, keepdims=True) - squared_mean = math_ops.reduce_sum( - math_ops.square(y), axis=axes, keepdims=True) - - if self._support_zero_size_input(): - # Keras assumes that batch dimension is the first dimension for Batch - # Normalization. - input_batch_size = array_ops.shape(y)[0] - else: - input_batch_size = None - - # get the number of total params you are averaging including batchsize(local) - axes_vals = [(array_ops.shape_v2(y))[i] for i in range(1, len(axes))] - multiplier = math_ops.cast( - math_ops.reduce_prod(axes_vals), dtypes.float32) - - squared_mean = squared_mean / multiplier - net_sum = net_sum / multiplier - - if input_batch_size is None: - mean, variance = nn.moments(y, axes, keep_dims=True) - input_batch_size = 0 - else: - batches_ = math_ops.cast(input_batch_size, self._param_dtype) - # # if you only have one replica dont worry about it - # # Compute true mean while keeping the dims for proper broadcasting. - mean = net_sum / batches_ - variance = squared_mean / batches_ - math_ops.square(mean) - - input_batch_size = math_ops.cast(input_batch_size, dtypes.int32) - if not keep_dims: - mean = array_ops.squeeze(mean, axes) - net_sum = array_ops.squeeze(net_sum, axes) - variance = array_ops.squeeze(variance, axes) - squared_mean = array_ops.squeeze(squared_mean, axes) - if x.dtype == dtypes.float16: - return (math_ops.cast(mean, dtypes.float16), - math_ops.cast(net_sum, dtypes.float16), - math_ops.cast(variance, dtypes.float16), - math_ops.cast(squared_mean, dtypes.float16), input_batch_size) - else: - return (mean, net_sum, variance, squared_mean, input_batch_size) - - -class ShuffleBatchNormalization(normalization.BatchNormalizationBase): - - def __init__(self, - axis=-1, - momentum=0.99, - epsilon=1e-3, - center=True, - scale=True, - beta_initializer='zeros', - gamma_initializer='ones', - moving_mean_initializer='zeros', - moving_variance_initializer='ones', - beta_regularizer=None, - gamma_regularizer=None, - beta_constraint=None, - gamma_constraint=None, - renorm=False, - renorm_clipping=None, - renorm_momentum=0.99, - trainable=True, - adjustment=None, - name=None, - **kwargs): - - # Currently we only support aggregating over the global batch size. - super(ShuffleBatchNormalization, self).__init__( - axis=axis, - momentum=momentum, - epsilon=epsilon, - center=center, - scale=scale, - beta_initializer=beta_initializer, - gamma_initializer=gamma_initializer, - moving_mean_initializer=moving_mean_initializer, - moving_variance_initializer=moving_variance_initializer, - beta_regularizer=beta_regularizer, - gamma_regularizer=gamma_regularizer, - beta_constraint=beta_constraint, - gamma_constraint=gamma_constraint, - renorm=renorm, - renorm_clipping=renorm_clipping, - renorm_momentum=renorm_momentum, - fused=False, - trainable=trainable, - virtual_batch_size=None, - name=name, - **kwargs) - - def _calculate_mean_and_var(self, x, axes, keep_dims): - - with K.name_scope('moments'): - # The dynamic range of fp16 is too limited to support the collection of - # sufficient statistics. As a workaround we simply perform the operations - # on 32-bit floats before converting the mean and variance back to fp16 - y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x - # if you only have one replica dont worry about it - # Compute true mean while keeping the dims for proper broadcasting. - mean = math_ops.reduce_mean(y, axes, keepdims=True, name='mean') - # sample variance, not unbiased variance - # Note: stop_gradient does not change the gradient that gets - # backpropagated to the mean from the variance calculation, - # because that gradient is zero - variance = math_ops.reduce_mean( - math_ops.squared_difference(y, array_ops.stop_gradient(mean)), - axes, - keepdims=True, - name='variance') - - replica_ctx = ds.get_replica_context() - if replica_ctx: - tf.print(replica_ctx.num_replicas_in_sync) - tf.print(replica_ctx.replica_id_in_sync_group) - - if not keep_dims: - mean = array_ops.squeeze(mean, axes) - variance = array_ops.squeeze(variance, axes) - if x.dtype == dtypes.float16: - return (math_ops.cast(mean, dtypes.float16), - math_ops.cast(variance, dtypes.float16)) - else: - return (mean, variance) diff --git a/centernet/utils/weight_utils/load_weights.py b/centernet/utils/weight_utils/load_weights.py index 0dbf04808..64bcb26e9 100644 --- a/centernet/utils/weight_utils/load_weights.py +++ b/centernet/utils/weight_utils/load_weights.py @@ -8,9 +8,10 @@ from centernet.configs.centernet import CenterNetTask from centernet.modeling.CenterNet import build_centernet -from centernet.modeling.layers.nn_blocks import (CenterNetDecoderConv, +from centernet.modeling.layers.nn_blocks import (CenterNetConvBN, + CenterNetDecoderConv, CenterNetResidualBlock, - ConvBN, HourglassBlock) + HourglassBlock) from centernet.utils.weight_utils.config_classes import (convBnCFG, decoderConvCFG, hourglassCFG, @@ -82,7 +83,7 @@ def load_weights_backbone(backbone, weights_dict, backbone_name): cfg = cfgs.pop(0) for i in range(len(backbone_layers)): layer = backbone_layers[i] - if isinstance(layer, (ConvBN, HourglassBlock, CenterNetResidualBlock)): + if isinstance(layer, (CenterNetConvBN, HourglassBlock, CenterNetResidualBlock)): n_weights = cfg.load_weights(layer) print("Loading weights for: {}, weights loaded: {}".format(cfg, n_weights)) n_weights_total += n_weights From c50c5eb7030bcfbee313e5ca5f1b83ea0d029a80 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 30 Apr 2021 12:21:18 -0400 Subject: [PATCH 127/132] eval normalization --- centernet/dataloaders/centernet_input.py | 12 +++++++- centernet/demo.py | 38 ++++++++++++++++++++++++ centernet/tasks/centernet.py | 2 +- 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 centernet/demo.py diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index d7810d57f..5f984426f 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -4,6 +4,7 @@ from centernet.ops import preprocessing_ops from official.vision.beta.dataloaders import parser, utils +from official.vision.beta.ops.preprocess_ops import normalize_image from yolo.ops import preprocessing_ops as yolo_preprocessing_ops @@ -319,7 +320,16 @@ def _parse_eval_data(self, data): images: the image tensor. labels: a dict of Tensors that contains labels. """ - image = data['image'] / 255 + + channel_means: [104.01362025, 114.03422265, 119.9165958] + channel_stds: [73.6027665 , 69.89082075, 70.9150767] + + red, green, blue = tf.unstack(image, axis=-1) + image = tf.stack([blue, green, red], axis=-1) + + image = normalize_image(image, offset=channel_means, scale=channel_stds) + + # image = data['image'] / 255 boxes = data['groundtruth_boxes'] classes = data['groundtruth_classes'] diff --git a/centernet/demo.py b/centernet/demo.py new file mode 100644 index 000000000..8b2275a67 --- /dev/null +++ b/centernet/demo.py @@ -0,0 +1,38 @@ +import tensorflow as tf + +from centernet.configs.centernet import CenterNetTask +from centernet.modeling.CenterNet import build_centernet +from centernet.utils.weight_utils.load_weights import ( + get_model_weights_as_dict, load_weights_backbone, load_weights_model) +from yolo.demos.video_detect_cpu import runner +from yolo.demos.video_detect_gpu import FastVideo +from yolo.utils.run_utils import prep_gpu + +CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' +CLIP_PATH = r'D:\\Documents\Research\Software\nyc_demo_fast.mp4' + +if __name__ == '__main__': + prep_gpu() + input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) + config = CenterNetTask() + + model, loss = build_centernet(input_specs=input_specs, + task_config=config, l2_regularization=0) + + weights_dict, _ = get_model_weights_as_dict(CENTERNET_CKPT_PATH) + load_weights_model(model, weights_dict, 'hourglass104_512', 'detection_2d') + + cap = FastVideo( + CLIP_PATH, # set to 0 if using webcam + model=model, + process_width=512, + process_height=512, + preprocess_with_gpu=False, + classes=91, + print_conf=True, + max_batch=1, + disp_h=512, + scale_que=1, + wait_time='dynamic') + cap.run() + runner(model, 0, 512, 512) diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index 122e06cfa..a3693238d 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -121,7 +121,7 @@ def build_losses(self, outputs, labels, num_replicas=1, scale_replicas=1, aux_lo metric_dict = dict() - # TODO: actually compute loss + # TODO: The loss components in centernet/losses are tested but # returning 0 for now, just trying to run eval metric_dict['total_loss'] = total_loss return total_loss, metric_dict From 93e2a13a5bfcf160723a05d4e85ece892d32829e Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 30 Apr 2021 19:58:40 +0000 Subject: [PATCH 128/132] normalization to datapipline --- centernet/configs/experiments/centernet-eval-tpu.yaml | 2 +- centernet/dataloaders/centernet_input.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index e9df99153..ced148176 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -21,7 +21,7 @@ task: net_down_scale: 4 input_image_dims: 512 use_nms: False - nms_pre_thresh: 0.1 + nms_pre_thresh: 0.3 nms_thresh: 0.4 use_reduction_sum: True validation_data: diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 5f984426f..da54b993d 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -320,16 +320,15 @@ def _parse_eval_data(self, data): images: the image tensor. labels: a dict of Tensors that contains labels. """ - - channel_means: [104.01362025, 114.03422265, 119.9165958] - channel_stds: [73.6027665 , 69.89082075, 70.9150767] + image = tf.cast(data['image'], dtype=tf.float32) + channel_means = [104.01362025, 114.03422265, 119.9165958] #bgr + channel_stds = [73.6027665 , 69.89082075, 70.9150767] #bgr - red, green, blue = tf.unstack(image, axis=-1) - image = tf.stack([blue, green, red], axis=-1) + red, green, blue = tf.unstack(image, num=3, axis=2) + image = tf.stack([blue, green, red], axis=2) image = normalize_image(image, offset=channel_means, scale=channel_stds) - # image = data['image'] / 255 boxes = data['groundtruth_boxes'] classes = data['groundtruth_classes'] From dca114c475f3229e6ba1f7952b2250af67abe699 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 30 Apr 2021 16:56:27 -0400 Subject: [PATCH 129/132] documentation, normalization in config --- centernet/configs/centernet.py | 15 +- .../experiments/centernet-eval-tpu.yaml | 9 +- centernet/dataloaders/centernet_input.py | 244 +----------------- centernet/demo.py | 14 + centernet/modeling/layers/nn_blocks.py | 7 +- centernet/tasks/centernet.py | 6 +- 6 files changed, 45 insertions(+), 250 deletions(-) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 211d50a65..80a72c0bf 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -93,17 +93,16 @@ class DataDecoder(hyperparams.OneOfConfig): simple_decoder: TfExampleDecoder = TfExampleDecoder() label_map_decoder: TfExampleDecoderLabelMap = TfExampleDecoderLabelMap() -# dataset parsers +# dataset parser @dataclasses.dataclass class Parser(hyperparams.Config): image_w: int = 512 image_h: int = 512 - num_classes: int = 90 - max_num_instances: int = 128 - use_gaussian_bump: bool = True - gaussian_rad: int = -1 - gaussian_iou: float = 0.7 - output_dims: int = 128 + bgr_ordering: bool = True + channel_means: List[int] = dataclasses.field( + default_factory=lambda: [104.01362025, 114.03422265, 119.9165958]) + channel_stds: List[int] = dataclasses.field( + default_factory=lambda: [73.6027665, 69.89082075, 70.9150767]) dtype: str = 'float32' @dataclasses.dataclass @@ -164,7 +163,7 @@ class CenterNetLayer(hyperparams.Config): class_offset: int = 1 net_down_scale: int = 4 input_image_dims: int = 512 - use_nms: bool = True + use_nms: bool = False nms_pre_thresh: float = 0.1 nms_thresh: float = 0.4 use_reduction_sum: bool = True diff --git a/centernet/configs/experiments/centernet-eval-tpu.yaml b/centernet/configs/experiments/centernet-eval-tpu.yaml index ced148176..7d413cbcc 100644 --- a/centernet/configs/experiments/centernet-eval-tpu.yaml +++ b/centernet/configs/experiments/centernet-eval-tpu.yaml @@ -34,12 +34,11 @@ task: parser: image_h: 512 image_w: 512 - num_classes: 90 max_num_instances: 128 - use_gaussian_bump: true - gaussian_rad: -1 - gaussian_iou: 0.7 - output_dims: 128 + bgr_ordering: True + channel_means: [104.01362025, 114.03422265, 119.9165958] + channel_stds: [73.6027665, 69.89082075, 70.9150767] + dtype: 'float32' shuffle_buffer_size: 100 subtasks: detection: diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index da54b993d..0bf3940df 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -28,13 +28,10 @@ class CenterNetParser(parser.Parser): def __init__(self, image_w: int = 512, image_h: int = 512, - num_classes: int = 90, max_num_instances: int = 128, - use_gaussian_bump: bool = True, - gaussian_rad: int = -1, - gaussian_iou: float = 0.7, - output_dims: int = 128, - class_offset: int = 1, + bgr_ordering: bool = True, + channel_means: List[int] = [104.01362025, 114.03422265, 119.9165958], + channel_stds: List[int] = [73.6027665, 69.89082075, 70.9150767], dtype: str = 'float32'): """Initializes parameters for parsing annotations in the dataset. Args: @@ -50,17 +47,13 @@ def __init__(self, gaussian_iou: A `float` number for the minimum desired IOU used when determining the gaussian radius of center locations in the heatmap. output_dims: A `Tensor` or `int` for output dimensions of the heatmap. - class_offset: A `int` for subtracting a value from the ground truth classes """ self._image_w = image_w self._image_h = image_h - self._num_classes = num_classes self._max_num_instances = max_num_instances - self._gaussian_iou = gaussian_iou - self._use_gaussian_bump = use_gaussian_bump - self._gaussian_rad = -1 - self._output_dims = output_dims - self._class_offset = class_offset + self._bgr_ordering = bgr_ordering + self._channel_means = channel_means + self._channel_stds = channel_stds if dtype == 'float16': self._dtype = tf.float16 @@ -73,214 +66,6 @@ def __init__(self, 'Unsupported datatype used in parser only {float16, bfloat16, or float32}' ) - def _build_heatmap_and_regressed_features(self, - labels, - output_size=[128, 128], - input_size=[512, 512]): - """ Generates the ground truth labels for centernet. - - Ground truth labels are generated by splatting gaussians on heatmaps for - corners and centers. Regressed features (offsets and sizes) are also - generated. - - Args: - labels: A dictionary of COCO ground truth labels with at minimum the following fields: - bbox: A `Tensor` of shape [max_num_instances, num_boxes, 4], where the last dimension - corresponds to the top left x, top left y, bottom right x, and - bottom left y coordinates of the bounding box - classes: A `Tensor` of shape [max_num_instances, num_boxes] that contains the class of each - box, given in the same order as the boxes - num_detections: A `Tensor` or int that gives the number of objects in the image - output_size: A `list` of length 2 containing the desired output height - and width of the heatmaps - input_size: A `list` of length 2 the expected input height and width of - the image - Returns: - Dictionary of labels with the following fields: - 'tl_heatmaps': A `Tensor` of shape [output_h, output_w, num_classes], - heatmap with splatted gaussians centered at the positions and channels - corresponding to the top left location and class of the object - 'br_heatmaps': `Tensor` of shape [output_h, output_w, num_classes], - heatmap with splatted gaussians centered at the positions and channels - corresponding to the bottom right location and class of the object - 'ct_heatmaps': Tensor of shape [output_h, output_w, num_classes], - heatmap with splatted gaussians centered at the positions and channels - corresponding to the center location and class of the object - 'tl_offset': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the x-offset and y-offset of the top-left - corner of an object. All other entires are 0 - 'br_offset': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the x-offset and y-offset of the - bottom-right corner of an object. All other entires are 0 - 'ct_offset': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the x-offset and y-offset of the center of - an object. All other entires are 0 - 'size': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the width and height of an object. All - other entires are 0 - 'box_mask': `Tensor` of shape [max_num_instances], where the first - num_boxes entries are 1. All other entires are 0 - 'box_indices': `Tensor` of shape [max_num_instances, 2], where the first - num_boxes entries contain the y-center and x-center of a valid box. - These are used to extract the regressed box features from the - prediction when computing the loss - """ - - # Get relevant bounding box and class information from labels - # only keep the first num_objects boxes and classes - num_objects = labels['num_detections'] - boxes = labels['bbox'][:num_objects] - classes = labels['classes'][:num_objects] - self._class_offset - - # Compute scaling factors for center/corner positions on heatmap - input_size = tf.cast(input_size, self._dtype) - output_size = tf.cast(output_size, self._dtype) - input_h, input_w = input_size[0], input_size[1] - output_h, output_w = output_size[0], output_size[1] - - width_ratio = output_w / input_w - height_ratio = output_h / input_h - - # Original box coordinates - ytl, ybr = boxes[..., 0], boxes[..., 2] - xtl, xbr = boxes[..., 1], boxes[..., 3] - yct = (ytl + ybr) / 2 - xct = (xtl + xbr) / 2 - - # Scaled box coordinates (could be floating point) - fxtl = xtl * width_ratio - fytl = ytl * height_ratio - fxbr = xbr * width_ratio - fybr = ybr * height_ratio - fxct = xct * width_ratio - fyct = yct * height_ratio - - # Floor the scaled box coordinates to be placed on heatmaps - xtl = tf.math.floor(fxtl) - ytl = tf.math.floor(fytl) - xbr = tf.math.floor(fxbr) - ybr = tf.math.floor(fybr) - xct = tf.math.floor(fxct) - yct = tf.math.floor(fyct) - - # Offset computations to make up for discretization error - # used for offset maps - # tl_offset_values = tf.stack([fxtl - xtl, fytl - ytl], axis=-1) - # br_offset_values = tf.stack([fxbr - xbr, fybr - ybr], axis=-1) - ct_offset_values = tf.stack([fxct - xct, fyct - yct], axis=-1) - - # Get the scaled box dimensions for computing the gaussian radius - box_widths = boxes[..., 3] - boxes[..., 1] - box_heights = boxes[..., 2] - boxes[..., 0] - - box_widths = box_widths * width_ratio - box_heights = box_heights * height_ratio - - # Used for size map - box_widths_heights = tf.stack([box_widths, box_heights], axis=-1) - - # Center/corner heatmaps - # tl_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) - # br_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) - ct_heatmap = tf.zeros((output_h, output_w, self._num_classes), self._dtype) - - # Maps for offset and size features for each instance of a box - # tl_offset = tf.zeros((self._max_num_instances, 2), self._dtype) - # br_offset = tf.zeros((self._max_num_instances, 2), self._dtype) - ct_offset = tf.zeros((self._max_num_instances, 2), self._dtype) - size = tf.zeros((self._max_num_instances, 2), self._dtype) - - # Mask for valid box instances and their center indices in the heatmap - box_mask = tf.zeros((self._max_num_instances), tf.int32) - box_indices = tf.zeros((self._max_num_instances, 2), tf.int32) - - if self._use_gaussian_bump: - # Need to gaussians around the centers and corners of the objects - - # First compute the desired gaussian radius - if self._gaussian_rad == -1: - radius = tf.map_fn(fn=lambda x: preprocessing_ops.gaussian_radius(x), - elems=tf.math.ceil(box_widths_heights)) - radius = tf.math.maximum(tf.math.floor(radius), - tf.cast(1.0, radius.dtype)) - else: - radius = tf.constant([self._gaussian_rad] * num_objects, self._dtype) - # These blobs contain information needed to draw the gaussian - # tl_blobs = tf.stack([classes, xtl, ytl, radius], axis=-1) - # br_blobs = tf.stack([classes, xbr, ybr, radius], axis=-1) - ct_blobs = tf.stack([classes, xct, yct, radius], axis=-1) - - # Get individual gaussian contributions from each bounding box - # tl_gaussians = tf.map_fn( - # fn=lambda x: preprocessing_ops.draw_gaussian( - # tf.shape(tl_heatmap), x, self._dtype), elems=tl_blobs) - # br_gaussians = tf.map_fn( - # fn=lambda x: preprocessing_ops.draw_gaussian( - # tf.shape(br_heatmap), x, self._dtype), elems=br_blobs) - ct_gaussians = tf.map_fn( - fn=lambda x: preprocessing_ops.draw_gaussian( - tf.shape(ct_heatmap), x, self._dtype), elems=ct_blobs) - - # Combine contributions into single heatmaps - # tl_heatmap = tf.math.reduce_max(tl_gaussians, axis=0) - # br_heatmap = tf.math.reduce_max(br_gaussians, axis=0) - ct_heatmap = tf.math.reduce_max(ct_gaussians, axis=0) - - else: - # Instead of a gaussian, insert 1s in the center and corner heatmaps - # tl_hm_update_indices = tf.cast( - # tf.stack([ytl, xtl, classes], axis=-1), tf.int32) - # br_hm_update_indices = tf.cast( - # tf.stack([ybr, xbr, classes], axis=-1), tf.int32) - ct_hm_update_indices = tf.cast( - tf.stack([yct, xct, classes], axis=-1), tf.int32) - - # tl_heatmap = tf.tensor_scatter_nd_update(tl_heatmap, - # tl_hm_update_indices, [1] * num_objects) - # br_heatmap = tf.tensor_scatter_nd_update(br_heatmap, - # br_hm_update_indices, [1] * num_objects) - ct_heatmap = tf.tensor_scatter_nd_update(ct_heatmap, - ct_hm_update_indices, [1] * num_objects) - - # Indices used to update offsets and sizes for valid box instances - update_indices = preprocessing_ops.cartesian_product( - tf.range(num_objects), tf.range(2)) - update_indices = tf.reshape(update_indices, shape=[num_objects, 2, 2]) - - # Write the offsets of each box instance - # tl_offset = tf.tensor_scatter_nd_update( - # tl_offset, update_indices, tl_offset_values) - # br_offset = tf.tensor_scatter_nd_update( - # br_offset, update_indices, br_offset_values) - ct_offset = tf.tensor_scatter_nd_update( - ct_offset, update_indices, ct_offset_values) - - # Write the size of each bounding box - size = tf.tensor_scatter_nd_update(size, update_indices, box_widths_heights) - - # Initially the mask is zeros, so now we unmask each valid box instance - mask_indices = tf.expand_dims(tf.range(num_objects), -1) - mask_values = tf.repeat(1, num_objects) - box_mask = tf.tensor_scatter_nd_update(box_mask, mask_indices, mask_values) - - # Write the y and x coordinate of each box center in the heatmap - box_index_values = tf.cast(tf.stack([yct, xct], axis=-1), dtype=tf.int32) - box_indices = tf.tensor_scatter_nd_update( - box_indices, update_indices, box_index_values) - - labels = { - # 'tl_heatmaps': tl_heatmap, - # 'br_heatmaps': br_heatmap, - 'ct_heatmaps': ct_heatmap, - # 'tl_offset': tl_offset, - # 'br_offset': br_offset, - 'ct_offset': ct_offset, - 'size': size, - 'box_mask': box_mask, - 'box_indices': box_indices - } - return labels - def _parse_train_data(self, data): """Generates images and labels that are usable for model training. @@ -321,13 +106,12 @@ def _parse_eval_data(self, data): labels: a dict of Tensors that contains labels. """ image = tf.cast(data['image'], dtype=tf.float32) - channel_means = [104.01362025, 114.03422265, 119.9165958] #bgr - channel_stds = [73.6027665 , 69.89082075, 70.9150767] #bgr - red, green, blue = tf.unstack(image, num=3, axis=2) - image = tf.stack([blue, green, red], axis=2) + if self._bgr_ordering: + red, green, blue = tf.unstack(image, num=3, axis=2) + image = tf.stack([blue, green, red], axis=2) - image = normalize_image(image, offset=channel_means, scale=channel_stds) + image = normalize_image(image, offset=self._channel_means, scale=self._channel_stds) boxes = data['groundtruth_boxes'] classes = data['groundtruth_classes'] @@ -389,11 +173,6 @@ def _build_label(self, image, boxes, classes, width, height, info, data, 'num_detections': num_detections } - # heatmap_feature_labels = self._build_heatmap_and_regressed_features( - # labels, output_size=[self._output_dims, self._output_dims], - # input_size=[self._image_h, self._image_w] - # ) - # labels.update(heatmap_feature_labels) return image, labels def postprocess_fn(self, is_training): @@ -404,7 +183,8 @@ def postprocess_fn(self, is_training): if __name__ == '__main__': - # This code is for visualization + # This code is for visualization (outdated since we moved ground truth label + # builder to the loss function) import matplotlib.pyplot as plt diff --git a/centernet/demo.py b/centernet/demo.py index 8b2275a67..6f66d2927 100644 --- a/centernet/demo.py +++ b/centernet/demo.py @@ -4,6 +4,7 @@ from centernet.modeling.CenterNet import build_centernet from centernet.utils.weight_utils.load_weights import ( get_model_weights_as_dict, load_weights_backbone, load_weights_model) +from official.vision.beta.ops.preprocess_ops import normalize_image from yolo.demos.video_detect_cpu import runner from yolo.demos.video_detect_gpu import FastVideo from yolo.utils.run_utils import prep_gpu @@ -11,6 +12,18 @@ CENTERNET_CKPT_PATH = 'D:\\weights\centernet_hg104_512x512_coco17_tpu-8\checkpoint' CLIP_PATH = r'D:\\Documents\Research\Software\nyc_demo_fast.mp4' +def preprocess_fn(image, + channel_means=(104.01362025, 114.03422265, 119.9165958), + channel_stds=(73.6027665 , 69.89082075, 70.9150767)): + + image = tf.cast(image, dtype=tf.float32) + red, green, blue = tf.unstack(image, num=3, axis=3) + image = tf.stack([blue, green, red], axis=3) + image = normalize_image(image, offset=channel_means, scale=channel_stds) + + return image + + if __name__ == '__main__': prep_gpu() input_specs = tf.keras.layers.InputSpec(shape=[1, 512, 512, 3]) @@ -25,6 +38,7 @@ cap = FastVideo( CLIP_PATH, # set to 0 if using webcam model=model, + preprocess_function=preprocess_fn, process_width=512, process_height=512, preprocess_with_gpu=False, diff --git a/centernet/modeling/layers/nn_blocks.py b/centernet/modeling/layers/nn_blocks.py index baa8255d4..5a01337fd 100644 --- a/centernet/modeling/layers/nn_blocks.py +++ b/centernet/modeling/layers/nn_blocks.py @@ -34,7 +34,8 @@ def __init__(self, norm_epsilon=0.001, **kwargs): """A residual block with BN after convolutions. Modified with padding for - the CenterNet model. + the CenterNet model. The input is first padded with 0 along the top, bottom, + left, and right prior to the first convolutional layer. Args: filters: `int` number of filters for the first two convolutions. Note that @@ -216,7 +217,9 @@ def __init__(self, such that it is compatiable with the CenterNet backbone. The Layer is a standards combination of Conv BatchNorm Activation, however, the use of bias in the conv is determined by the use of batch - normalization. Modified with padding for the CenterNet model. + normalization. Modified with padding for the CenterNet model. The input is + first padded with 0 along the top, bottom, left, and right prior to the + first convolutional layer. Cross Stage Partial networks (CSPNets) were proposed in: [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang Chen, Jun-Wei Hsieh diff --git a/centernet/tasks/centernet.py b/centernet/tasks/centernet.py index a3693238d..c4f519420 100644 --- a/centernet/tasks/centernet.py +++ b/centernet/tasks/centernet.py @@ -34,10 +34,10 @@ def build_inputs(self, params, input_context=None): parser = centernet_input.CenterNetParser( image_w=params.parser.image_w, image_h=params.parser.image_h, - num_classes=model.num_classes, max_num_instances=params.parser.max_num_instances, - gaussian_iou=params.parser.gaussian_iou, - output_dims=params.parser.output_dims, + bgr_ordering=params.parser.bgr_ordering, + channel_means=params.parser.channel_means, + channel_stds=params.parser.channel_stds, dtype=params.parser.dtype ) From a5867588116a618b7cee91f146912af5316b9a64 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 30 Apr 2021 21:07:22 +0000 Subject: [PATCH 130/132] tested eval for new preprocessing on vm --- centernet/configs/centernet.py | 1 + centernet/dataloaders/centernet_input.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/centernet/configs/centernet.py b/centernet/configs/centernet.py index 80a72c0bf..197a7163e 100644 --- a/centernet/configs/centernet.py +++ b/centernet/configs/centernet.py @@ -98,6 +98,7 @@ class DataDecoder(hyperparams.OneOfConfig): class Parser(hyperparams.Config): image_w: int = 512 image_h: int = 512 + max_num_instances: int = 128 bgr_ordering: bool = True channel_means: List[int] = dataclasses.field( default_factory=lambda: [104.01362025, 114.03422265, 119.9165958]) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 0bf3940df..8d96fdaa1 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -7,6 +7,8 @@ from official.vision.beta.ops.preprocess_ops import normalize_image from yolo.ops import preprocessing_ops as yolo_preprocessing_ops +from typing import List + def pad_max_instances(value, instances, pad_value=0, pad_axis=0): shape = tf.shape(value) From 516bd68cd0ece54f63d581c374dd6dc1a6a1e35f Mon Sep 17 00:00:00 2001 From: Anirudh Vegesana Date: Wed, 12 May 2021 15:17:50 -0400 Subject: [PATCH 131/132] Create README.md --- centernet/README.md | 130 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 centernet/README.md diff --git a/centernet/README.md b/centernet/README.md new file mode 100644 index 000000000..5f17173f7 --- /dev/null +++ b/centernet/README.md @@ -0,0 +1,130 @@ +> :memo: A README.md template for releasing a paper code implementation to a GitHub repository. +> +> * Template version: 1.0.2020.170 +> * Please modify sections depending on needs. + +# CenterNet + +[![Paper](http://img.shields.io/badge/Paper-arXiv.1904.07850-B3181B?logo=arXiv)](https://arxiv.org/abs/1904.07850) + +This repository is the unofficial implementation of the following paper. + +* Paper title: [Objects as Points](https://arxiv.org/abs/1904.07850) + +## Description + +CenterNet [1] builds upon CornerNet [2], an anchor-free model for object +detection. + +Many other models, such as YOLO and RetinaNet, use anchor boxes. These anchor +boxes are predefined to be close to the aspect ratios and scales of the objects +in the training dataset. Anchor-based models do not predict the bounding boxes +of objects directly. They instead predict the location and size/shape +refinements to a predefined anchor box. The detection generator then computes +the final confidences, positions, and size of the detection. + +CornerNet eliminates the need for anchor boxes. RetinaNet needs thousands of +anchor boxes in order to cover the most common ground truth boxes [2]. This adds +unnecessary complexity to the model which slow down training and create +imbalances in positive and negative anchor boxes [2]. Instead, CornerNet creates +heatmaps for each of the corners and pools them together in order to get the +final detection boxes for the objects. CenterNet removes even more complexity +by using the center instead of the corners, meaning that only one set of +heatmaps (one heatmap for each class) is needed to predict the object. CenterNet +proves that this can be done without a significant difference in accuracy. + +## History + +> :memo: Provide a changelog. + +## Authors or Maintainers + +> :memo: Provide maintainer information. + +* Full name ([@GitHub username](https://github.com/username)) +* Full name ([@GitHub username](https://github.com/username)) + +## Table of Contents + +> :memo: Provide a table of contents to help readers navigate a lengthy README document. + +## Requirements + +[![TensorFlow 2.4](https://img.shields.io/badge/TensorFlow-2.4-FF6F00?logo=tensorflow)](https://github.com/tensorflow/tensorflow/releases/tag/v2.4.0) +[![Python 3.7](https://img.shields.io/badge/Python-3.7-3776AB)](https://www.python.org/downloads/release/python-370/) + +```setup +pip install -r requirements.txt +``` + +## Results + +[![TensorFlow Hub](https://img.shields.io/badge/TF%20Hub-Models-FF6F00?logo=tensorflow)](https://tfhub.dev/...) + +> :memo: Provide a table with results. (e.g., accuracy, latency) +> +> * Provide links to the pre-trained models (checkpoint, SavedModel files). +> * Publish TensorFlow SavedModel files on TensorFlow Hub (tfhub.dev) if possible. +> * Add links to [TensorBoard.dev](https://tensorboard.dev/) for visualizing metrics. +> +> An example table for image classification results +> +> ### Image Classification +> +> | Model name | Download | Top 1 Accuracy | Top 5 Accuracy | +> |------------|----------|----------------|----------------| +> | Model name | [Checkpoint](https://drive.google.com/...), [SavedModel](https://tfhub.dev/...) | xx% | xx% | + +## Dataset + +> :memo: Provide information of the dataset used. + +## Training + +> :memo: Provide training information. +> +> * Provide details for preprocessing, hyperparameters, random seeds, and environment. +> * Provide a command line example for training. + +Please run this command line for training. + +```shell +python3 ... +``` + +## Evaluation + +> :memo: Provide an evaluation script with details of how to reproduce results. +> +> * Describe data preprocessing / postprocessing steps. +> * Provide a command line example for evaluation. + +Please run this command line for evaluation. + +```shell +python3 ... +``` + +## References + +[1] CenterNet: Xingyi Zhou, Dequan Wang, and Philipp Krähenbühl. Objects as Points. arXiv preprint arXiv:1904.07850, 2019. (https://arxiv.org/abs/1904.07850) +[2] CornerNet: Hei Law and Jia Deng. CornerNet: Detecting Objects as Paired Keypoints. arXiv preprint arXiv:1808.01244, 2018. (https://arxiv.org/abs/1808.01244) + +## License + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +> :memo: Place your license text in a file named LICENSE in the root of the repository. +> +> * Include information about your license. +> * Reference: [Adding a license to a repository](https://help.github.com/en/github/building-a-strong-community/adding-a-license-to-a-repository) + +This project is licensed under the terms of the **Apache License 2.0**. + +## Citation + +> :memo: Make your repository citable. +> +> * Reference: [Making Your Code Citable](https://guides.github.com/activities/citable-code/) + +If you want to cite this repository in your research paper, please use the following information. From a14b489fc0b6b1cf7478d8da219e646ada14f40c Mon Sep 17 00:00:00 2001 From: Anirudh Vegesana Date: Wed, 12 May 2021 15:21:48 -0400 Subject: [PATCH 132/132] Create requirements.txt --- centernet/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 centernet/requirements.txt diff --git a/centernet/requirements.txt b/centernet/requirements.txt new file mode 100644 index 000000000..180e930be --- /dev/null +++ b/centernet/requirements.txt @@ -0,0 +1,3 @@ +# When using a wheel for the model garden, use the commented line instead of the requirements file +# tf-models-official +-r ../official/requirements.txt