From 6769348daf9129439baab9aa0029e5de1ffddc97 Mon Sep 17 00:00:00 2001 From: Feny Patel Date: Wed, 31 Mar 2021 20:49:12 -0400 Subject: [PATCH 1/3] data augmentation start --- centernet/dataloaders/centernet_input.py | 39 ++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 1758e0496..fe61d9e8f 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -1,7 +1,8 @@ import tensorflow as tf from official.vision.beta.dataloaders import parser - -from centernet.ops import preprocessing_ops +from official.vision.beta.ops import box_ops, preprocess_ops +from centernet.ops import preprocessing_ops as ops +from yolo.ops import preprocessing_ops class CenterNetParser(parser.Parser): def __init__( @@ -16,6 +17,11 @@ def __init__( self._gaussian_iou = gaussian_iou self._gaussian_bump = True self._gaussian_rad = -1 + self._aug_rand_zoom = aug_rand_zoom + self._mosaic_frequency + self._jitter_im + self._seed + self._random_flip def _generate_heatmap(self, boxes, output_size, input_size): boxes = tf.cast(boxes, dtype=tf.float32) @@ -127,11 +133,38 @@ def _parse_train_data(self, decoded_tensors): labels: a dict of Tensors that contains labels. """ # TODO: input size, output size - image = decoded_tensors["image"] + image = decoded_tensors["image"] / 255 labels = self._generate_heatmap( decoded_tensors["groundtruth_boxes"], output_size, input_size ) + boxes = data['groundtruth_boxes'] + + image_shape = tf.shape(image)[:2] + + #CROP + if self._aug_rand_zoom > 0.0 and self._mosaic_frequency > 0.0: + zfactor = preprocessing_ops.rand_uniform_strong(self._aug_rand_zoom, 1.0) + elif self._aug_rand_zoom > 0.0: + zfactor = preprocessing_ops.rand_scale(self._aug_rand_zoom) + else: + zfactor = tf.convert_to_tensor(1.0) + + image, crop_info = preprocessing_ops.random_op_image( + image, self._jitter_im, zfactor, zfactor, self._aug_rand_translate) + + + #RESIZE + + #CLIP DETECTION TO BOUNDARIES + + #RANDOM HORIZONTAL FLIP + if self._random_flip: + image, boxes, _ = preprocess_ops.random_horizontal_flip( + image, boxes, seed=self._seed) + + + return image, labels def _parse_eval_data(self, data): From f6d3a9bed709c56077825b45c166003c5ad146b3 Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Fri, 2 Apr 2021 01:05:24 -0700 Subject: [PATCH 2/3] added augmentations to data pipeline --- centernet/dataloaders/centernet_input.py | 46 ++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index fe61d9e8f..12e67a7ba 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -11,6 +11,10 @@ def __init__( max_num_instances: int, gaussian_iou: float, + aug_rand_saturation=True, + aug_rand_brightness=True, + aug_rand_zoom=True, + aug_rand_hue=True, ): self._num_classes = num_classes self._max_num_instances = max_num_instances @@ -134,11 +138,13 @@ def _parse_train_data(self, decoded_tensors): """ # TODO: input size, output size image = decoded_tensors["image"] / 255 + labels = self._generate_heatmap( decoded_tensors["groundtruth_boxes"], output_size, input_size ) - boxes = data['groundtruth_boxes'] + boxes = decoded_tensors['groundtruth_boxes'] + classes = decoded_tensors['groundtruth_classes'] image_shape = tf.shape(image)[:2] @@ -150,20 +156,56 @@ def _parse_train_data(self, decoded_tensors): else: zfactor = tf.convert_to_tensor(1.0) + # TODO: random_op_image not defined image, crop_info = preprocessing_ops.random_op_image( image, self._jitter_im, zfactor, zfactor, self._aug_rand_translate) + #RESIZE + shape = tf.shape(image) + width = shape[1] + height = shape[0] + image, boxes, classes = preprocessing_ops.resize_crop_filter( + image, + boxes, + classes, + default_width=width, # randscale * self._net_down_scale, + default_height=height, # randscale * self._net_down_scale, + target_width=self._image_w, + target_height=self._image_h, + randomize=False) #CLIP DETECTION TO BOUNDARIES + + #RANDOM HORIZONTAL FLIP if self._random_flip: image, boxes, _ = preprocess_ops.random_horizontal_flip( image, boxes, seed=self._seed) - + # Color and lighting jittering + image = tf.image.rgb_to_hsv(image) + i_h, i_s, i_v = tf.split(image, 3, axis=-1) + if self._aug_rand_hue: + delta = preprocessing_ops.rand_uniform_strong( + -0.1, 0.1 + ) # tf.random.uniform([], minval= -0.1,maxval=0.1, seed=self._seed, dtype=tf.float32) + i_h = i_h + delta # Hue + i_h = tf.clip_by_value(i_h, 0.0, 1.0) + if self._aug_rand_saturation: + delta = preprocessing_ops.rand_scale( + 0.75 + ) # tf.random.uniform([], minval= 0.5,maxval=1.1, seed=self._seed, dtype=tf.float32) + i_s = i_s * delta + if self._aug_rand_brightness: + delta = preprocessing_ops.rand_scale( + 0.75 + ) # tf.random.uniform([], minval= -0.15,maxval=0.15, seed=self._seed, dtype=tf.float32) + i_v = i_v * delta + image = tf.concat([i_h, i_s, i_v], axis=-1) + image = tf.image.hsv_to_rgb(image) return image, labels From 836e8d098bdf58bd642be3355d10bba7647c05e4 Mon Sep 17 00:00:00 2001 From: Joshua Yeung Date: Wed, 7 Apr 2021 00:05:54 -0700 Subject: [PATCH 3/3] added data augmentations to centernet_input --- centernet/dataloaders/centernet_input.py | 44 +++++++++++++----------- centernet/ops/preprocessing_ops.py | 8 ++--- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/centernet/dataloaders/centernet_input.py b/centernet/dataloaders/centernet_input.py index 12e67a7ba..d6517efd5 100644 --- a/centernet/dataloaders/centernet_input.py +++ b/centernet/dataloaders/centernet_input.py @@ -15,6 +15,7 @@ def __init__( aug_rand_brightness=True, aug_rand_zoom=True, aug_rand_hue=True, + seed=1, ): self._num_classes = num_classes self._max_num_instances = max_num_instances @@ -22,10 +23,10 @@ def __init__( self._gaussian_bump = True self._gaussian_rad = -1 self._aug_rand_zoom = aug_rand_zoom - self._mosaic_frequency - self._jitter_im - self._seed - self._random_flip + # self._mosaic_frequency + # self._jitter_im + self._seed = seed + # self._random_flip def _generate_heatmap(self, boxes, output_size, input_size): boxes = tf.cast(boxes, dtype=tf.float32) @@ -78,7 +79,7 @@ def _generate_heatmap(self, boxes, output_size, input_size): height = tf.math.ceil(height * height_ratio) if self._gaussian_rad == -1: - radius = preprocessing_ops.gaussian_radius((height, width), self._gaussian_iou) + radius = ops.gaussian_radius((height, width), self._gaussian_iou) radius = tf.math.maximum(0, tf.math.floor(radius)) else: radius = self._gaussian_rad @@ -86,9 +87,9 @@ def _generate_heatmap(self, boxes, output_size, input_size): # test # tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps[category], category, [xtl, ytl], radius) # inputs heatmap, center, radius, k=1 - tl_heatmaps = preprocessing_ops.draw_gaussian(tl_heatmaps, [[category, xtl, ytl, radius]]) - br_heatmaps = preprocessing_ops.draw_gaussian(br_heatmaps, [[category, xbr, ybr, radius]]) - ct_heatmaps = preprocessing_ops.draw_gaussian(ct_heatmaps, [[category, xct, yct, radius]], scaling_factor=5) + tl_heatmaps = ops.draw_gaussian(tl_heatmaps, [[category, xtl, ytl, radius]]) + br_heatmaps = ops.draw_gaussian(br_heatmaps, [[category, xbr, ybr, radius]]) + ct_heatmaps = ops.draw_gaussian(ct_heatmaps, [[category, xct, yct, radius]], scaling_factor=5) else: # TODO: See if this is a typo @@ -136,6 +137,7 @@ def _parse_train_data(self, decoded_tensors): images: the image tensor. labels: a dict of Tensors that contains labels. """ + print('running _parse_train_data') # TODO: input size, output size image = decoded_tensors["image"] / 255 @@ -148,20 +150,21 @@ def _parse_train_data(self, decoded_tensors): image_shape = tf.shape(image)[:2] + print('image_shape: ', image_shape) #CROP - if self._aug_rand_zoom > 0.0 and self._mosaic_frequency > 0.0: - zfactor = preprocessing_ops.rand_uniform_strong(self._aug_rand_zoom, 1.0) - elif self._aug_rand_zoom > 0.0: + # if self._aug_rand_zoom > 0.0 and self._mosaic_frequency > 0.0: + # zfactor = preprocessing_ops.rand_uniform_strong(self._aug_rand_zoom, 1.0) + if self._aug_rand_zoom > 0.0: zfactor = preprocessing_ops.rand_scale(self._aug_rand_zoom) else: zfactor = tf.convert_to_tensor(1.0) - # TODO: random_op_image not defined - image, crop_info = preprocessing_ops.random_op_image( - image, self._jitter_im, zfactor, zfactor, self._aug_rand_translate) + # # TODO: random_op_image not defined + # image, crop_info = preprocessing_ops.random_op_image( + # image, self._jitter_im, zfactor, zfactor, self._aug_rand_translate) + + image = tf.image.stateless_random_crop(image, size=[image_shape[0]*zfactor, image_shape[1]*zfactor, 3], seed = seed) - - #RESIZE shape = tf.shape(image) width = shape[1] @@ -177,9 +180,8 @@ def _parse_train_data(self, decoded_tensors): randomize=False) #CLIP DETECTION TO BOUNDARIES + boxes = box_ops.clip_boxes(boxes, shape) - - #RANDOM HORIZONTAL FLIP if self._random_flip: image, boxes, _ = preprocess_ops.random_horizontal_flip( @@ -245,7 +247,9 @@ def generate_heatmaps(self, dectections): # tl_heatmaps, br_heatmaps, ct_heatmaps = generate_heatmaps(1, 2, (416, 416), detections) # ct_heatmaps[batch_id, class_id, ...] - plt.imshow(ct_heatmaps[0, ...]) - plt.show() + + # plt.imshow(ct_heatmaps[0, ...]) + # plt.show() + # This is to run the test # tf.test.main() diff --git a/centernet/ops/preprocessing_ops.py b/centernet/ops/preprocessing_ops.py index 784ab3e53..91d6ee294 100644 --- a/centernet/ops/preprocessing_ops.py +++ b/centernet/ops/preprocessing_ops.py @@ -149,9 +149,9 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): left, right = tf.math.minimum(x, radius), tf.math.minimum(width - x, radius + 1) top, bottom = tf.math.minimum(y, radius), tf.math.minimum(height - y, radius + 1) - print('heatmap ',heatmap) - print(len(heatmap)) - print('category ',category) + # print('heatmap ',heatmap) + # print(len(heatmap)) + # print('category ',category) # TODO: make sure this replicates original functionality # masked_heatmap = heatmap[0, category, y - top:y + bottom, x - left:x + right] @@ -180,7 +180,7 @@ def draw_gaussian(heatmap, blobs, scaling_factor=1, dtype=tf.float32): heatmap_mask = heatmap_mask_ta.stack() heatmap_mask = tf.reshape(heatmap_mask, (-1, 3)) heatmap = tf.tensor_scatter_nd_max(heatmap, heatmap_mask, masked_gaussian * scaling_factor) - print('after ',heatmap) + # print('after ',heatmap) return heatmap # def draw_gaussian(heatmap, category, center, radius, scaling_factor=1):