commit 6a16d3fafb992c026d09d883ae148828cc57796d Author: wangjian Date: Fri Mar 10 22:06:22 2023 +0800 初始化模型应用 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e66840f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.7.7-slim-stretch +ENV PYTHONUNBUFFERED 1 +RUN sed -i s@/deb.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list +RUN cat /etc/apt/sources.list +RUN apt-get update \ + && apt-get install -y make \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN mkdir -p /app +WORKDIR /app +COPY requirements.txt /app +RUN python -m venv . +RUN pip install pip==20.1.1 +RUN pip install setuptools==46.1.3 +RUN pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +COPY ./app /app +EXPOSE 5000 +CMD ["gunicorn", "--bind", ":5000", "server:app"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..5a8419d --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +# 道路病害检测 + +## 利用了cnn网络和unet网络进行道路裂缝和坑洼图片的检测. + +## API 接口 + +### 道路裂缝检测接口(U-Net CNN) + +- 请求 + +```curl -k -X POST -F 'image=@image_path/ -v http://0.0.0.0:5000/segment ``` + +- 返回接口 + +| 名称 | 参数 | 类型 | 说明 | +|------|------|-------|-------| +| 返回结果 | result | bool | 是否有裂缝 | +| 返回图片 | img | string | 图像的base64编码字符串 | + + +### 道路坑洼检测接口(R-CNN) + +```curl -k -X POST -F 'image=@image_path/ -v http://0.0.0.0:5000/detect/rcnn ``` + + +- 返回接口 + +| 名称 | 参数 | 类型 | 说明 | +|------|------|-------|-------| +| 返回结果 | result | bool | 是否有坑洼 | +| 返回图片 | img | string | 图像的base64编码字符串 | + + +### 裂缝和坑洼检测接口 + +```curl -k -X POST -F 'image=@image_path/ -v http://0.0.0.0:5000/ ``` + + +- 返回接口 + +| 名称 | 参数 | 类型 | 说明 | +|------|------|--------|------------------| +| 接口编码 | code | int | 0：正常 ； 10001： 异常 | +| 原始图片 | img_src | string | 图像的base64编码字符串 | +| 是否有裂缝 | crack | bool | 是否有裂缝 | +| 是否有坑洼 | pothole | bool | 是否有坑洼 | diff --git a/app/UNet_25_Crack.tflite b/app/UNet_25_Crack.tflite new file mode 100644 index 0000000..f507d29 Binary files /dev/null and b/app/UNet_25_Crack.tflite differ diff --git a/app/frozen_inference_graph.pb b/app/frozen_inference_graph.pb new file mode 100644 index 0000000..6b12ced Binary files /dev/null and b/app/frozen_inference_graph.pb differ diff --git a/app/ops.py b/app/ops.py new file mode 100644 index 0000000..7c88fe4 --- /dev/null +++ b/app/ops.py @@ -0,0 +1,70 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""A module for helper tensorflow ops.""" + +import tensorflow as tf + + +def reframe_box_masks_to_image_masks(box_masks, boxes, image_height, + image_width): + """Transforms the box masks back to full image masks. + + Embeds masks in bounding boxes of larger masks whose shapes correspond to + image shape. + + Args: + box_masks: A tf.float32 tensor of size [num_masks, mask_height, mask_width]. + boxes: A tf.float32 tensor of size [num_masks, 4] containing the box + corners. Row i contains [ymin, xmin, ymax, xmax] of the box + corresponding to mask i. Note that the box corners are in + normalized coordinates. + image_height: Image height. The output mask will have the same height as + the image height. + image_width: Image width. The output mask will have the same width as the + image width. + + Returns: + A tf.float32 tensor of size [num_masks, image_height, image_width]. + """ + + # TODO(rathodv): Make this a public function. + def reframe_box_masks_to_image_masks_default(): + """The default function when there are more than 0 box masks.""" + + def transform_boxes_relative_to_boxes(boxes, reference_boxes): + boxes = tf.reshape(boxes, [-1, 2, 2]) + min_corner = tf.expand_dims(reference_boxes[:, 0:2], 1) + max_corner = tf.expand_dims(reference_boxes[:, 2:4], 1) + transformed_boxes = (boxes - min_corner) / (max_corner - min_corner) + return tf.reshape(transformed_boxes, [-1, 4]) + + box_masks_expanded = tf.expand_dims(box_masks, axis=3) + num_boxes = tf.shape(box_masks_expanded)[0] + unit_boxes = tf.concat( + [tf.zeros([num_boxes, 2]), tf.ones([num_boxes, 2])], axis=1) + reverse_boxes = transform_boxes_relative_to_boxes(unit_boxes, boxes) + return tf.image.crop_and_resize( + image=box_masks_expanded, + boxes=reverse_boxes, + box_ind=tf.range(num_boxes), + crop_size=[image_height, image_width], + extrapolation_value=0.0) + + image_masks = tf.cond( + tf.shape(box_masks)[0] > 0, + reframe_box_masks_to_image_masks_default, + lambda: tf.zeros([0, image_height, image_width, 1], dtype=tf.float32)) + return tf.squeeze(image_masks, axis=3) diff --git a/app/serve.py b/app/serve.py new file mode 100644 index 0000000..6a5e6d0 --- /dev/null +++ b/app/serve.py @@ -0,0 +1,27 @@ +import tensorflow as tf +import numpy as np + + +def serve_unet_model(): + TFLITE_MODEL = "/app/UNet_25_Crack.tflite" + + tflite_interpreter = tf.lite.Interpreter(model_path=TFLITE_MODEL) + + input_details = tflite_interpreter.get_input_details() + output_details = tflite_interpreter.get_output_details() + tflite_interpreter.allocate_tensors() + height = input_details[0]['shape'][1] + width = input_details[0]['shape'][2] + + return tflite_interpreter, height, width, input_details, output_details + + +def serve_rcnn_model(): + detection_graph = tf.Graph() + with detection_graph.as_default(): + od_graph_def = tf.compat.v1.GraphDef() + with tf.compat.v1.gfile.GFile("/app/frozen_inference_graph.pb", 'rb') as fid: + serialized_graph = fid.read() + od_graph_def.ParseFromString(serialized_graph) + tf.import_graph_def(od_graph_def, name='') + return detection_graph diff --git a/app/server.py b/app/server.py new file mode 100644 index 0000000..1b1bd47 --- /dev/null +++ b/app/server.py @@ -0,0 +1,239 @@ +import base64 + +import flask +from flask import Flask, jsonify +import numpy as np +import io +from PIL import Image, ImageDraw +import tensorflow as tf +import ops as utils_ops +import visualization_utils as vis_util + +from serve import serve_unet_model +from serve import serve_rcnn_model + +app = Flask(__name__) + + +def load_unet_model(): + global tflite_interpreter_c, height_c, width_c, input_details_c, output_details_c + tflite_interpreter_c, height_c, width_c, input_details_c, output_details_c = serve_unet_model() + + +def load_rcnn_model(): + global detection_graph + detection_graph = serve_rcnn_model() + + +load_unet_model() +load_rcnn_model() + + +def prepare_img(image, type): + if type == "detect": + return image.resize((width, height)) + elif type == "segment": + return image.resize((width_c, height_c)) + + +def load_image_into_numpy_array(image): + (im_width, im_height) = image.size + return np.array(image.getdata()).reshape( + (im_height, im_width, 3)).astype(np.uint8) + + +@app.route("/detect/rcnn", methods=["POST"]) +def detect_rcnn(): + if flask.request.method == "POST": + if flask.request.files.get("image"): + image = Image.open(flask.request.files["image"]) + image_np = load_image_into_numpy_array(image) + # image_np_expanded = np.expand_dims(image_np, axis=0) + output_dict = run_inference_for_single_image(image_np, detection_graph) + category_index = {0: {"name": "pothole"}, 1: {"name": "pothole"}} + print(output_dict.get('detection_masks')) + i, is_crack = vis_util.visualize_boxes_and_labels_on_image_array( + image_np, + output_dict['detection_boxes'], + output_dict['detection_classes'], + output_dict['detection_scores'], + category_index, + instance_masks=output_dict.get('detection_masks'), + use_normalized_coordinates=True, + line_thickness=8, + skip_scores=True, + skip_labels=True) + img = Image.fromarray(image_np.astype("uint8")) + img = img.resize((128, 128)) + raw_bytes = io.BytesIO() + img.save(raw_bytes, "JPEG") + raw_bytes.seek(0) + img_byte = raw_bytes.getvalue() + img_str = base64.b64encode(img_byte) + data = { + "result": is_crack, + "img": img_str.decode('utf-8') + } + return jsonify(data) + else: + return "Could not find image" + return "Please use POST method" + + +@app.route("/segment", methods=["POST"]) +def segment(): + if flask.request.method == "POST": + if flask.request.files.get("image"): + # read the image in PIL format + img = prepare_img(Image.open(flask.request.files["image"]), "segment") + + input_data = np.expand_dims(img, axis=0) + input_data = np.float32(input_data) / 255.0 + tflite_interpreter_c.set_tensor(input_details_c[0]['index'], input_data) + tflite_interpreter_c.invoke() + result = tflite_interpreter_c.get_tensor(output_details_c[0]['index']) + result = result > 0.5 + result = result * 255 + mask = np.squeeze(result) + bg = np.asarray(img).copy() + is_crack = False + for i in range(len(mask)): + for j in range(len(mask[i])): + if mask[i][j] > 0: + bg[i][j][0] = 0 + bg[i][j][1] = 0 + bg[i][j][2] = 255 + is_crack = True + + img = Image.fromarray(bg.astype("uint8")) + raw_bytes = io.BytesIO() + img.save(raw_bytes, "JPEG") + raw_bytes.seek(0) + img_byte = raw_bytes.getvalue() + img_str = base64.b64encode(img_byte) + data = { + "result": is_crack, + "img": img_str.decode('utf-8') + } + return jsonify(data) + else: + return "Could not find image" + return "Please use POST method" + + +def run_inference_for_single_image(image, graph): + with graph.as_default(): + with tf.compat.v1.Session() as sess: + # Get handles to input and output tensors + ops = tf.compat.v1.get_default_graph().get_operations() + all_tensor_names = { + output.name for op in ops for output in op.outputs} + tensor_dict = {} + for key in [ + 'num_detections', 'detection_boxes', 'detection_scores', + 'detection_classes', 'detection_masks' + ]: + tensor_name = key + ':0' + if tensor_name in all_tensor_names: + tensor_dict[key] = tf.compat.v1.get_default_graph().get_tensor_by_name( + tensor_name) + if 'detection_masks' in tensor_dict: + # The following processing is only for single image + detection_boxes = tf.squeeze( + tensor_dict['detection_boxes'], [0]) + detection_masks = tf.squeeze( + tensor_dict['detection_masks'], [0]) + # Reframe is required to translate mask from box coordinates to image coordinates and fit the image + # size. + real_num_detection = tf.cast( + tensor_dict['num_detections'][0], tf.int32) + detection_boxes = tf.slice(detection_boxes, [0, 0], [ + real_num_detection, -1]) + detection_masks = tf.slice(detection_masks, [0, 0, 0], [ + real_num_detection, -1, -1]) + detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks( + detection_masks, detection_boxes, image.shape[0], image.shape[1]) + detection_masks_reframed = tf.cast( + tf.greater(detection_masks_reframed, 0.5), tf.uint8) + # Follow the convention by adding back the batch dimension + tensor_dict['detection_masks'] = tf.expand_dims( + detection_masks_reframed, 0) + image_tensor = tf.compat.v1.get_default_graph().get_tensor_by_name('image_tensor:0') + + # Run inference + output_dict = sess.run(tensor_dict, + feed_dict={image_tensor: np.expand_dims(image, 0)}) + + # all outputs are float32 numpy arrays, so convert types as appropriate + output_dict['num_detections'] = int( + output_dict['num_detections'][0]) + output_dict['detection_classes'] = output_dict[ + 'detection_classes'][0].astype(np.uint8) + output_dict['detection_boxes'] = output_dict['detection_boxes'][0] + output_dict['detection_scores'] = output_dict['detection_scores'][0] + if 'detection_masks' in output_dict: + output_dict['detection_masks'] = output_dict['detection_masks'][0] + return output_dict + + +@app.route('/', methods=["POST"]) +def index(): + if flask.request.method == "POST": + if flask.request.files.get("image"): + img_src = Image.open(flask.request.files["image"]) + # start crack detection + img_segment = prepare_img(img_src, "segment") + input_data = np.expand_dims(img_segment, axis=0) + input_data = np.float32(input_data) / 255.0 + tflite_interpreter_c.set_tensor(input_details_c[0]['index'], input_data) + tflite_interpreter_c.invoke() + result = tflite_interpreter_c.get_tensor(output_details_c[0]['index']) + result = result > 0.5 + result = result * 255 + mask = np.squeeze(result) + is_crack = False + for i in range(len(mask)): + for j in range(len(mask[i])): + if mask[i][j] > 0: + is_crack = True + break + + + # start pothole detection + image_np = load_image_into_numpy_array(img_src) + # image_np_expanded = np.expand_dims(image_np, axis=0) + output_dict = run_inference_for_single_image(image_np, detection_graph) + category_index = {0: {"name": "pothole"}, 1: {"name": "pothole"}} + _, is_pothole = vis_util.visualize_boxes_and_labels_on_image_array( + image_np, + output_dict['detection_boxes'], + output_dict['detection_classes'], + output_dict['detection_scores'], + category_index, + instance_masks=output_dict.get('detection_masks'), + use_normalized_coordinates=True, + line_thickness=8, + skip_scores=True, + skip_labels=True) + raw_bytes = io.BytesIO() + img_src.save(raw_bytes, "JPEG") + raw_bytes.seek(0) + img_byte = raw_bytes.getvalue() + img_str = base64.b64encode(img_byte) + data = { + "code": 0, + "crack": is_crack, + "pothole": is_pothole, + "img_src": img_str.decode('utf-8') + } + return jsonify(data) + else: + data = { + "code": 10001, + "msg": "Could not find image" + } + return jsonify(data) + return "Road Damage Detection" + +if __name__ == "__main__": + app.run() diff --git a/app/visualization_utils.py b/app/visualization_utils.py new file mode 100644 index 0000000..91c98f9 --- /dev/null +++ b/app/visualization_utils.py @@ -0,0 +1,508 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""A set of functions that are used for visualization. +These functions often receive an image, perform some visualization on the image. +The functions do not return a value, instead they modify the image itself. +""" +import abc +import collections +# Set headless-friendly backend. +import matplotlib; + +matplotlib.use('Agg') # pylint: disable=multiple-statements +import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top +import numpy as np +import PIL.Image as Image +import PIL.ImageColor as ImageColor +import PIL.ImageDraw as ImageDraw +import PIL.ImageFont as ImageFont +import six +from six.moves import range +from six.moves import zip +import tensorflow as tf + +_TITLE_LEFT_MARGIN = 10 +_TITLE_TOP_MARGIN = 10 +STANDARD_COLORS = [ + 'AliceBlue', 'Chartreuse', 'Aqua', 'Aquamarine', 'Azure', 'Beige', 'Bisque', + 'BlanchedAlmond', 'BlueViolet', 'BurlyWood', 'CadetBlue', 'AntiqueWhite', + 'Chocolate', 'Coral', 'CornflowerBlue', 'Cornsilk', 'Crimson', 'Cyan', + 'DarkCyan', 'DarkGoldenRod', 'DarkGrey', 'DarkKhaki', 'DarkOrange', + 'DarkOrchid', 'DarkSalmon', 'DarkSeaGreen', 'DarkTurquoise', 'DarkViolet', + 'DeepPink', 'DeepSkyBlue', 'DodgerBlue', 'FireBrick', 'FloralWhite', + 'ForestGreen', 'Fuchsia', 'Gainsboro', 'GhostWhite', 'Gold', 'GoldenRod', + 'Salmon', 'Tan', 'HoneyDew', 'HotPink', 'IndianRed', 'Ivory', 'Khaki', + 'Lavender', 'LavenderBlush', 'LawnGreen', 'LemonChiffon', 'LightBlue', + 'LightCoral', 'LightCyan', 'LightGoldenRodYellow', 'LightGray', 'LightGrey', + 'LightGreen', 'LightPink', 'LightSalmon', 'LightSeaGreen', 'LightSkyBlue', + 'LightSlateGray', 'LightSlateGrey', 'LightSteelBlue', 'LightYellow', 'Lime', + 'LimeGreen', 'Linen', 'Magenta', 'MediumAquaMarine', 'MediumOrchid', + 'MediumPurple', 'MediumSeaGreen', 'MediumSlateBlue', 'MediumSpringGreen', + 'MediumTurquoise', 'MediumVioletRed', 'MintCream', 'MistyRose', 'Moccasin', + 'NavajoWhite', 'OldLace', 'Olive', 'OliveDrab', 'Orange', 'OrangeRed', + 'Orchid', 'PaleGoldenRod', 'PaleGreen', 'PaleTurquoise', 'PaleVioletRed', + 'PapayaWhip', 'PeachPuff', 'Peru', 'Pink', 'Plum', 'PowderBlue', 'Purple', + 'Red', 'RosyBrown', 'RoyalBlue', 'SaddleBrown', 'Green', 'SandyBrown', + 'SeaGreen', 'SeaShell', 'Sienna', 'Silver', 'SkyBlue', 'SlateBlue', + 'SlateGray', 'SlateGrey', 'Snow', 'SpringGreen', 'SteelBlue', 'GreenYellow', + 'Teal', 'Thistle', 'Tomato', 'Turquoise', 'Violet', 'Wheat', 'White', + 'WhiteSmoke', 'Yellow', 'YellowGreen' +] + + +def _get_multiplier_for_color_randomness(): + """Returns a multiplier to get semi-random colors from successive indices. + This function computes a prime number, p, in the range [2, 17] that: + - is closest to len(STANDARD_COLORS) / 10 + - does not divide len(STANDARD_COLORS) + If no prime numbers in that range satisfy the constraints, p is returned as 1. + Once p is established, it can be used as a multiplier to select + non-consecutive colors from STANDARD_COLORS: + colors = [(p * i) % len(STANDARD_COLORS) for i in range(20)] + """ + num_colors = len(STANDARD_COLORS) + prime_candidates = [5, 7, 11, 13, 17] + + # Remove all prime candidates that divide the number of colors. + prime_candidates = [p for p in prime_candidates if num_colors % p] + if not prime_candidates: + return 1 + + # Return the closest prime number to num_colors / 10. + abs_distance = [np.abs(num_colors / 10. - p) for p in prime_candidates] + num_candidates = len(abs_distance) + inds = [i for _, i in sorted(zip(abs_distance, range(num_candidates)))] + return prime_candidates[inds[0]] + + +def draw_bounding_box_on_image_array(image, + ymin, + xmin, + ymax, + xmax, + color='red', + thickness=4, + display_str_list=(), + use_normalized_coordinates=True): + """Adds a bounding box to an image (numpy array). + Bounding box coordinates can be specified in either absolute (pixel) or + normalized coordinates by setting the use_normalized_coordinates argument. + Args: + image: a numpy array with shape [height, width, 3]. + ymin: ymin of bounding box. + xmin: xmin of bounding box. + ymax: ymax of bounding box. + xmax: xmax of bounding box. + color: color to draw bounding box. Default is red. + thickness: line thickness. Default value is 4. + display_str_list: list of strings to display in box + (each to be shown on its own line). + use_normalized_coordinates: If True (default), treat coordinates + ymin, xmin, ymax, xmax as relative to the image. Otherwise treat + coordinates as absolute. + """ + image_pil = Image.fromarray(np.uint8(image)).convert('RGB') + draw_bounding_box_on_image(image_pil, ymin, xmin, ymax, xmax, color, + thickness, display_str_list, + use_normalized_coordinates) + np.copyto(image, np.array(image_pil)) + + +def draw_bounding_box_on_image(image, + ymin, + xmin, + ymax, + xmax, + color='red', + thickness=4, + display_str_list=(), + use_normalized_coordinates=True): + """Adds a bounding box to an image. + Bounding box coordinates can be specified in either absolute (pixel) or + normalized coordinates by setting the use_normalized_coordinates argument. + Each string in display_str_list is displayed on a separate line above the + bounding box in black text on a rectangle filled with the input 'color'. + If the top of the bounding box extends to the edge of the image, the strings + are displayed below the bounding box. + Args: + image: a PIL.Image object. + ymin: ymin of bounding box. + xmin: xmin of bounding box. + ymax: ymax of bounding box. + xmax: xmax of bounding box. + color: color to draw bounding box. Default is red. + thickness: line thickness. Default value is 4. + display_str_list: list of strings to display in box + (each to be shown on its own line). + use_normalized_coordinates: If True (default), treat coordinates + ymin, xmin, ymax, xmax as relative to the image. Otherwise treat + coordinates as absolute. + """ + draw = ImageDraw.Draw(image) + im_width, im_height = image.size + if use_normalized_coordinates: + (left, right, top, bottom) = (xmin * im_width, xmax * im_width, + ymin * im_height, ymax * im_height) + else: + (left, right, top, bottom) = (xmin, xmax, ymin, ymax) + if thickness > 0: + draw.line([(left, top), (left, bottom), (right, bottom), (right, top), + (left, top)], + width=thickness, + fill=color) + try: + font = ImageFont.truetype('arial.ttf', 24) + except IOError: + font = ImageFont.load_default() + + # If the total height of the display strings added to the top of the bounding + # box exceeds the top of the image, stack the strings below the bounding box + # instead of above. + display_str_heights = [font.getsize(ds)[1] for ds in display_str_list] + # Each display_str has a top and bottom margin of 0.05x. + total_display_str_height = (1 + 2 * 0.05) * sum(display_str_heights) + + if top > total_display_str_height: + text_bottom = top + else: + text_bottom = bottom + total_display_str_height + # Reverse list and print from bottom to top. + for display_str in display_str_list[::-1]: + text_width, text_height = font.getsize(display_str) + margin = np.ceil(0.05 * text_height) + draw.rectangle( + [(left, text_bottom - text_height - 2 * margin), (left + text_width, + text_bottom)], + fill=color) + draw.text( + (left + margin, text_bottom - text_height - margin), + display_str, + fill='black', + font=font) + text_bottom -= text_height - 2 * margin + + +def draw_keypoints_on_image_array(image, + key_points, + keypoint_scores=None, + min_score_thresh=0.5, + color='red', + radius=2, + use_normalized_coordinates=True, + keypoint_edges=None, + keypoint_edge_color='green', + keypoint_edge_width=2): + """Draws keypoints on an image (numpy array). + Args: + image: a numpy array with shape [height, width, 3]. + key_points: a numpy array with shape [num_keypoints, 2]. + keypoint_scores: a numpy array with shape [num_keypoints]. If provided, only + those keypoints with a score above score_threshold will be visualized. + min_score_thresh: A scalar indicating the minimum keypoint score required + for a keypoint to be visualized. Note that keypoint_scores must be + provided for this threshold to take effect. + color: color to draw the keypoints with. Default is red. + radius: keypoint radius. Default value is 2. + use_normalized_coordinates: if True (default), treat keypoint values as + relative to the image. Otherwise treat them as absolute. + keypoint_edges: A list of tuples with keypoint indices that specify which + keypoints should be connected by an edge, e.g. [(0, 1), (2, 4)] draws + edges from keypoint 0 to 1 and from keypoint 2 to 4. + keypoint_edge_color: color to draw the keypoint edges with. Default is red. + keypoint_edge_width: width of the edges drawn between keypoints. Default + value is 2. + """ + image_pil = Image.fromarray(np.uint8(image)).convert('RGB') + draw_keypoints_on_image(image_pil, + key_points, + keypoint_scores=keypoint_scores, + min_score_thresh=min_score_thresh, + color=color, + radius=radius, + use_normalized_coordinates=use_normalized_coordinates, + keypoint_edges=keypoint_edges, + keypoint_edge_color=keypoint_edge_color, + keypoint_edge_width=keypoint_edge_width) + np.copyto(image, np.array(image_pil)) + + +def draw_keypoints_on_image(image, + key_points, + keypoint_scores=None, + min_score_thresh=0.5, + color='red', + radius=2, + use_normalized_coordinates=True, + keypoint_edges=None, + keypoint_edge_color='green', + keypoint_edge_width=2): + """Draws keypoints on an image. + Args: + image: a PIL.Image object. + key_points: a numpy array with shape [num_keypoints, 2]. + keypoint_scores: a numpy array with shape [num_keypoints]. + min_score_thresh: a score threshold for visualizing keypoints. Only used if + keypoint_scores is provided. + color: color to draw the keypoints with. Default is red. + radius: keypoint radius. Default value is 2. + use_normalized_coordinates: if True (default), treat keypoint values as + relative to the image. Otherwise treat them as absolute. + keypoint_edges: A list of tuples with keypoint indices that specify which + keypoints should be connected by an edge, e.g. [(0, 1), (2, 4)] draws + edges from keypoint 0 to 1 and from keypoint 2 to 4. + keypoint_edge_color: color to draw the keypoint edges with. Default is red. + keypoint_edge_width: width of the edges drawn between keypoints. Default + value is 2. + """ + draw = ImageDraw.Draw(image) + im_width, im_height = image.size + key_points = np.array(key_points) + key_points_x = [k[1] for k in key_points] + key_points_y = [k[0] for k in key_points] + if use_normalized_coordinates: + key_points_x = tuple([im_width * x for x in key_points_x]) + key_points_y = tuple([im_height * y for y in key_points_y]) + if keypoint_scores is not None: + keypoint_scores = np.array(keypoint_scores) + valid_kpt = np.greater(keypoint_scores, min_score_thresh) + else: + valid_kpt = np.where(np.any(np.isnan(key_points), axis=1), + np.zeros_like(key_points[:, 0]), + np.ones_like(key_points[:, 0])) + valid_kpt = [v for v in valid_kpt] + + for keypoint_x, keypoint_y, valid in zip(key_points_x, key_points_y, valid_kpt): + if valid: + draw.ellipse([(keypoint_x - radius, keypoint_y - radius), + (keypoint_x + radius, keypoint_y + radius)], + outline=color, fill=color) + if keypoint_edges is not None: + for keypoint_start, keypoint_end in keypoint_edges: + if (keypoint_start < 0 or keypoint_start >= len(key_points) or + keypoint_end < 0 or keypoint_end >= len(key_points)): + continue + if not (valid_kpt[keypoint_start] and valid_kpt[keypoint_end]): + continue + edge_coordinates = [ + key_points_x[keypoint_start], key_points_y[keypoint_start], + key_points_x[keypoint_end], key_points_y[keypoint_end] + ] + draw.line( + edge_coordinates, fill=keypoint_edge_color, width=keypoint_edge_width) + + +def draw_mask_on_image_array(image, mask, color='red', alpha=0.4): + """Draws mask on an image. + Args: + image: uint8 numpy array with shape (img_height, img_height, 3) + mask: a uint8 numpy array of shape (img_height, img_height) with + values between either 0 or 1. + color: color to draw the keypoints with. Default is red. + alpha: transparency value between 0 and 1. (default: 0.4) + Raises: + ValueError: On incorrect data type for image or masks. + """ + if image.dtype != np.uint8: + raise ValueError('`image` not of type np.uint8') + if mask.dtype != np.uint8: + raise ValueError('`mask` not of type np.uint8') + if np.any(np.logical_and(mask != 1, mask != 0)): + raise ValueError('`mask` elements should be in [0, 1]') + if image.shape[:2] != mask.shape: + raise ValueError('The image has spatial dimensions %s but the mask has ' + 'dimensions %s' % (image.shape[:2], mask.shape)) + rgb = ImageColor.getrgb(color) + pil_image = Image.fromarray(image) + + solid_color = np.expand_dims( + np.ones_like(mask), axis=2) * np.reshape(list(rgb), [1, 1, 3]) + pil_solid_color = Image.fromarray(np.uint8(solid_color)).convert('RGBA') + pil_mask = Image.fromarray(np.uint8(255.0 * alpha * mask)).convert('L') + pil_image = Image.composite(pil_solid_color, pil_image, pil_mask) + np.copyto(image, np.array(pil_image.convert('RGB'))) + + +def visualize_boxes_and_labels_on_image_array( + image, + boxes, + classes, + scores, + category_index, + instance_masks=None, + instance_boundaries=None, + keypoints=None, + keypoint_scores=None, + keypoint_edges=None, + track_ids=None, + use_normalized_coordinates=False, + max_boxes_to_draw=20, + min_score_thresh=.5, + agnostic_mode=False, + line_thickness=4, + groundtruth_box_visualization_color='black', + skip_boxes=False, + skip_scores=False, + skip_labels=False, + skip_track_ids=False): + """Overlay labeled boxes on an image with formatted scores and label names. + This function groups boxes that correspond to the same location + and creates a display string for each detection and overlays these + on the image. Note that this function modifies the image in place, and returns + that same image. + Args: + image: uint8 numpy array with shape (img_height, img_width, 3) + boxes: a numpy array of shape [N, 4] + classes: a numpy array of shape [N]. Note that class indices are 1-based, + and match the keys in the label map. + scores: a numpy array of shape [N] or None. If scores=None, then + this function assumes that the boxes to be plotted are groundtruth + boxes and plot all boxes as black with no classes or scores. + category_index: a dict containing category dictionaries (each holding + category index `id` and category name `name`) keyed by category indices. + instance_masks: a numpy array of shape [N, image_height, image_width] with + values ranging between 0 and 1, can be None. + instance_boundaries: a numpy array of shape [N, image_height, image_width] + with values ranging between 0 and 1, can be None. + keypoints: a numpy array of shape [N, num_keypoints, 2], can + be None. + keypoint_scores: a numpy array of shape [N, num_keypoints], can be None. + keypoint_edges: A list of tuples with keypoint indices that specify which + keypoints should be connected by an edge, e.g. [(0, 1), (2, 4)] draws + edges from keypoint 0 to 1 and from keypoint 2 to 4. + track_ids: a numpy array of shape [N] with unique track ids. If provided, + color-coding of boxes will be determined by these ids, and not the class + indices. + use_normalized_coordinates: whether boxes is to be interpreted as + normalized coordinates or not. + max_boxes_to_draw: maximum number of boxes to visualize. If None, draw + all boxes. + min_score_thresh: minimum score threshold for a box or keypoint to be + visualized. + agnostic_mode: boolean (default: False) controlling whether to evaluate in + class-agnostic mode or not. This mode will display scores but ignore + classes. + line_thickness: integer (default: 4) controlling line width of the boxes. + groundtruth_box_visualization_color: box color for visualizing groundtruth + boxes + skip_boxes: whether to skip the drawing of bounding boxes. + skip_scores: whether to skip score when drawing a single detection + skip_labels: whether to skip label when drawing a single detection + skip_track_ids: whether to skip track id when drawing a single detection + Returns: + uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes. + """ + # Create a display string (and color) for every box location, group any boxes + # that correspond to the same location. + box_to_display_str_map = collections.defaultdict(list) + box_to_color_map = collections.defaultdict(str) + box_to_instance_masks_map = {} + box_to_instance_boundaries_map = {} + box_to_keypoints_map = collections.defaultdict(list) + box_to_keypoint_scores_map = collections.defaultdict(list) + box_to_track_ids_map = {} + is_crack = False + if not max_boxes_to_draw: + max_boxes_to_draw = boxes.shape[0] + for i in range(boxes.shape[0]): + if max_boxes_to_draw == len(box_to_color_map): + break + if scores is None or scores[i] > min_score_thresh: + box = tuple(boxes[i].tolist()) + if instance_masks is not None: + box_to_instance_masks_map[box] = instance_masks[i] + if instance_boundaries is not None: + box_to_instance_boundaries_map[box] = instance_boundaries[i] + if keypoints is not None: + box_to_keypoints_map[box].extend(keypoints[i]) + if keypoint_scores is not None: + box_to_keypoint_scores_map[box].extend(keypoint_scores[i]) + if track_ids is not None: + box_to_track_ids_map[box] = track_ids[i] + if scores is None: + box_to_color_map[box] = groundtruth_box_visualization_color + else: + display_str = '' + if not skip_labels: + if not agnostic_mode: + if classes[i] in six.viewkeys(category_index): + class_name = category_index[classes[i]]['name'] + else: + class_name = 'N/A' + display_str = str(class_name) + if not skip_scores: + if not display_str: + display_str = '{}%'.format(round(100 * scores[i])) + else: + display_str = '{}: {}%'.format(display_str, round(100 * scores[i])) + if not skip_track_ids and track_ids is not None: + if not display_str: + display_str = 'ID {}'.format(track_ids[i]) + else: + display_str = '{}: ID {}'.format(display_str, track_ids[i]) + box_to_display_str_map[box].append(display_str) + if agnostic_mode: + box_to_color_map[box] = 'DarkOrange' + elif track_ids is not None: + prime_multipler = _get_multiplier_for_color_randomness() + box_to_color_map[box] = STANDARD_COLORS[ + (prime_multipler * track_ids[i]) % len(STANDARD_COLORS)] + else: + box_to_color_map[box] = STANDARD_COLORS[ + classes[i] % len(STANDARD_COLORS)] + is_crack = len(box_to_color_map) > 0 + # Draw all boxes onto image. + for box, color in box_to_color_map.items(): + ymin, xmin, ymax, xmax = box + if instance_masks is not None: + draw_mask_on_image_array( + image, + box_to_instance_masks_map[box], + color=color + ) + if instance_boundaries is not None: + draw_mask_on_image_array( + image, + box_to_instance_boundaries_map[box], + color='red', + alpha=1.0 + ) + draw_bounding_box_on_image_array( + image, + ymin, + xmin, + ymax, + xmax, + color=color, + thickness=0 if skip_boxes else line_thickness, + display_str_list=box_to_display_str_map[box], + use_normalized_coordinates=use_normalized_coordinates) + if keypoints is not None: + keypoint_scores_for_box = None + if box_to_keypoint_scores_map: + keypoint_scores_for_box = box_to_keypoint_scores_map[box] + draw_keypoints_on_image_array( + image, + box_to_keypoints_map[box], + keypoint_scores_for_box, + min_score_thresh=min_score_thresh, + color=color, + radius=line_thickness / 2, + use_normalized_coordinates=use_normalized_coordinates, + keypoint_edges=keypoint_edges, + keypoint_edge_color=color, + keypoint_edge_width=line_thickness // 2) + + return image, is_crack diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..7224b74 --- /dev/null +++ b/build.sh @@ -0,0 +1,3 @@ +# /usr/bash + +docker build --tag hpds-python . diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..548a498 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,48 @@ +absl-py==0.9.0 +astunparse==1.6.3 +cachetools==4.1.0 +certifi==2020.4.5.1 +chardet==3.0.4 +click==7.1.2 +cycler==0.10.0 +Flask==1.1.2 +gast==0.3.3 +gevent==20.5.0 +google-auth==1.15.0 +google-auth-oauthlib==0.4.1 +google-pasta==0.2.0 +greenlet==0.4.15 +grpcio==1.29.0 +gunicorn==20.0.4 +h5py==2.10.0 +idna==2.9 +importlib-metadata==1.6.0 +itsdangerous==1.1.0 +Jinja2==2.11.2 +Keras-Preprocessing==1.1.2 +Markdown==3.2.2 +MarkupSafe==1.1.1 +matplotlib==3.2.1 +numpy==1.18.4 +oauthlib==3.1.0 +opt-einsum==3.2.1 +Pillow==7.1.2 +protobuf==3.11.3 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +requests==2.23.0 +requests-oauthlib==1.3.0 +rsa==4.0 +scipy==1.4.1 +six==1.15.0 +tensorboard==2.2.1 +tensorboard-plugin-wit==1.6.0.post3 +tensorflow==2.2.0 +tensorflow-estimator==2.2.0 +termcolor==1.1.0 +urllib3==1.25.9 +Werkzeug==1.0.1 +wrapt==1.12.1 +zipp==3.1.0